We Keep Coding

I can't pass parameters to foreach loop while implementing Structured Streaming + Kafka in Spark SQL

I followed the instructions at Structured Streaming + Kafka and built a program that receives data streams sent from kafka as input, when I receive the data stream I want to pass it to SparkSession variable to do some query work with Spark SQL, so I extend the ForeachWriter class again as follows:
package stream;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import org.apache.spark.sql.ForeachWriter;
import org.apache.spark.sql.SparkSession;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import dataservices.OrderDataServices;
import models.SuccessEvent;
public class MapEventWriter extends ForeachWriter<String>{
private SparkSession spark;
public MapEventWriter(SparkSession spark) {
this.spark = spark;
}
private static final long serialVersionUID = 1L;
#Override
public void close(Throwable errorOrNull) {
// TODO Auto-generated method stub
}
#Override
public boolean open(long partitionId, long epochId) {
// TODO Auto-generated method stub
return true;
}
#Override
public void process(String input) {
OrderDataServices services = new OrderDataServices(this.spark);
}
}
however in the process function, if I use spark variable, the program gives an error, the program passes in my spark as follows:
package demo;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.concurrent.TimeoutException;
import org.apache.hadoop.fs.Path;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.streaming.StreamingQuery;
import org.apache.spark.sql.streaming.StreamingQueryException;
import org.json.simple.parser.ParseException;
import dataservices.OrderDataServices;
import models.MapperEvent;
import models.OrderEvent;
import models.SuccessEvent;
import stream.MapEventWriter;
import stream.MapEventWriter1;
public class Demo {
public static void main(String[] args) throws TimeoutException, StreamingQueryException, ParseException, IOException {
try (SparkSession spark = SparkSession.builder().appName("Read kafka").getOrCreate()) {
Dataset<String> data = spark
.readStream()
.format("kafka")
.option("kafka.bootstrap.servers", "localhost:9092")
.option("subscribe", "tiki-1")
.load()
.selectExpr("CAST(value AS STRING)")
.as(Encoders.STRING());
MapEventWriter eventWriter = new MapEventWriter(spark);
StreamingQuery query = data
.writeStream()
.foreach(eventWriter)
.start();
query.awaitTermination();
}
}
}
The error is NullPointerException at the spark call location, that is, no spark variable is initialized.
Hope anyone can help me, I really appreciate it.
Caused by: java.lang.NullPointerException
at org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:151)
at org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:149)
at org.apache.spark.sql.DataFrameReader.<init>(DataFrameReader.scala:998)
at org.apache.spark.sql.SparkSession.read(SparkSession.scala:655)
at dataservices.OrderDataServices.<init>(OrderDataServices.java:18)
at stream.MapEventWriter.process(MapEventWriter.java:38)
at stream.MapEventWriter.process(MapEventWriter.java:15)

do some query work with Spark SQL
You wouldn't use a ForEachWriter for that
.selectExpr("CAST(value AS STRING)")
.as(Encoders.STRING()); // or parse your JSON here using a schema
data.select(...) // or move this to a method / class that takes the Dataset as a parameter
// await termination

invalid hexadecimal representation of an ObjectId in Mongo DB Rest API

I am using REST API to fetch data using #Form param from Mongo DB and got the exception: 'Invalid hexadecimal representation of ObjectId'. The syntax seems to be correct, not sure whats going wrong there. I am passing new ObjectId (id) in the rest parameter. The code is as below:
//Country.java
package com.speed.infoaxon;
import java.io.IOException;
import java.net.UnknownHostException;
import org.bson.types.ObjectId;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
public class Country {
public BasicDBObject addDemo(long _id ) throws IOException {
DB db=ConnectToDB.getConnection();
DBCollection collection = db.getCollection("demo");
BasicDBObject buildList = null;
BasicDBObject document = new BasicDBObject();
document.put("_id",new ObjectId("id"));
collection.save(document);
return buildList;
}
}
//getResponse.java
package com.speed.infoaxon;
import java.io.IOException;
import java.net.UnknownHostException;
import javax.ws.rs.Consumes;
import javax.ws.rs.FormParam;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
#Path("/add")
public class GetResponse {
#POST
#Path("/addDemo")
#Consumes({MediaType.APPLICATION_JSON, MediaType.APPLICATION_FORM_URLENCODED} )
public BasicDBObject addDemo(#FormParam("_id") long _id) throws IOException
{
System.out.println("inside demo");
Country d = new Country();
BasicDBObject basicDBList=d.addDemo(_id);
return basicDBList;
}
}
Please let me know where is the issue. Thanks in advance.

document.put("_id",new ObjectId("id"));
you're using "id" in quotes which means its id in string you need to pass in the actual id

Strange Behaviour in Eclipse 3.8.1

I'm a newbie here. I have a simple problem in ONE java source file: the row System.out.pritln(...) has been treated as an erroneous expression. Here's the code snippet:
package vk.gui;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Properties;
import com.itextpdf.text.BadElementException;
import com.itextpdf.text.Chunk;
import com.itextpdf.text.Document;
import com.itextpdf.text.DocumentException;
import com.itextpdf.text.Font;
import com.itextpdf.text.Image;
import com.itextpdf.text.PageSize;
import com.itextpdf.text.Paragraph;
import com.itextpdf.text.Rectangle;
import com.itextpdf.text.pdf.BarcodeEAN;
import com.itextpdf.text.pdf.BaseFont;
import com.itextpdf.text.pdf.PdfContentByte;
import com.itextpdf.text.pdf.PdfPCell;
import com.itextpdf.text.pdf.PdfPCellEvent;
import com.itextpdf.text.pdf.PdfPTable;
import com.itextpdf.text.pdf.PdfWriter;
public class MatrixSheet1 {
Properties p;
File file;
Document document;
PdfWriter writer;
Image logo = null;
Image EANimg = null;
float mnoz = new Double(72/25.6).floatValue();
int IMG_WIDTH= new Double(35*mnoz).intValue();
int IMG_HEIGHT=new Double(35*mnoz).intValue();
String err=p.getProperty("cell.height");
System.out.println("Arrgh!"); ///-------------->ERROR!
float cell_Height = Float.parseFloat(p.getProperty("cell.height"))*mnoz;
float cell_Width = Float.parseFloat(p.getProperty("cell.width"))*mnoz;
The reported error is
Multiple markers at this line
Syntax error on token ""Arrgh!"", delete this token
Syntax error on token(s), misplaced construct(s)
The sout and sysout shortcuts do not work neither. In other existing source files of same package everything is OK, the shortcuts work and the expression does not trigger an error.
I tried to create another source file and copy/paste the content, but I got the same error. What and where went wrong?
I need the printing just for debugging, but this is a bit annoying symptom.
Thanks in advance.

This happens because you can use System.out.println() only inside methodes. If you would do something like this, it would work:
public class MatrixSheet1 {
Properties p;
File file;
Document document;
PdfWriter writer;
Image logo = null;
Image EANimg = null;
float mnoz = new Double(72/25.6).floatValue();
int IMG_WIDTH= new Double(35*mnoz).intValue();
int IMG_HEIGHT=new Double(35*mnoz).intValue();
String err=p.getProperty("cell.height");
systemMessage("Argh!");
float cell_Height = Float.parseFloat(p.getProperty("cell.height"))*mnoz;
float cell_Width = Float.parseFloat(p.getProperty("cell.width"))*mnoz;
private void systemMessage(String message){
System.out.println(message);
}
}

Java jax - rs program to accept files from external systems

I am trying to create a simple Java Jax-RS based webservice which accepts the file in a byte[] format or as a blob and pushes it to an FTP folder. This service is used by Salesforce to push files to directories via this Java API. For testing i have hosted this application in https://ftptransfer.herokuapp.com/myresource
I need to create a class which basically accepts files from external systems
package com.example;
import javax.ws.rs.Consumes;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import javax.ws.rs.POST;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataParam;
#Path("sendFiles")
public class FileTransfer {
#POST
#Consumes({MediaType.MULTIPART_FORM_DATA})
public String uploadPdfFile( )
{
return "Uploaded successfully";
}
}
Can some help with any good reference to achieve this, as i have tried many approaches from different blogs but none helped me much.

I am able to achieve this in the following way
import org.apache.commons.net.ftp.FTPClient;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import javax.ws.rs.Consumes;
import javax.ws.rs.Path;
import java.io.InputStream;
import javax.ws.rs.POST;
#Path("sendFiles")
public class FileTransfer {
#POST
#Path("file")
#Consumes("*/*")
public String uploadPdfFile(InputStream fileInputStream)
{
FTPClient client = new FTPClient();
boolean login;
FileInputStream fis = null;
int read = 0;
byte[] bytes = new byte[1024];
OutputStream out;
try {
File file=new File("testhm001.txt");
out = new FileOutputStream(file);
while ((read = fileInputStream.read(bytes)) != -1)
{
out.write(bytes, 0, read);
}
out.flush();
out.close();
InputStream in = new FileInputStream(file);
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
StringBuilder out1 = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
out1.append(line);
}
return "done";
}
}

UIMA Pipeline with Stanford NER

I have integrated Stanford NER in UIMA and developed a pipeline.
The pipeline contains a FileSystemCollectionReader,an NERAnnotator and a CasConsumer but the output so come isn't desired. In my input directory i have two files and after running the pipeline, i get two files as ouput but the second file is getting merged with the first file in second ouput. I don't know what's happening here.
The code for CasConsumer:
`
package org.gds.uima;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import org.apache.uima.UimaContext;
import org.apache.uima.analysis_component.AnalysisComponent_ImplBase;
import org.apache.uima.analysis_component.CasAnnotator_ImplBase;
import org.apache.uima.analysis_component.JCasAnnotator_ImplBase;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.cas.CAS;
import org.apache.uima.cas.CASException;
import org.apache.uima.cas.Type;
import org.apache.uima.cas.text.AnnotationFS;
import org.apache.uima.fit.component.CasConsumer_ImplBase;
import org.apache.uima.fit.component.JCasConsumer_ImplBase;
import org.apache.uima.fit.descriptor.ConfigurationParameter;
import org.apache.uima.fit.util.CasUtil;
import org.apache.uima.jcas.JCas;
import org.apache.uima.resource.ResourceInitializationException;
public class CasConsumer extends JCasConsumer_ImplBase
{
public final static String PARAM_OUTPUT="outputDir";
#ConfigurationParameter(name = PARAM_OUTPUT)
private String outputDirectory;
public final static String PARAM_ANNOTATION_TYPES = "annotationTypes";
enter code here
#ConfigurationParameter(name = PARAM_ANNOTATION_TYPES,defaultValue="String")
public List<String> annotationTypes;
public void initialize(final UimaContext context) throws ResourceInitializationException
{
super.initialize(context);
}
#Override
public void process(JCas jcas)
{
String original = jcas.getDocumentText();
try
{
String onlyText="";
JCas sofaText = jcas.getView(NERAnnotator.SOFA_NAME);
onlyText = sofaText.getDocumentText();
String name = UUID.randomUUID().toString().substring(20);
File outputDir = new File(this.outputDirectory+"/"+name);
System.out.print("Saving file to "+outputDir.getAbsolutePath());
FileOutputStream fos = new FileOutputStream(outputDir.getAbsoluteFile());
PrintWriter pw = new PrintWriter(fos);
pw.println(onlyText);
pw.close();
}
catch(CASException cae)
{
System.out.println(cae);
}
catch(FileNotFoundException fne)
{
System.out.print(fne);
}
}
}
`
}