We Keep Coding

How to use TwitterObjectFactory.jsonStoreEnabled to get a tweet as JSON?

I seem to be using a deprecated API:
thufir#dur:~/NetBeansProjects/twitterBaseX$
thufir#dur:~/NetBeansProjects/twitterBaseX$ gradle clean run
> Task :compileJava
Note: /home/thufir/NetBeansProjects/twitterBaseX/src/main/java/twitterBaseX/App.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
BUILD SUCCESSFUL in 2s
4 actionable tasks: 4 executed
<-------------> 0% WAITING
> IDLE
thufir#dur:~/NetBeansProjects/twitterBaseX$
code:
package twitterBaseX;
import java.io.IOException;
import java.util.Properties;
import java.util.Set;
import java.util.logging.Logger;
import twitter4j.JSONException;
import twitter4j.JSONObject;
import twitter4j.Query;
import twitter4j.QueryResult;
import twitter4j.Status;
import twitter4j.Twitter;
import twitter4j.TwitterException;
import twitter4j.TwitterFactory;
import twitter4j.conf.ConfigurationBuilder;
import twitter4j.json.DataObjectFactory;
public class App {
private static final Logger log = Logger.getLogger(App.class.getName());
private Properties loadProperties(String filePrefix) throws IOException {
Properties properties = new Properties();
properties.loadFromXML(App.class.getResourceAsStream("/" + filePrefix + ".xml"));
Set<Object> keySet = properties.keySet();
String key = null;
String value = null;
for (Object obj : keySet) {
key = obj.toString();
value = System.getenv(key);
properties.setProperty(key, value);
}
return properties;
}
private TwitterFactory configTwitterFactory(Properties properties) throws IOException {
ConfigurationBuilder configurationBuilder = new ConfigurationBuilder();
configurationBuilder.setDebugEnabled(true)
.setJSONStoreEnabled(true)
.setOAuthConsumerKey(properties.getProperty("oAuthConsumerKey"))
.setOAuthConsumerSecret(properties.getProperty("oAuthConsumerSecret"))
.setOAuthAccessToken(properties.getProperty("oAuthAccessToken"))
.setOAuthAccessTokenSecret(properties.getProperty("oAuthAccessTokenSecret"));
return new TwitterFactory(configurationBuilder.build());
}
private void twitterQueryResult() throws TwitterException, IOException, JSONException {
Twitter twitter = configTwitterFactory(loadProperties("twitter")).getInstance();
Query query = new Query("lizardbill");
QueryResult result = twitter.search(query);
String string = null;
JSONObject JSON_complete = null;
for (Status status : result.getTweets()) {
insertStatus(status);
}
}
/*
// Content of the new document
String doc = "<xml>Hello World!</xml>";
System.out.println("\n* Create new resource.");
// Create a new XML resource with the specified ID
XMLResource res = (XMLResource) col.createResource(id,
XMLResource.RESOURCE_TYPE);
// Set the content of the XML resource as the document
res.setContent(doc);
System.out.println("\n* Store new resource.");
// Store the resource into the database
col.storeResource(res);
*/
private void insertStatus(Status status) throws JSONException {
String string = DataObjectFactory.getRawJSON(status);
JSONObject json = new JSONObject(string);
String language = json.getString("lang");
log.fine(json.toString());
// XMLResource res = (XMLResource) col.createResource(id, XMLResource.RESOURCE_TYPE);
}
public static void main(String[] args) throws TwitterException, IOException, JSONException {
new App().twitterQueryResult();
}
}
really, more looking for xml -- but json will suffice.

This seems to not use deprecated methods:
private void insertStatus(Status status) throws JSONException {
String string = TwitterObjectFactory.getRawJSON(status);
JSONObject json = new JSONObject(string);
String language = json.getString("lang");
log.info(language);
// XMLResource res = (XMLResource) col.createResource(id, XMLResource.RESOURCE_TYPE);
}

I am trying to use a url xml parse but it looks like i keep getting an empty xml

I am trying to get info from a weather API called even though when i am making the request i am getting a response, but when i am trying to get only a specific part of the response i get null response every time can someone help? here is the code for my handler :
package weathercalls;
import java.util.ArrayList;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
public class Handler extends DefaultHandler
{
// Create three array lists to store the data
public ArrayList<Integer> lows = new ArrayList<Integer>();
public ArrayList<Integer> highs = new ArrayList<Integer>();
public ArrayList<String> regions = new ArrayList<String>();
// Make sure that the code in DefaultHandler's
// constructor is called:
public Handler()
{
super();
}
/*** Below are the three methods that we are extending ***/
#Override
public void startDocument()
{
System.out.println("Start document");
}
#Override
public void endDocument()
{
System.out.println("End document");
}
// This is where all the work is happening:
#Override
public void startElement(String uri, String name, String qName, Attributes atts)
{
if(qName.compareTo("region") == 0)
{
String region = atts.getLocalName(0);
System.out.println("Day: " + region);
this.regions.add(region);
}
if(qName.compareToIgnoreCase("wind_degree") == 0)
{
int low = atts.getLength();
System.out.println("Low: " + low);
this.lows.add(low);
}
if(qName.compareToIgnoreCase("high") == 0)
{
int high = Integer.parseInt(atts.getValue(0));
System.out.println("High: " + high);
this.highs.add(high);
}
}
}
and here is my main file code :
package weathercalls;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import org.xml.sax.InputSource;
import org.xml.sax.XMLReader;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.xml.sax.SAXException;
public class weatherCalls {
public static void main(String[] args) throws Exception {
//Main url
String main_url = "http://api.weatherapi.com/v1/";
//Live or Weekly forecast
String live_weather = "current.xml?key=";
//String sevendays_weather = "orecast.xml?key=";
//API Key + q
String API_Key = "c2e285e55db74def97f151114201701&q=";
//Location Setters
String location = "London";
InputSource inSource = null;
InputStream in = null;
XMLReader xr = null;
/**
URL weather = new URL(main_url + live_weather + API_Key + location);
URLConnection yc = weather.openConnection();
BufferedReader in1 = new BufferedReader(
new InputStreamReader(
yc.getInputStream()));
String inputLine;
while ((inputLine = in1.readLine()) != null)
System.out.println(inputLine);
in1.close();**/
try
{
// Turn the string into a URL object
String complete_url = main_url + live_weather + API_Key + location;
URL urlObject = new URL(complete_url);
// Open the stream (which returns an InputStream):
in = urlObject.openStream();
/** Now parse the data (the stream) that we received back ***/
// Create an XML reader
SAXParserFactory parserFactory = SAXParserFactory.newInstance();
SAXParser parser = parserFactory.newSAXParser();
xr = parser.getXMLReader();
// Tell that XML reader to use our special Google Handler
Handler ourSpecialHandler = new Handler();
xr.setContentHandler(ourSpecialHandler);
// We have an InputStream, but let's just wrap it in
// an InputSource (the SAX parser likes it that way)
inSource = new InputSource(in);
// And parse it!
xr.parse(inSource);
System.out.println(complete_url);
System.out.println(urlObject);
System.out.println(in);
System.out.println(xr);
System.out.println(inSource);
System.out.println(parser);
}
catch(IOException ioe)
{
ioe.printStackTrace();
}
catch(SAXException se)
{
se.printStackTrace();
}
}
}
and this is my console print:
Start document
Day: null
Low: 0
End document
http://api.weatherapi.com/v1/current.xml?key=c2e285e55db74def97f151114201701&q=London
http://api.weatherapi.com/v1/current.xml?key=c2e285e55db74def97f151114201701&q=London
sun.net.www.protocol.http.HttpURLConnection$HttpInputStream#2471cca7
com.sun.org.apache.xerces.internal.jaxp.SAXParserImpl$JAXPSAXParser#5fe5c6f
org.xml.sax.InputSource#6979e8cb
com.sun.org.apache.xerces.internal.jaxp.SAXParserImpl#763d9750

I think you are trying to extract the values from the XML tags and if it is the case then you are doing it wrong. Attributes object contains the attributes of a particular tag and to get the value you have to do some extra work. Similar to the start of a tag, there are separate events for the contents and the end of a tag. current_tag variable will keep track of the current tag being processed. Below is a sample code:
class Handler extends DefaultHandler {
// Create three array lists to store the data
public ArrayList<Integer> lows = new ArrayList<Integer>();
public ArrayList<Integer> highs = new ArrayList<Integer>();
public ArrayList<String> regions = new ArrayList<String>();
// Make sure that the code in DefaultHandler's
// constructor is called:
public Handler() {
super();
}
/*** Below are the three methods that we are extending ***/
#Override
public void startDocument() {
System.out.println("Start document");
}
#Override
public void endDocument() {
System.out.println("End document");
}
//Keeps track of the current tag;
String currentTag = "";
// This is where all the work is happening:
#Override
public void startElement(String uri, String name, String qName, Attributes atts) {
//Save the current tag being handled
currentTag = qName;
}
//Detect end tag
#Override
public void endElement(String uri, String localName, String qName) throws SAXException {
//Reset it
currentTag = "";
}
#Override
public void characters(char[] ch, int start, int length) throws SAXException {
//Rules based on current tag
switch (currentTag) {
case "region":
String region = String.valueOf(ch, start, length);
this.regions.add(region);
System.out.println("Day: " + region);
break;
case "wind_degree":
int low = Integer.parseInt(String.valueOf(ch, start, length));
System.out.println("Low: " + low);
this.lows.add(low);
break;
case "high":
int high = Integer.parseInt(String.valueOf(ch, start, length));
System.out.println("High: " + high);
this.highs.add(high);
break;
}
}}
NOTE: Please refrain from sharing your API keys or passwords on the internet.

Batch Insert In Cassandra using Apache Spark hanging and context not getting closed when triggered from a Web Ser

I am a new to spark . I am trying to insert csv files into cassandra table using spark-cassandra connector as below:
The files are in Hdfs and I am getting the Paths of all files and for each path I call a method which does converts the csv data to corressponding cassandra data types and creates a prepared statement binds the data to the prepared statement and adds it to a batch. Finally I execute the batch when its 1000.
Key Points
1. I am using Apache Cassandra 2.1.8 and Spark 1.5
2. I read the Csv files using Spark Context
3. I am using the com.datastax.spark.connector.cql.CassandraConnector to create a Session with Cassandra.
I have 9 Files , each files data goes to a table in cassandra. Every Things works fine All inserts are happening as expected and the job gets completed when I submit the jar on spark submit.
The problem I am facing is When the same Jar is invoked through a web-service (web service calls the script to invoke the jar) One of the files data doesn’t get inserted and the spark context doesn’t stop due to which the jobs is forever running.
When I insert 4 files or 5 files everything works fine even through the web-service. But all together it hanging and I get 10 records less in one of the tables and context doesn’t stop.
Its strange because when I am submitting the jar on the spark submit directly everything works fine and through the web service I face this issue , Its strange bcz even the web-service submits the job to the same spark submit.
Here is my code
package com.pz.loadtocassandra;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.ConsoleHandler;
import java.util.logging.FileHandler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.PreparedStatement;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.exceptions.InvalidTypeException;
import com.datastax.spark.connector.cql.CassandraConnector;
import com.datastax.spark.connector.japi.CassandraRow;
import com.pz.shared.UnicodeBOMInputStream;
import com.pz.shared.fileformat.Header;
import com.pz.shared.mr.fileformat.MRFileFormats.CSVInputFormat;
import com.pz.shared.mr.fileformat.MRFileFormats.TextArrayWritable;
public class LoadToCassandra {
public static final String STUDYID = "STUDYID";
public static final String PROJECTNAME = "PROJECTNAME";
public static final String FILEID = "FILEID";
public static int count = 0;
public static final String FILE_SERPERATOR = "/";
public static Logger log = Logger.getLogger(LoadToCassandra.class.getName());
public static void main(String[] args) {
String propFileLoc = args[0];
String hdfsHome = args[1];
String hdfs_DtdXmlPath = args[2];
String hdfs_NormalizedDataPath = args[3];
run(propFileLoc, hdfsHome, hdfs_DtdXmlPath,hdfs_NormalizedDataPath);
} catch (IOException exception) {
log.log(Level.SEVERE, "Error occur in FileHandler.", exception);
}
}
public static void run(String propFileLoc, String hdfsHome,
String hdfs_DtdXmlPath, String hdfs_NormalizedDataPath) {
JavaSparkContext ctx = null;
FileSystem hadoopFs = null;
try {
PropInitialize.initailizeConfig(propFileLoc);
//setting spark context
ctx = setSparkContext(propFileLoc);
ParseDtdXml.parseDTDXML(hdfsHome, hdfs_DtdXmlPath);
Configuration configuration = setHadoopConf();
hadoopFs = getHadoopFs(hdfsHome, configuration);
FileStatus[] fstat = hadoopFs.listStatus(new Path(hdfs_NormalizedDataPath));
//Getting the csv paths
Path[] paths = FileUtil.stat2Paths(fstat);
log.info("PATH.size - " + paths.length);
for (Path path : paths) {
log.info("path is : "+path.toString());
loadToCassandra(propFileLoc, path, configuration,hdfsHome, ctx);
}
} catch (IOException | URISyntaxException e) {
log.log(Level.SEVERE, "run method", e);
e.printStackTrace();
} finally {
log.info("finally ");
if (ctx!= null) {
ctx.stop();
System.out.println("SC Stopped");
}
if (hadoopFs != null) {
try {
hadoopFs.close();
} catch (IOException e) {
log.log(Level.SEVERE, "run method", e);
}
}
}
}
// input : 1. String hdfs home ,
// 2. Configuration hadoop conf object
// returns : hadoop File System object
private static FileSystem getHadoopFs(String hdfsHome,
Configuration configuration) throws IOException, URISyntaxException {
return FileSystem.get(new URI(hdfsHome), configuration);
}
// input : no inputs
// process : sets hadoop config parameters
// output : retuns hadoop conf object
private static Configuration setHadoopConf() throws IOException,
URISyntaxException {
Configuration configuration = new Configuration();
configuration.setBoolean("csvFileFormat.encoded.flag", true);
configuration.set("csvinputformat.token.delimiter", ",");
return configuration;
}
// input : string Properties File Location
// process : creates and sets the configurations of spark context
// retuns : JavaSparkContext object with configurations set to it.
private static JavaSparkContext setSparkContext(String propFileLoc) {
PropInitialize.initailizeConfig(propFileLoc);
SparkConf conf = new SparkConf();
conf.set("spark.serializer",
"org.apache.spark.serializer.KryoSerializer");
conf.setAppName("Loading Data");
conf.setMaster(PropInitialize.spark_master);
conf.set("spark.cassandra.connection.host",
PropInitialize.cassandra_hostname);
conf.setJars(PropInitialize.external_jars);
return new JavaSparkContext(conf);
}
private static void loadToCassandra(String propFileLoc, Path sourceFileHdfsPath,
Configuration hadoopConf, String hdfsHome,JavaSparkContext ctx) {
System.out.println("File :: " + sourceFileHdfsPath.toString());
FileSystem hadoopFs = null;
PropInitialize.initailizeConfig(propFileLoc);
String cassKeyspaceName = PropInitialize.cass_keyspace_name;
log.info("entered here for file "+sourceFileHdfsPath.toString());
final String strInputFileName = StringUtils.split(
sourceFileHdfsPath.getName(), "#")[0].toLowerCase();
final String strTableNameInCass = StringUtils.split(
sourceFileHdfsPath.getName(), "-")[0].split("#")[1]
.toLowerCase();
final String strSourceFilePath = sourceFileHdfsPath.toString();
try {
hadoopFs = getHadoopFs(hdfsHome, hadoopConf);
//getting the cassandra connection using spark conf
final CassandraConnector connector = getCassandraConnection(ctx);
final JavaRDD<CassandraRow> cassTableObj=getCassTableObj(ctx,cassKeyspaceName,strTableNameInCass);
final Map<String, List<String>> tabColMapWithColTypes1 = ParseDtdXml.tabColMapWithColTypes;
final String headersUpdated;
final String headers;
UnicodeBOMInputStream ubis = new UnicodeBOMInputStream(
hadoopFs.open(sourceFileHdfsPath));
Header CsvHeader = Header.getCSVHeader(ubis, ",");
if (!strTableNameInCass.equalsIgnoreCase("PCMASTER")) {
String fString = "";
for (int i = 0; i < CsvHeader.size() - 1; i++) {
fString = fString + CsvHeader.get(i).ColumnName + ",";
}
fString = fString
+ CsvHeader.get(CsvHeader.size() - 1).ColumnName;
headers = fString; // StringUtils.join(stringArr.toString(),",");
headersUpdated = strTableNameInCass.toUpperCase() + "ID,"
+ headers;
} else {
String[] stringArr = new String[CsvHeader.size()];
String fString = "";
for (int i = 0; i < CsvHeader.size() - 1; i++) {
// stringArr[i] = CsvHeader.get(i).ColumnName;
fString = fString + CsvHeader.get(i).ColumnName + ",";
}
fString = fString
+ CsvHeader.get(CsvHeader.size() - 1).ColumnName;
headers = StringUtils.join(stringArr.toString(), ",");
headersUpdated = fString;
}
ubis.close();
//Reading the file using spark context
JavaPairRDD<LongWritable, TextArrayWritable> fileRdd = ctx
.newAPIHadoopFile(strSourceFilePath, CSVInputFormat.class,
LongWritable.class, TextArrayWritable.class,
hadoopConf);
final long recCount = fileRdd.count();
final String[] actCols = headersUpdated.split(",");
final LinkedHashMap<Object, String> mapOfColNameAndType = new LinkedHashMap<Object, String>();
final List<String> colNameAndType = tabColMapWithColTypes1
.get(strTableNameInCass.toUpperCase());
for (int i = 0; i < actCols.length; i++) {
if (colNameAndType.contains(actCols[i] + " " + "text")) {
int indexOfColName = colNameAndType.indexOf(actCols[i]
+ " " + "text");
mapOfColNameAndType.put(i,
colNameAndType.get(indexOfColName).split(" ")[1]);
} else if (colNameAndType
.contains(actCols[i] + " " + "decimal")) {
int indexOfColName = colNameAndType.indexOf(actCols[i]
+ " " + "decimal");
mapOfColNameAndType.put(i,
colNameAndType.get(indexOfColName).split(" ")[1]);
} else {
continue;
}
}
//creates the query for prepared statement
final String makeStatement = makeSt(cassKeyspaceName,
strTableNameInCass, actCols);
final long seqId1 = cassTableObj.count();
//calling map on the fileRdd
JavaRDD<String> data = fileRdd.values().map(
new Function<TextArrayWritable, String>() {
/**
*
*/
private static final long serialVersionUID = 1L;
Session session;
boolean isssession = false;
PreparedStatement statement;
BatchStatement batch;
int lineCount = 0;
long seqId = seqId1;
/*for each line returned as an TextArrayWritable convert each cell the corresponding
* bind the data to prepared statement
* add it to batch
*/
#Override
public String call(TextArrayWritable tup)
throws Exception {
seqId++;
lineCount++;
log.info("entered here 3 for file "+strSourceFilePath.toString());
String[] part = tup.toStrings();
Object[] parts = getDataWithUniqueId(
strTableNameInCass, part);
//For each file
//Creates the session
//creates the PreparedStatement
if (!isssession) {
session = connector.openSession();
statement = session.prepare(makeStatement);
log.info("entered here 4 for file "+strSourceFilePath.toString());
// System.out.println("statement :" +
// statement);
isssession = true;
batch = new BatchStatement();
}
List<Object> typeConvData = new ArrayList<Object>();
for (int i = 0; i < parts.length; i++) {
String type = mapOfColNameAndType.get(i);
try {
if (type.equalsIgnoreCase("text")) {
typeConvData.add(parts[i]);
} else {
// parts[i] =
// parts[i].toString().replace("\"",
// "");
// check if the String the has to
// converted to a BigDecimal is any
// positive or negetive integer or not.
// if its not a positive integer or
// negative forcefully convert it to
// zero (avoiding NumberFormatException)
if (!((String) parts[i])
.matches("-?\\d+")) {
parts[i] = "0";
}
long s = Long
.valueOf((String) parts[i]);
typeConvData.add(BigDecimal.valueOf(s));
}
} catch (NullPointerException e) {
log.log(Level.SEVERE, "loadToCass method", e);
} catch (NumberFormatException e) {
log.log(Level.SEVERE, "loadToCass method", e);
} catch (InvalidTypeException e) {
log.log(Level.SEVERE, "loadToCass method", e);
}
}
List<Object> data = typeConvData;
//bind data to query
final BoundStatement query = statement.bind(data
.toArray(new Object[data.size()]));
//add query to batch
batch.add(query);
int count = LoadToCassandra.count;
//when count is 1k execute batch
if (count == 1000) {
log.info("entered here 5 for file "+strSourceFilePath.toString());
log.info("batch done");
session.execute(batch);
LoadToCassandra.count = 0;
batch = new BatchStatement();
return StringUtils.join(tup.toStrings());
}
//if its the last batch and its not of size 1k
if (lineCount == (recCount))
{
log.info("Last Batch");
session.executeAsync(batch);
log.info("entered here 6 for file "+strSourceFilePath.toString());
//session.execute(batch);
session.close();
log.info("Session closed");
}
LoadToCassandra.count++;
return StringUtils.join(tup.toStrings());
}
private Object[] getDataWithUniqueId(
String strTableNameInCass, String[] part) {
Object[] parts = null;
ArrayList<String> tempArraylist = new ArrayList<String>();
if (!strTableNameInCass
.equalsIgnoreCase("PCMASTER")) {
for (int i = 0; i < part.length; i++) {
if (i == 0) {
tempArraylist.add(0,
String.valueOf(seqId));
}
tempArraylist.add(part[i]);
}
parts = tempArraylist.toArray();
} else {
parts = part;
}
return parts;
}
});
data.count();
hadoopFs.close();
} catch (Exception e) {
e.printStackTrace();
}
}
private static JavaRDD<CassandraRow> getCassTableObj(
JavaSparkContext ctx, String cassKeyspaceName,
String strTableNameInCass) {
return javaFunctions(ctx)
.cassandraTable(cassKeyspaceName,
strTableNameInCass.toLowerCase());
}
private static CassandraConnector getCassandraConnection(
JavaSparkContext ctx) {
return CassandraConnector.apply(ctx.getConf());
}
private static String makeSt(String keyspace, String tabName,
String[] colNames) {
StringBuilder sb = new StringBuilder();
sb.append("INSERT INTO " + keyspace + "." + tabName + " ( ");
List<String> vars = new ArrayList<>();
for (int i = 0; i < (colNames.length - 1); i++) {
sb.append(colNames[i] + ",");
vars.add("?");
}
vars.add("?");
sb.append(colNames[colNames.length - 1] + " ) values ( "
+ StringUtils.join(vars, ",") + " ) ");
return sb.toString();
}}
Can anyone tell me what could the reason that causes this problem and how can it be resolved. Thanks

Once you inserted your data into cassandra, call ctx.stop() method, it will stop spark context.

Different Result on DBPedia Spotlight by using the code and DBPedia Spotlight endpoint

This is the main class in which query is being fired
package extractKeyword;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.methods.GetMethod;
import org.dbpedia.spotlight.exceptions.AnnotationException;
import org.dbpedia.spotlight.model.DBpediaResource;
import org.dbpedia.spotlight.model.Text;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.File;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.LinkedList;
import java.util.List;
public class db extends AnnotationClient {
//private final static String API_URL = "http://jodaiber.dyndns.org:2222/";
private static String API_URL = "http://spotlight.dbpedia.org/";
private static double CONFIDENCE = 0.0;
private static int SUPPORT = 0;
// private static String powered_by ="non";
// private static String spotter ="CoOccurrenceBasedSelector";//"LingPipeSpotter"=Annotate all spots
//AtLeastOneNounSelector"=No verbs and adjs.
//"CoOccurrenceBasedSelector" =No 'common words'
//"NESpotter"=Only Per.,Org.,Loc.
//private static String disambiguator ="Default";//Default ;Occurrences=Occurrence-centric;Document=Document-centric
//private static String showScores ="yes";
#SuppressWarnings("static-access")
public void configiration(double CONFIDENCE,int SUPPORT)
//, String powered_by,String spotter,String disambiguator,String showScores)
{
this.CONFIDENCE=CONFIDENCE;
this.SUPPORT=SUPPORT;
// this.powered_by=powered_by;
//this.spotter=spotter;
//this.disambiguator=disambiguator;
//showScores=showScores;
}
public List<DBpediaResource> extract(Text text) throws AnnotationException {
// LOG.info("Querying API.");
String spotlightResponse;
try {
String Query=API_URL + "rest/annotate/?" +
"confidence=" + CONFIDENCE
+ "&support=" + SUPPORT
// + "&spotter=" + spotter
// + "&disambiguator=" + disambiguator
// + "&showScores=" + showScores
// + "&powered_by=" + powered_by
+ "&text=" + URLEncoder.encode(text.text(), "utf-8");
//LOG.info(Query);
GetMethod getMethod = new GetMethod(Query);
getMethod.addRequestHeader(new Header("Accept", "application/json"));
spotlightResponse = request(getMethod);
} catch (UnsupportedEncodingException e) {
throw new AnnotationException("Could not encode text.", e);
}
assert spotlightResponse != null;
JSONObject resultJSON = null;
JSONArray entities = null;
try {
resultJSON = new JSONObject(spotlightResponse);
entities = resultJSON.getJSONArray("Resources");
} catch (JSONException e) {
//throw new AnnotationException("Received invalid response from DBpedia Spotlight API.");
}
LinkedList<DBpediaResource> resources = new LinkedList<DBpediaResource>();
if(entities!=null)
for(int i = 0; i < entities.length(); i++) {
try {
JSONObject entity = entities.getJSONObject(i);
resources.add(
new DBpediaResource(entity.getString("#URI"),
Integer.parseInt(entity.getString("#support"))));
} catch (JSONException e) {
//((Object) LOG).error("JSON exception "+e);
}
}
return resources;
}
}
The extended class
package extractKeyword;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.dbpedia.spotlight.exceptions.AnnotationException;
import org.dbpedia.spotlight.model.DBpediaResource;
import org.dbpedia.spotlight.model.Text;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.logging.Logger;
import javax.ws.rs.HttpMethod;
/**
* #author pablomendes
*/
public abstract class AnnotationClient {
//public Logger LOG = Logger.getLogger(this.getClass());
private List<String> RES = new ArrayList<String>();
// Create an instance of HttpClient.
private static HttpClient client = new HttpClient();
public List<String> getResu(){
return RES;
}
public String request(GetMethod getMethod) throws AnnotationException {
String response = null;
// Provide custom retry handler is necessary
( getMethod).getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler(3, false));
try {
// Execute the method.
int statusCode = client.executeMethod((org.apache.commons.httpclient.HttpMethod) getMethod);
if (statusCode != HttpStatus.SC_OK) {
// LOG.error("Method failed: " + ((HttpMethodBase) method).getStatusLine());
}
// Read the response body.
byte[] responseBody = ((HttpMethodBase) getMethod).getResponseBody(); //TODO Going to buffer response body of large or unknown size. Using getResponseBodyAsStream instead is recommended.
// Deal with the response.
// Use caution: ensure correct character encoding and is not binary data
response = new String(responseBody);
} catch (HttpException e) {
// LOG.error("Fatal protocol violation: " + e.getMessage());
throw new AnnotationException("Protocol error executing HTTP request.",e);
} catch (IOException e) {
//((Object) LOG).error("Fatal transport error: " + e.getMessage());
//((Object) LOG).error(((HttpMethodBase) method).getQueryString());
throw new AnnotationException("Transport error executing HTTP request.",e);
} finally {
// Release the connection.
((HttpMethodBase) getMethod).releaseConnection();
}
return response;
}
protected static String readFileAsString(String filePath) throws java.io.IOException{
return readFileAsString(new File(filePath));
}
protected static String readFileAsString(File file) throws IOException {
byte[] buffer = new byte[(int) file.length()];
#SuppressWarnings("resource")
BufferedInputStream f = new BufferedInputStream(new FileInputStream(file));
f.read(buffer);
return new String(buffer);
}
static abstract class LineParser {
public abstract String parse(String s) throws ParseException;
static class ManualDatasetLineParser extends LineParser {
public String parse(String s) throws ParseException {
return s.trim();
}
}
static class OccTSVLineParser extends LineParser {
public String parse(String s) throws ParseException {
String result = s;
try {
result = s.trim().split("\t")[3];
} catch (ArrayIndexOutOfBoundsException e) {
throw new ParseException(e.getMessage(), 3);
}
return result;
}
}
}
public void saveExtractedEntitiesSet(String Question, LineParser parser, int restartFrom) throws Exception {
String text = Question;
int i=0;
//int correct =0 ; int error = 0;int sum = 0;
for (String snippet: text.split("\n")) {
String s = parser.parse(snippet);
if (s!= null && !s.equals("")) {
i++;
if (i<restartFrom) continue;
List<DBpediaResource> entities = new ArrayList<DBpediaResource>();
try {
entities = extract(new Text(snippet.replaceAll("\\s+"," ")));
System.out.println(entities.get(0).getFullUri());
} catch (AnnotationException e) {
// error++;
//LOG.error(e);
e.printStackTrace();
}
for (DBpediaResource e: entities) {
RES.add(e.uri());
}
}
}
}
public abstract List<DBpediaResource> extract(Text text) throws AnnotationException;
public void evaluate(String Question) throws Exception {
evaluateManual(Question,0);
}
public void evaluateManual(String Question, int restartFrom) throws Exception {
saveExtractedEntitiesSet(Question,new LineParser.ManualDatasetLineParser(), restartFrom);
}
}
The Main Class
package extractKeyword;
public class startAnnonation {
public static void main(String[] args) throws Exception {
String question = "What is the winning chances of BJP in New Delhi elections?";
db c = new db ();
c.configiration(0.25,0);
//, 0, "non", "AtLeastOneNounSelector", "Default", "yes");
c.evaluate(question);
System.out.println("resource : "+c.getResu());
}
}
The main problem is here when I am using DBPedia spotlight using spotlight jar (above code)then i am getting different result as compared to the dbpedia spotlight endpoint(dbpedia-spotlight.github.io/demo/)
Result using the above code:-
Text :-What is the winning chances of BJP in New Delhi elections?
Confidence level:-0.35
resource : [Election]
Result on DBPedia Spotlight endpoint(//dbpedia-spotlight.github.io/demo/)
Text:-What is the winning chances of BJP in New Delhi elections?
Confidence level:-0.35
resource : [Bharatiya_Janata_Party, New_Delhi, Election]
Why also the spotlight now don't have support as a parameter?

query page(web services in crmod) java code reading from excel !! skip value not found

I have written a code in java for web services of crm on demand. The code reads the 1st column (id) from excel and queries it in the crm and inserts the values accordingly. If the 'id' is found it inserts the other values, but when the 'id' is not found in the crm the code is terminated with error message. I want the code to query for a record and if found insert the other values, but if the record is not found it should skip that record and query for the next id without it being terminated in between.
Please help...
and also how to skip cells with no value(null)??

import crmondemand.ws.account.AccountWS_AccountInsertChild_Input;
import crmondemand.ws.account.AccountWS_AccountInsertChild_Output;
import crmondemand.ws.account.AccountWS_AccountUpdate_Input;
import crmondemand.ws.account.AccountWS_AccountUpdate_Output;
import crmondemand.ws.account.Default_BindingStub;
import crmondemand.xml.account.Account;
import crmondemand.xml.account.RelatedAccount;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.Proxy;
import java.net.URL;
import java.rmi.RemoteException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.TimeZone;
import javax.xml.rpc.ServiceException;
import jxl.Cell;
import jxl.CellType;
import jxl.Sheet;
import jxl.Workbook;
import jxl.read.biff.BiffException;
import org.apache.axis.AxisProperties;
import org.apache.log4j.Logger;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFDateUtil;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
public class accrel1 {
private String inputFile;
public static int length;
public void setInputFile(String inputFile) {
this.inputFile = inputFile;
}
static Logger logger = Logger.getLogger(accrel1.class);
public static URL reauthentication(Properties properties) throws MalformedURLException {
// System.setProperty("https.proxyHost", "172.17.24.24");
// System.setProperty("https.proxyPort", "8003");
//System.setProperty("https.proxyHost", "145.247.13.164");
// System.setProperty("https.proxyPort", "3128");
// System.setProperty("javax.net.ssl.trustStore","C:\\ProgramFiles\\Java\\jdk1.6.0_31\\jre\\lib\\cacerts");
System.out.println("Loggin In");
String jsessionid_full =
logon("https://secure-ausomxapa.crmondemand.com/Services/Integration",
"################", "##########", properties);
System.out.println("Connecting to CRM..." + jsessionid_full);
String jsessionid = getSessionId(jsessionid_full);
System.out.println("JSessionid: " + jsessionid);
String endpoint =
"https://secure-ausomxapa.crmondemand.com/Services/Integration" +
";jsessionid=" + jsessionid;
URL urlAddr = new java.net.URL(endpoint);
System.out.println("Establishing Connection...");
return urlAddr;
}
public static void urlqueryWS1(URL urlAddr, List cellDataList,
Properties properties) throws RemoteException,
ServiceException,
MalformedURLException {
AxisProperties.setProperty("https.proxyHost",
properties.getProperty("proxyhost"));
AxisProperties.setProperty("https.proxyPort",
properties.getProperty("proxyport"));
// AxisProperties.setProperty("http.nonProxyHosts", "secure-ausomxapa.crmondemand.com");
crmondemand.ws.account.AccountWS_AccountQueryPage_Input accountlist =
new crmondemand.ws.account.AccountWS_AccountQueryPage_Input();
crmondemand.ws.account.AccountWS_AccountQueryPage_Output outlist =
new crmondemand.ws.account.AccountWS_AccountQueryPage_Output();
crmondemand.xml.account.Account[] accounts =
new crmondemand.xml.account.Account[1];
crmondemand.xml.account.Account account =
new crmondemand.xml.account.Account();
String stringCellValue[] = new String[6000];
int k = 0;
int flag = 0, notflag = 0, noparentflag = 0;
System.out.println("CellDatasize:" + cellDataList.size());
for (int i = 1; i < cellDataList.size(); i++) {
crmondemand.ws.account.Account service =
new crmondemand.ws.account.AccountLocator();
Default_BindingStub stub =
(Default_BindingStub)service.getDefault(urlAddr);
List cellTempList = (List)cellDataList.get(i);
for (int j = 0; j < cellTempList.size(); j++) {
HSSFCell hssfCell = (HSSFCell)cellTempList.get(j);
//System.out.println("Cell Type:" + i + "," + j + " " + hssfCell.getCellType());
if (hssfCell.getCellType() == HSSFCell.CELL_TYPE_STRING) {
stringCellValue[j] = hssfCell.getStringCellValue();
// logger.info(i + "," + j + "\t" + stringCellValue[j] + "\t");
} else if (hssfCell.getCellType() ==
HSSFCell.CELL_TYPE_NUMERIC) {
if (HSSFDateUtil.isCellDateFormatted(hssfCell)) {
System.out.println(hssfCell.getDateCellValue());
Date date = new Date();
date = hssfCell.getDateCellValue();
SimpleDateFormat df =
new SimpleDateFormat("MM/dd/yyyy");
stringCellValue[j] = df.format(date);
} else {
stringCellValue[j] = new BigDecimal(hssfCell.getNumericCellValue()).toPlainString();
//logger.info("Number"+i+","+j+ "\t"+stringCellValue[j] + "\t");
}
}
}
AccountWS_AccountInsertChild_Input child =
new AccountWS_AccountInsertChild_Input();
AccountWS_AccountInsertChild_Output childs =
new AccountWS_AccountInsertChild_Output();
crmondemand.xml.account.Account[] accrels =
new crmondemand.xml.account.Account[1];
crmondemand.xml.account.Account accrel =
new crmondemand.xml.account.Account();
RelatedAccount[] relaccounts = new RelatedAccount[1];
RelatedAccount relaccount = new RelatedAccount();
logger.info("Inserting " + i + "Record: "+stringCellValue[0]);
relaccount.setRelationshipRole(stringCellValue[1]);
relaccount.setRelatedAccountId(stringCellValue[2]);
relaccount.setStartDate(stringCellValue[6]);
relaccount.setEndDate(stringCellValue[7]);
relaccount.setReverseRelationshipRole(stringCellValue[3]);
relaccount.setComments(stringCellValue[4]);
relaccount.setRelationshipStatus(stringCellValue[5]);
relaccounts[0] = relaccount;
accrel.setAccountId(stringCellValue[0]); //JDE Account ID
accrel.setListOfRelatedAccount(relaccounts);
accrels[0] = accrel;
child.setListOfAccount(accrels);
try {
childs = stub.accountInsertChild(child);
logger.info(i + "th Record Inserted");
++flag;
} catch (Exception e) {
logger.info("Network Error: Re-Authenticating" + e);
urlAddr = reauthentication(properties);
stub = (Default_BindingStub)service.getDefault(urlAddr);
childs = stub.accountInsertChild(child);
logger.info(i + "th Record Inserted in 2nd Attempt");
++flag;
}
//logger.info("Total No. Of Records Processed"+flag);
}
logger.info("Total No. Of Records Processed"+flag);
}
private void readExcelFile(URL urlAddr, String fileName,
Properties properties) throws ServiceException,
RemoteException,
MalformedURLException {
System.out.println("Reading Excel File");
/**
* Create a new instance for cellDataList
*/
List cellDataList = new ArrayList();
try {
/**
* Create a new instance for FileInputStream class
*/
FileInputStream fileInputStream = new FileInputStream(fileName);
/**
* Create a new instance for POIFSFileSystem class
*/
POIFSFileSystem fsFileSystem =
new POIFSFileSystem(fileInputStream);
/*
* Create a new instance for HSSFWorkBook Class
*/
HSSFWorkbook workBook = new HSSFWorkbook(fsFileSystem);
HSSFSheet hssfSheet = workBook.getSheetAt(0);
/**
* Iterate the rows and cells of the spreadsheet
* to get all the datas.
*/
Iterator rowIterator = hssfSheet.rowIterator();
while (rowIterator.hasNext()) {
HSSFRow hssfRow = (HSSFRow)rowIterator.next();
Iterator iterator = hssfRow.cellIterator();
List cellTempList = new ArrayList();
while (iterator.hasNext()) {
HSSFCell hssfCell = (HSSFCell)iterator.next();
cellTempList.add(hssfCell);
}
cellDataList.add(cellTempList);
}
} catch (Exception e) {
e.printStackTrace();
}
/**
* Call the printToConsole method to print the cell data in the
* console.
*/
urlqueryWS1(urlAddr, cellDataList, properties);
}
private static String logon(String wsLocation, String userName,
String password, Properties properties) {
String sessionString = "FAIL";
int port =
Integer.parseInt(properties.getProperty("proxyport")); //Converting String Port Number to Integer.
try {
Proxy proxy =
new Proxy(Proxy.Type.HTTP, new InetSocketAddress(properties.getProperty("proxyhost"),
port));
// create an HTTPS connection to the On Demand webservices
URL wsURL = new URL(wsLocation + "?command=login");
HttpURLConnection wsConnection =
(HttpURLConnection)wsURL.openConnection(proxy);
// we don't want any caching to occur
wsConnection.setUseCaches(false);
// we want to send data to the server
// wsConnection.setDoOutput(true);
// set some http headers to indicate the username and passwod we are using to logon
wsConnection.setRequestProperty("UserName", userName);
wsConnection.setRequestProperty("Password", password);
wsConnection.setRequestMethod("GET");
// see if we got a successful response
if (wsConnection.getResponseCode() == HttpURLConnection.HTTP_OK) {
// get the session id from the cookie setting
sessionString = getCookieFromHeaders(wsConnection);
}
} catch (Exception e) {
System.out.println("Logon Exception generated :: " + e);
}
return sessionString;
}
private static String getCookieFromHeaders(HttpURLConnection wsConnection) {
// debug code - display all the returned headers
String headerName;
String headerValue = "FAIL";
for (int i = 0; ; i++) {
headerName = wsConnection.getHeaderFieldKey(i);
if (headerName != null && headerName.equals("Set-Cookie")) {
// found the Set-Cookie header (code assumes only one cookie is being set)
headerValue = wsConnection.getHeaderField(i);
break;
}
}
// return the header value (FAIL string for not found)
return headerValue;
}
private static String getSessionId(String cookie) {
StringTokenizer st = new StringTokenizer(cookie, ";");
String jsessionid = st.nextToken();
st = new StringTokenizer(jsessionid, "=");
st.nextToken();
return st.nextToken();
}
public static void main(String[] args) throws IOException,
ServiceException {
String jsessionid, jsessionid_full;
String endpoint;
AxisProperties.setProperty("https.proxyHost", "172.25.9.240");
AxisProperties.setProperty("https.proxyPort", "2006");
System.setProperty("https.proxyHost", "172.25.9.240");
System.setProperty("https.proxyPort", "2006");
// System.setProperty("https.proxyHost", "145.247.13.164");
// System.setProperty("https.proxyPort", "3128");
Properties properties = new Properties();
properties.load(new FileInputStream("C:\\Users\\10608011\\Documents\\Account_Config.properties")); //Windows Path
System.setProperty("javax.net.ssl.trustStore","C:\\Oracle\\Middleware3\\jdk160_24\\jre\\lib\\security\\cacerts");
System.out.println("Logging In");
jsessionid_full =
logon("https://secure-ausomxapa.crmondemand.com/Services/Integration",
"############", "###########", properties);
System.out.println("Establishing " + jsessionid_full);
jsessionid = getSessionId(jsessionid_full);
System.out.println("Jsessionid: " + jsessionid);
endpoint =
"https://secure-ausomxapa.crmondemand.com/Services/Integration" +
";jsessionid=" + jsessionid;
URL urlAddr = new java.net.URL(endpoint);
String fileName =
"D:" + File.separator + "Test2.xls"; // Windows Path
// String fileName =File.separator + "u01"+File.separator +"CRM_DEV"+File.separator +
// "sofwaredepot"+File.separator +"PALS_200_11Feb2013.xls"; //Linux Path /u01/CRM_DEV/softwaredepot
// String fileName="PALS_200_11Feb2013.xls";
final long start = System.currentTimeMillis();
logger.info("Start Time:" + start);
new accrel1().readExcelFile(urlAddr, fileName, properties);
final long durationInMilliseconds = System.currentTimeMillis() - start;
System.out.println("Time(Min) for Data Upload: " +
durationInMilliseconds / 60000 + "mins.");
logger.info("Duration for Data Upload: " +
durationInMilliseconds / 60000 + "mins.");
}
}