We Keep Coding

How to break this Mail Method in two parts without breaking its functionality [closed]

Closed. This question needs details or clarity. It is not currently accepting answers.
Want to improve this question? Add details and clarify the problem by editing this post.
Closed 2 years ago.
Improve this question
I have write a readFromSharedInbox() method. When I am checking the code with sonarLint i am getting this error "Refractor this method to reduce its cognitive complexity". So i want to break this method in two parts. But i am not able to do so.I am trying to create a method to read the mail from inbox.
My Code look like this
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import javax.mail.Address;
import javax.mail.BodyPart;
import javax.mail.Flags;
import javax.mail.Folder;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.Part;
import javax.mail.Session;
import javax.mail.Store;
import javax.mail.internet.ContentType;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeBodyPart;
import javax.mail.internet.MimeMultipart;
import javax.mail.search.ComparisonTerm;
import javax.mail.search.FlagTerm;
import javax.mail.search.ReceivedDateTerm;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Example;
import org.springframework.stereotype.Service;
#Service
public class MailServiceImpl implements MailService {
#Value("${mail.host}")
private String host;
#Value("${mail.id}")
private String mailBoxId;
#Value("${mail.password}")
private String password;
#Value("${mail.protocol}")
private String protocol;
#Value("${mail.socketfactory}")
private String socketfactory;
#Value("${mail.fallback}")
private String fallback;
#Value("${mail.port}")
private String port;
#Value("${mail.savepath}")
private String savepath;
#Value("${submission.fields}")
private String fields;
#Value("${mail.endidentifier}")
private String mailEndIdentifier;
#Autowired
private EmailRequestRepository emailRequestRepository;
#Autowired
private CoreWorkflowService coreWorkflowService;
public void readFromSharedInbox() {
Properties props = new Properties();
props.setProperty("mail.store.protocol", protocol);
props.setProperty("mail.imaps.socketFactory.class", socketfactory);
props.setProperty("mail.imaps.socketFactory.fallback", fallback);
props.setProperty("mail.imaps.port", port);
props.setProperty("mail.imaps.socketFactory.port", port);
Session session = Session.getDefaultInstance(props, null);
try {
Store store = session.getStore(protocol);
store.connect(host, mailBoxId, password);
Folder inbox = store.getFolder("INBOX");
inbox.open(Folder.READ_WRITE);
ReceivedDateTerm receivedDateTerm = new ReceivedDateTerm(ComparisonTerm.EQ, new Date());
FlagTerm seenTerm = new FlagTerm(new Flags(Flags.Flag.SEEN), false);
Message[] todaysMessages = inbox.search(receivedDateTerm);
Message[] foundMessages = inbox.search(seenTerm, todaysMessages);
List<EmailRequest> emailRequestList = new ArrayList<>();
for (int i = 0; i < foundMessages.length; i++) {
Message message = foundMessages[i];
String subject = message.getSubject();
String content = message.getContent().toString();
String contentType = message.getContentType();
Address[] froms = message.getFrom();
String sender = froms == null ? null : ((InternetAddress) froms[0]).getAddress();
Date recieveDate = message.getReceivedDate();
// store attachment file name, separated by comma
StringBuilder attachFiles=new StringBuilder();
if (contentType.contains("multipart")) {
// content may contain attachments
MimeMultipart multiPart = (MimeMultipart) message.getContent();
content = getTextFromMimeMultipart(multiPart);
int numberOfParts = multiPart.getCount();
for (int partCount = 0; partCount < numberOfParts; partCount++) {
MimeBodyPart part = (MimeBodyPart) multiPart.getBodyPart(partCount);
if (Part.ATTACHMENT.equalsIgnoreCase(part.getDisposition())) {
// this part is attachment
String fileName = part.getFileName();
if (attachFiles.length() > 0) {
attachFiles.append(",");
attachFiles.append(fileName);
} else {
attachFiles.append(fileName);
}
System.out.println(fileName);
}
}
message.writeTo(new FileOutputStream(new File(savepath + sanitizeFileName(subject) + ".eml")));
}
System.out.println("Content type: " + contentType);
EmailRequest emailRequest = EmailRequest.builder().emailRecieveDate(recieveDate).subjectEmail(subject)
.emailContent(content).fromEmail(sender).toEmail(mailBoxId).isMailProcessed(false).build();
emailRequestList.add(emailRequest);
}
inbox.close(false);
store.close();
System.out.println("reading done!");
emailRequestRepository.saveAll(emailRequestList);
System.out.println("Email data saved successfully in db table");
} catch (Exception e) {
e.printStackTrace();
}
}
private String sanitizeFileName(String subject) {
return subject.replaceAll("[:\\\\/*?|<>]", "_");
}
private String getTextFromMimeMultipart(MimeMultipart mimeMultipart) throws IOException, MessagingException {
int count = mimeMultipart.getCount();
if (count == 0)
throw new MessagingException("Multipart with no body parts not supported.");
boolean multipartAlt = new ContentType(mimeMultipart.getContentType()).match("multipart/alternative");
if (multipartAlt)
// alternatives appear in an order of increasing
// faithfulness to the original content. Customize as req'd.
return getTextFromBodyPart(mimeMultipart.getBodyPart(count - 1));
StringBuilder result = new StringBuilder();
for (int i = 0; i < count; i++) {
BodyPart bodyPart = mimeMultipart.getBodyPart(i);
result.append(getTextFromBodyPart(bodyPart));
}
return result.toString();
}
private String getTextFromBodyPart(BodyPart bodyPart) throws IOException, MessagingException {
String result = "";
if (bodyPart.isMimeType("text/plain")) {
result = (String) bodyPart.getContent();
} else if (bodyPart.isMimeType("text/html")) {
result = (String) bodyPart.getContent();
} else if (bodyPart.getContent() instanceof MimeMultipart) {
result = getTextFromMimeMultipart((MimeMultipart) bodyPart.getContent());
}
return result;
}
#Override
public void readMailDetailsFromDb() {
EmailRequest email = new EmailRequest();
email.setIsMailProcessed(false);
Example<EmailRequest> whereClause = Example.of(email);
List<EmailRequest> findAll = emailRequestRepository.findAll(whereClause);
Map<String, Object> processVariables = null;
for (EmailRequest emaeil : findAll) {
System.out.println(emaeil.getSubjectEmail() + " : " + emaeil.getEmailId());
processVariables = EmailParserUtil.parse(fields, emaeil.getEmailContent(), mailEndIdentifier);
// workflow service
try {
Workflow startWorkflow = coreWorkflowService.startWorkflow(Constants.SUBMISSION_PROCESS_ID, Constants.MAIL, processVariables);
startWorkflow.toString();
} catch (IllegalAccessException | InvocationTargetException e) {
e.printStackTrace();
}
// update mail
emaeil.setMailProcessedOn(LocalDateTime.now() ) ;
emaeil.setIsMailProcessed(true) ;
emailRequestRepository.save(emaeil) ;
}
}
}
This is MailService Class
import java.lang.reflect.InvocationTargetException;
public interface MailService {
public void readFromSharedInbox();
public void readMailDetailsFromDb() throws IllegalAccessException, InvocationTargetException;
}

You can use your IDE for things like that.
You select a part of your code, right click on it and refactor it by extracting the code in a new private method.
For exemple, you can do this on the "for" statement.
Or the "if (contentType.contains("multipart"))" part with a manageMultipart method.

How to upload file (txt) from java to restful webservice?

I want to upload txt file from java application(software) to restful web-service , Because inside txt file i have list of sql query to execute. So , I want to upload file into web-service first then execute query line by line to database.
I converted txt file into List object then again into string and passed it but could not get success.
Here is Webservice code
package com.java.webservice;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import com.sun.jersey.api.client.WebResource;
#Path("UploadFile")
public class UploadFile {
private List<String> storedQueries = new ArrayList<String>();
private Connection conn = null;
// private PreparedStatement pstm = null;
private Statement pstm = null;
private ResultSet rs = null;
private String q = "";
// #GET here defines, this method will method will process HTTP GET
// requests.
#GET
// #Path here defines method level path. Identifies the URI path that a
// resource class method will serve requests for.
#Path("/getTest")
// #Produces here defines the media type(s) that the methods
// of a resource class can produce.
#Produces(MediaType.APPLICATION_JSON)
// #PathParam injects the value of URI parameter that defined in #Path
// expression, into the method.
public Response getTest(#QueryParam("name") String name) {
// String name = "Lokendra Pun ,Java Engineer";
System.out.println("Your Name IS: " + name);
java.util.StringTokenizer st = new java.util.StringTokenizer(name, ",");
java.util.ArrayList newArrayList = new java.util.ArrayList();
while (st.hasMoreTokens()) {
// the trim() below may not always be desired or necessary
newArrayList.add(st.nextToken().trim());
}
// this if statement only necessary because a String created from
// ArrayList.toString() ends up as "[1, 2, 3, 4]" not, "1, 2, 3, 4"
if (newArrayList.size() == 1) {
// remove [] around item
int length = ((String) newArrayList.get(0)).length();
newArrayList.set(0, ((String) newArrayList.get(0)).substring(1, length - 1));
} else if (newArrayList.size() > 1) {
// remove '[' in first item and ']' in last
int lastIndex = newArrayList.size() - 1;
String oldString = (String) newArrayList.get(0);
String newString = oldString.substring(1, oldString.length());
newArrayList.set(0, newString);
oldString = (String) newArrayList.get(lastIndex);
newString = oldString.substring(0, oldString.length() - 1);
newArrayList.set(newArrayList.size() - 1, newString);
}
System.out.println("newArrayList->" + newArrayList.toString());
System.out.println("==================");
for (int i = 0; i < newArrayList.size(); i++) {
System.out.println("Name" + i + " :" + newArrayList.get(i));
}
System.out.println("==================");
return Response.status(200).entity(name).build();
}
/**
*
*
* method to get file name from remote server and process query into
* database
*
*
*
*/
#GET
#Produces(MediaType.APPLICATION_JSON)
#Path("/getUploadQuery")
public Response getUploadQuery(#QueryParam("file_name") String file_name,
#QueryParam("database_name") String database_name) {
System.out.println("I am inside getTeacher Method");
String list = null;
try {
conn = getConnection(database_name);
if (conn != null) {
try {
for (String line : Files.readAllLines(Paths.get(file_name))) {
storedQueries.add(line);
System.out.println(line);
}
for (int i = 0; i < storedQueries.size(); i++) {
System.out.println("Looping Query Length is: " + storedQueries.size());
System.out.println("Connection Object Inside Loop : " + conn);
pstm = conn.createStatement();
System.out.println("Looping Size now : " + i);
if (!storedQueries.get(i).equals("")) {
System.out.println("Query Before Executed is: " + storedQueries.get(i));
// pstm =
// conn.prepareStatement(storedQueries.get(i));
// pstm.executeUpdate();
pstm.addBatch(storedQueries.get(i));
pstm.executeBatch();
}
}
pstm.close();
conn.close();
} catch (Exception e) {
e.printStackTrace();
}
list = "true";
} else {
list = "false";
}
} catch (Exception e) {
e.printStackTrace();
}
return Response.status(200).entity(list).build();
}
/**
*
* method to connect to remote server database according to db and school
* name
*
* #param database_name
* #return
*/
public Connection getConnection(String database_name) {
System.out.println("Database Name before Connection: " + database_name);
try {
Class.forName("com.mysql.jdbc.Driver");
// Setup the connection with the DB
conn = DriverManager.getConnection(
"jdbc:mysql://localhost/" + database_name + "?" + "user=root&password=java#android2016");
System.out.println(conn);
} catch (Exception e) {
e.printStackTrace();
}
return conn;
}
}
And here is code for service calling from java application
package WSserviceCall;
import javax.ws.rs.core.MediaType;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
public class TestMain {
public static final String BASE_URI = "http://localhost:8080/MovieGalaxyService";
public static final String PATH_TEST = "/UploadFile/getTest";
public static final String PATH_LOAD_FILE = "/UploadFile/getUploadQuery";
public static final String PATH_AGE = "/UploadFile/age/";
public static void main(String[] args) {
try {
// TODO Auto-generated method stub
List<String> storedQueries = new ArrayList<String>();
String currentDirectory = System.getProperty("user.dir");
String file_name = currentDirectory + "/databaseBackupScript.txt";
String name = "Lokendra Pun Param";
String age ="26";
try{
for (String line : Files.readAllLines(Paths.get(file_name))) {
storedQueries.add(line);
System.out.println(line);
}
}catch(Exception e){
e.printStackTrace();
}
// String names = ""+aa;
//System.out.println("Array Objec is: "+aa);
//System.out.println("String Objects is: "+aa.toString());
ClientConfig config = new DefaultClientConfig();
Client client = Client.create(config);
WebResource resource = client.resource(BASE_URI);
String param = storedQueries.toString();
String temp_uri = URLEncoder.encode(param, "UTF-8");
WebResource testResource = resource.path(PATH_TEST).
queryParam("name",temp_uri);
//WebResource testResource = resource.path(PATH_TEST);
// System.out.println("Client Response \n"
// + getClientResponse(testResource));
// System.out.println("Response \n" + getResponse(testResource) + "\n\n");
String testValue = getResponse(testResource);
System.out.println("Test Resource Value is : "+testValue);
} catch (UnsupportedEncodingException ex) {
Logger.getLogger(TestMain.class.getName()).log(Level.SEVERE, null, ex);
}
}
/**
* Returns client response.
* e.g :
* GET http://localhost:8080/RESTfulWS/rest/UserInfoService/name/Pavithra
* returned a response status of 200 OK
*
* #param service
* #return
*/
private static String getClientResponse(WebResource resource) {
return resource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class).toString();
}
/**
* Returns the response as JSON data
* e.g : status : true or false
*
* #param service
* #return
*/
private static String getResponse(WebResource resource) {
return resource.accept(MediaType.APPLICATION_JSON).get(String.class);
}
}

How to use Java classes in Talend

I have the following three classes :
I tried making the routine of 1 & 2 and used tjava to call the main class and the method from 1 & 2 but I am unable to fetch those methods.
1)
package page_scraper;
import com.gargoylesoftware.htmlunit.Page;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebClientOptions;
import com.gargoylesoftware.htmlunit.html.FrameWindow;
import com.gargoylesoftware.htmlunit.html.HtmlButtonInput;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlOption;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSelect;
import com.gargoylesoftware.htmlunit.html.HtmlTextInput;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintStream;
import java.io.Writer;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import page_scraper.UnitArray;
public class PageScraper {
public void Scrape() throws IOException {
try {
UnitArray object = new UnitArray();
ArrayList<String> unitList = object.getUnitArray();
WebClient webClient = new WebClient();
webClient.getOptions().setThrowExceptionOnScriptError(false);
webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
HtmlPage page = (HtmlPage)webClient.getPage("http://www.bmreports.com/servlet/com.logica.neta.bwp_PanBMUData");
List frames = page.getFrames();
HtmlPage page1 = (HtmlPage)((FrameWindow)frames.get(0)).getEnclosedPage();
HtmlTextInput settlementDay = (HtmlTextInput)page1.getHtmlElementById("param5");
HtmlSelect period = (HtmlSelect)page1.getHtmlElementById("param6");
HtmlOption periodOption = period.getOption(1);
HtmlTextInput unitId = (HtmlTextInput)page1.getHtmlElementById("param1");
HtmlButtonInput button = (HtmlButtonInput)page1.getHtmlElementById("go_button");
String outputLocation = String.valueOf(System.getProperty("user.home")) + "/Documents/output.csv";
FileWriter fileWriter = new FileWriter(outputLocation);
String errorLocation = String.valueOf(System.getProperty("user.home")) + "/Documents/error.csv";
FileWriter errorWriter = new FileWriter(errorLocation);
int i = 0;
while (i < unitList.size()) {
int x = 0;
while (x < 365) {
String errorData;
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
Calendar cal = Calendar.getInstance();
cal.add(5, - x);
String dateValue = dateFormat.format(cal.getTime());
System.out.println(dateValue);
settlementDay.setValueAttribute(dateValue);
period.setSelectedAttribute(periodOption, true);
unitId.setValueAttribute(unitList.get(i));
System.out.println(unitList.get(i));
try {
button.click();
HtmlPage page2 = (HtmlPage)((FrameWindow)frames.get(1)).getEnclosedPage();
String pageSource = page2.asXml();
int firstIndex = pageSource.indexOf("csv=") + 38;
int secondIndex = pageSource.indexOf("n\"") + 1;
String csvData = pageSource.substring(firstIndex, secondIndex);
fileWriter.append(csvData);
}
catch (ClassCastException e) {
errorData = String.valueOf(dateValue) + " " + unitList.get(i) + System.getProperty("line.separator");
System.out.println(errorData);
errorWriter.append(errorData);
continue;
}
catch (StringIndexOutOfBoundsException e) {
errorData = String.valueOf(dateValue) + " " + unitList.get(i) + System.getProperty("line.separator");
System.out.println(errorData);
errorWriter.append(errorData);
continue;
}
++x;
}
++i;
}
webClient.close();
fileWriter.close();
errorWriter.close();
}
catch (IOException e) {
e.printStackTrace();
}
}
}
2)
package page_scraper;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
public class UnitArray {
public ArrayList<String> getUnitArray() {
String csvList = "abc,xyz";
ArrayList<String> list = new ArrayList<String>(Arrays.asList(csvList.split(",")));
return list;
}
}
3)
package page_scraper;
import page_scraper.PageScraper;
public class main {
public static void main(String[] args) throws Exception {
PageScraper test = new PageScraper();
test.Scrape();
}
}
I made the routines for the above code(1) & 2)) in Talend and then used tjava to call the method but unable to do so..I also tried using tjava for all and did a onSubjob ok on each of the tjava.
How can I call these classes in talend and call the method ?

Firstly, routines classes in Talend need to be in routines package
package routines;
public class PageScraper {
public void Scrape() {
System.out.println("PageScraper.Scrape");
}
}
Secondly, to use it in Job you need to drag'n'drop routine to opened job area.
Then you can use your class in that way

You can easily make a jar file that contains the three classes then load the jar using tLibraryLoad or include the jar in your routine if you want to get more reusability.

As suggested in the other answers, you need to define classes under routines package.
in case you are using Takend 7.3 & above, Right click on your routine and add it as Dependent package
Get routines as a jar and in case using in bigData jobs, you may need to use tLibraryLoad to package it together with other dependencies..

Different Result on DBPedia Spotlight by using the code and DBPedia Spotlight endpoint

This is the main class in which query is being fired
package extractKeyword;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.methods.GetMethod;
import org.dbpedia.spotlight.exceptions.AnnotationException;
import org.dbpedia.spotlight.model.DBpediaResource;
import org.dbpedia.spotlight.model.Text;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.File;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.LinkedList;
import java.util.List;
public class db extends AnnotationClient {
//private final static String API_URL = "http://jodaiber.dyndns.org:2222/";
private static String API_URL = "http://spotlight.dbpedia.org/";
private static double CONFIDENCE = 0.0;
private static int SUPPORT = 0;
// private static String powered_by ="non";
// private static String spotter ="CoOccurrenceBasedSelector";//"LingPipeSpotter"=Annotate all spots
//AtLeastOneNounSelector"=No verbs and adjs.
//"CoOccurrenceBasedSelector" =No 'common words'
//"NESpotter"=Only Per.,Org.,Loc.
//private static String disambiguator ="Default";//Default ;Occurrences=Occurrence-centric;Document=Document-centric
//private static String showScores ="yes";
#SuppressWarnings("static-access")
public void configiration(double CONFIDENCE,int SUPPORT)
//, String powered_by,String spotter,String disambiguator,String showScores)
{
this.CONFIDENCE=CONFIDENCE;
this.SUPPORT=SUPPORT;
// this.powered_by=powered_by;
//this.spotter=spotter;
//this.disambiguator=disambiguator;
//showScores=showScores;
}
public List<DBpediaResource> extract(Text text) throws AnnotationException {
// LOG.info("Querying API.");
String spotlightResponse;
try {
String Query=API_URL + "rest/annotate/?" +
"confidence=" + CONFIDENCE
+ "&support=" + SUPPORT
// + "&spotter=" + spotter
// + "&disambiguator=" + disambiguator
// + "&showScores=" + showScores
// + "&powered_by=" + powered_by
+ "&text=" + URLEncoder.encode(text.text(), "utf-8");
//LOG.info(Query);
GetMethod getMethod = new GetMethod(Query);
getMethod.addRequestHeader(new Header("Accept", "application/json"));
spotlightResponse = request(getMethod);
} catch (UnsupportedEncodingException e) {
throw new AnnotationException("Could not encode text.", e);
}
assert spotlightResponse != null;
JSONObject resultJSON = null;
JSONArray entities = null;
try {
resultJSON = new JSONObject(spotlightResponse);
entities = resultJSON.getJSONArray("Resources");
} catch (JSONException e) {
//throw new AnnotationException("Received invalid response from DBpedia Spotlight API.");
}
LinkedList<DBpediaResource> resources = new LinkedList<DBpediaResource>();
if(entities!=null)
for(int i = 0; i < entities.length(); i++) {
try {
JSONObject entity = entities.getJSONObject(i);
resources.add(
new DBpediaResource(entity.getString("#URI"),
Integer.parseInt(entity.getString("#support"))));
} catch (JSONException e) {
//((Object) LOG).error("JSON exception "+e);
}
}
return resources;
}
}
The extended class
package extractKeyword;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.dbpedia.spotlight.exceptions.AnnotationException;
import org.dbpedia.spotlight.model.DBpediaResource;
import org.dbpedia.spotlight.model.Text;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.logging.Logger;
import javax.ws.rs.HttpMethod;
/**
* #author pablomendes
*/
public abstract class AnnotationClient {
//public Logger LOG = Logger.getLogger(this.getClass());
private List<String> RES = new ArrayList<String>();
// Create an instance of HttpClient.
private static HttpClient client = new HttpClient();
public List<String> getResu(){
return RES;
}
public String request(GetMethod getMethod) throws AnnotationException {
String response = null;
// Provide custom retry handler is necessary
( getMethod).getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler(3, false));
try {
// Execute the method.
int statusCode = client.executeMethod((org.apache.commons.httpclient.HttpMethod) getMethod);
if (statusCode != HttpStatus.SC_OK) {
// LOG.error("Method failed: " + ((HttpMethodBase) method).getStatusLine());
}
// Read the response body.
byte[] responseBody = ((HttpMethodBase) getMethod).getResponseBody(); //TODO Going to buffer response body of large or unknown size. Using getResponseBodyAsStream instead is recommended.
// Deal with the response.
// Use caution: ensure correct character encoding and is not binary data
response = new String(responseBody);
} catch (HttpException e) {
// LOG.error("Fatal protocol violation: " + e.getMessage());
throw new AnnotationException("Protocol error executing HTTP request.",e);
} catch (IOException e) {
//((Object) LOG).error("Fatal transport error: " + e.getMessage());
//((Object) LOG).error(((HttpMethodBase) method).getQueryString());
throw new AnnotationException("Transport error executing HTTP request.",e);
} finally {
// Release the connection.
((HttpMethodBase) getMethod).releaseConnection();
}
return response;
}
protected static String readFileAsString(String filePath) throws java.io.IOException{
return readFileAsString(new File(filePath));
}
protected static String readFileAsString(File file) throws IOException {
byte[] buffer = new byte[(int) file.length()];
#SuppressWarnings("resource")
BufferedInputStream f = new BufferedInputStream(new FileInputStream(file));
f.read(buffer);
return new String(buffer);
}
static abstract class LineParser {
public abstract String parse(String s) throws ParseException;
static class ManualDatasetLineParser extends LineParser {
public String parse(String s) throws ParseException {
return s.trim();
}
}
static class OccTSVLineParser extends LineParser {
public String parse(String s) throws ParseException {
String result = s;
try {
result = s.trim().split("\t")[3];
} catch (ArrayIndexOutOfBoundsException e) {
throw new ParseException(e.getMessage(), 3);
}
return result;
}
}
}
public void saveExtractedEntitiesSet(String Question, LineParser parser, int restartFrom) throws Exception {
String text = Question;
int i=0;
//int correct =0 ; int error = 0;int sum = 0;
for (String snippet: text.split("\n")) {
String s = parser.parse(snippet);
if (s!= null && !s.equals("")) {
i++;
if (i<restartFrom) continue;
List<DBpediaResource> entities = new ArrayList<DBpediaResource>();
try {
entities = extract(new Text(snippet.replaceAll("\\s+"," ")));
System.out.println(entities.get(0).getFullUri());
} catch (AnnotationException e) {
// error++;
//LOG.error(e);
e.printStackTrace();
}
for (DBpediaResource e: entities) {
RES.add(e.uri());
}
}
}
}
public abstract List<DBpediaResource> extract(Text text) throws AnnotationException;
public void evaluate(String Question) throws Exception {
evaluateManual(Question,0);
}
public void evaluateManual(String Question, int restartFrom) throws Exception {
saveExtractedEntitiesSet(Question,new LineParser.ManualDatasetLineParser(), restartFrom);
}
}
The Main Class
package extractKeyword;
public class startAnnonation {
public static void main(String[] args) throws Exception {
String question = "What is the winning chances of BJP in New Delhi elections?";
db c = new db ();
c.configiration(0.25,0);
//, 0, "non", "AtLeastOneNounSelector", "Default", "yes");
c.evaluate(question);
System.out.println("resource : "+c.getResu());
}
}
The main problem is here when I am using DBPedia spotlight using spotlight jar (above code)then i am getting different result as compared to the dbpedia spotlight endpoint(dbpedia-spotlight.github.io/demo/)
Result using the above code:-
Text :-What is the winning chances of BJP in New Delhi elections?
Confidence level:-0.35
resource : [Election]
Result on DBPedia Spotlight endpoint(//dbpedia-spotlight.github.io/demo/)
Text:-What is the winning chances of BJP in New Delhi elections?
Confidence level:-0.35
resource : [Bharatiya_Janata_Party, New_Delhi, Election]
Why also the spotlight now don't have support as a parameter?

Use getJSONObject(int)

I'm trying to parse some JSON. Here is the code. frc-manual.usfirst.org/a/GetAllItems/ManualID=3 I have been trying for several hours to get it work but every example I have seen online uses getJSONObject(int) but I can only use getJSONObject(String). This is making impossible. Am I overlooking something?
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserFactory;
import android.annotation.SuppressLint;
public class JSON {
private String html = "html";
private String version = "version";
private String pageString = null;
private String urlString = "http://frc-manual.usfirst.org/a/GetAllItems/ManualID=3";
public volatile boolean parsingComplete = true;
public JSON(String page){
this.pageString = page;
}
public String getHTML(){
return html;
}
public String getVersion(){
return version;
}
#SuppressLint("NewApi")
public void readAndParseJSON(String in) {
try {
JSONObject reader = new JSONObject(in);
JSONObject head = reader.getJSONObject("data").getJSONObject("SubChapter").getJSONObject("3").getJSONObject("children").getJSONObject(pageString);
if(head != null){
html = head.getString("item_content_text");
html = html + head.length();
for(int i = 0; i < head.length();i++){
JSONObject children = head.getJSONObject(i);
if(children != null){
html = html + children.getString("item_content_text");
}
}
}
//html = html + listFromJsonSorted(head.getJSONObject("children"));
JSONObject main = reader.getJSONObject("data");
version = main.getString("LatestManualUpdate");
parsingComplete = false;
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public void fetchJSON(){
Thread thread = new Thread(new Runnable(){
#Override
public void run() {
try {
URL url = new URL(urlString);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setReadTimeout(10000 /* milliseconds */);
conn.setConnectTimeout(15000 /* milliseconds */);
conn.setRequestMethod("GET");
conn.setDoInput(true);
// Starts the query
conn.connect();
InputStream stream = conn.getInputStream();
String data = convertStreamToString(stream);
readAndParseJSON(data);
stream.close();
} catch (Exception e) {
e.printStackTrace();
}
}
});
thread.start();
}
static String convertStreamToString(java.io.InputStream is) {
java.util.Scanner s = new java.util.Scanner(is).useDelimiter("\\A");
return s.hasNext() ? s.next() : "";
}
}

Probably what you are seeing in this example is a JSONArray.
JSONArray arr = json.getJSONArray("array");
JSONObject obj = arr.getJSONObject(0);
In your case, i don't see any array in this JSON, head in your code is a JSONObject. To get the item_content_text you just need head.getString("item_content_text");
You can do this to get all childrens in your JSON:
html = head.getString("item_content_text");
JSONObject children = head;
while (children.containsKey("children")) {
children = children.getJSONObject("children");
html += children.getString("item_content_text");
}

You are only able to use getJSONObject(String) not getJSONObject(int) because the method only takes String.. Check documentation here..
The reason that is the because the keys in json are always strings only.. read here
I think you are looking to retrieve an int value from the json but you got confused.. The way to do that would be getInt("key_name") if the json indeed has a key with int value..

Sure you can, just that I think your "head" variable should have a return type of JSONArray instead of JSONObject.
I have this code running in my program -
//response = some http response
final JSONObject object = new JSONObject(response);
final JSONArray array = object.getJSONArray("repeatedStuff"); //$NON-NLS-1$
Assert.assertEquals(2, array.length());
for (int i = 0; i < array.length(); i++) {
final JSONObject element = array.getJSONObject(i);
//do something
}
Also you may refer this link to see what they are doing - android: The method getJSONObject(int) in the type JSONArray is not applicable for the arguments (String)
Let me know if that helped!

You have used JSONObject everywhere in your code. Thus you are only able to pass string parameter in getJSONObject().
I suggest you you change it to JSONArray and then you will be able to pass int parameter in getJSONObject().