We Keep Coding

Java Exception not understood

I am writing a search engine code using java, and I'm getting this error without knowing the cause:
Exception in thread "main" java.lang.NullPointerException
at WriteToFile.fileWriter(WriteToFile.java:29)
at Main.main(Main.java:14)
Process finished with exit code 1
this is my code :
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Scanner;
public class Search {
private static String URL="https://www.google.com/search?q=";
private Document doc;
private Elements links;
private String html;
public Search() throws IOException {};
public void SearchWeb() throws IOException {
//to get the keywords from the user
Scanner sc = new Scanner(System.in);
System.out.println("Please enter the keyword you want to search for: ");
String word = sc.nextLine();
//Search for the keyword over the net
String url = URL + word;
doc = Jsoup.connect(url).get();
html = doc.html();
Files.write(Paths.get("D:\\OOP\\OOPproj\\data.txt"), html.getBytes());
links = doc.select("cite");
}
public Document getDoc() {
return doc;
}
public String getHtml() {
return html;
}
public Elements getLinks() {
return links;
}
}
and this is the class writeToFile:
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.BufferedWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
public class WriteToFile extends Search {
public WriteToFile() throws IOException {};
String description = "<!> Could not fetch description <!>";
String keywords = "<!> Could not fetch keywords <!>";
private ArrayList<String> detail = new ArrayList<String>();
BufferedWriter bw = null;
public void fileWriter() throws IOException {
for (Element link : super.getLinks()) {
String text = link.text();
if (text.contains("›")) {
text = text.replaceAll(" › ", "/");
}
detail.add(text);
System.out.println(text);
}
System.out.println("***************************************************");
for (int i = 0; i < detail.size(); i++)
System.out.println("detail [" + (i + 1) + "]" + detail.get(i));
System.out.println("###################################################################");
for (int j = 0; j < detail.size(); j++) {
Document document = Jsoup.connect(detail.get(j)).get();
String web = document.html();
Document d = Jsoup.parse(web);
Elements metaTags = d.getElementsByTag("meta");
for (Element metaTag : metaTags) {
String content = metaTag.attr("content");
String name = metaTag.attr("name");
if ("description".equals(name)) {
description = content;
}
if ("keywords".equals(name)) {
keywords = content;
}
}
String title = d.title();
Files.write(Paths.get("D:\\OOP\\OOPproj\\search.txt"), (detail.get(j) + "\t" + "|" + "\t" + title + "\t" + "|" + "\t" + description + "\t" + "|" + "\t" + keywords + System.lineSeparator()).getBytes(), StandardOpenOption.APPEND);
}
}
}
This is the Main class:
import java.io.IOException;
public class Main {
public static void main(String[] args) throws IOException {
Search a = new Search();
a.SearchWeb();
WriteToFile b = new WriteToFile();
b.fileWriter();
}
}
I tried to print the getLinks() method in main to check if it was null , but it wasn't , the links were cited.
I would be really grateful if someone helps me out.

You are calling SearchWeb() on object a, but you're calling fileWriter() on object b. This means the links are set in a, but not in b.
Since WriteToFile extends Search, you just need an instance of that:
WriteToFile a = new WriteToFile();
a.SearchWeb();
a.fileWriter();

Web scraper not creating CSV file

I have created a web scraper which brings the market data of share rates from the website of stock exchange. www.psx.com.pk in that site there is a hyperlink of Market Summary. From that link I have to scrap the data. I have created a program which is as follows.
package com.market_summary;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.Locale;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class ComMarket_summary {
boolean writeCSVToConsole = true;
boolean writeCSVToFile = true;
boolean sortTheList = true;
boolean writeToConsole;
boolean writeToFile;
public static Document doc = null;
public static Elements tbodyElements = null;
public static Elements elements = null;
public static Elements tdElements = null;
public static Elements trElement2 = null;
public static String Dcomma = ",";
public static String line = "";
public static ArrayList<Elements> sampleList = new ArrayList<Elements>();
public static void createConnection() throws IOException {
System.setProperty("http.proxyHost", "191.1.1.202");
System.setProperty("http.proxyPort", "8080");
String tempUrl = "http://www.psx.com.pk/index.php";
doc = Jsoup.connect(tempUrl).get();
System.out.println("Successfully Connected");
}
public static void parsingHTML() throws Exception {
File fold = new File("C:\\market_smry.csv");
fold.delete();
File fnew = new File("C:\\market_smry.csv");
for (Element table : doc.getElementsByTag("table")) {
for (Element trElement : table.getElementsByTag("tr")) {
trElement2 = trElement.getElementsByTag("td");
tdElements = trElement.getElementsByTag("td");
FileWriter sb = new FileWriter(fnew, true);
if (trElement.hasClass("marketData")) {
for (Iterator<Element> it = tdElements.iterator(); it.hasNext();) {
if (it.hasNext()) {
sb.append("\r\n");
}
for (Iterator<Element> it2 = trElement2.iterator(); it.hasNext();) {
Element tdElement2 = it.next();
final String content = tdElement2.text();
if (it2.hasNext()) {
sb.append(formatData(content));
sb.append(" | ");
}
}
System.out.println(sb.toString());
sb.flush();
sb.close();
}
}
System.out.println(sampleList.add(tdElements));
}
}
}
private static final SimpleDateFormat FORMATTER_MMM_d_yyyy = new SimpleDateFormat("MMM d, yyyy", Locale.US);
private static final SimpleDateFormat FORMATTER_dd_MMM_yyyy = new SimpleDateFormat("dd-MMM-YYYY", Locale.US);
public static String formatData(String text) {
String tmp = null;
try {
Date d = FORMATTER_MMM_d_yyyy.parse(text);
tmp = FORMATTER_dd_MMM_yyyy.format(d);
} catch (ParseException pe) {
tmp = text;
}
return tmp;
}
public static void main(String[] args) throws IOException, Exception {
createConnection();
parsingHTML();
}
}
Now, the problem is when I execute this program it should create a .csv file but what actually happens is it's not creating any file. When I debug this code I found that program is not entering in the loop. I don't understand that why it is doing so. While when I run the same program on the other website which have slightly different page structure it is running fine.
What I understand that this data is present in the #document which is a virtual element and doesn't mean anything that's why program can't read it while there is no such thing in other website. Kindly, help me out to read the data inside the #document element.

Long Story Short
Change your temp url to http://www.psx.com.pk/phps/index1.php
Explanation
There is no table in the document of http://www.psx.com.pk/index.php.
Instead it is showing it's content in two frameset.
One is dummy with url http://www.psx.com.pk/phps/blank.php.
Another one is the real page which is showing actual data and it's url is
http://www.psx.com.pk/phps/index1.php

Java extracting every HTML content in a String list

I m actually making my first RSS reader with JAVA android and I have a problem.
In fact, I get some RSS informations, but there are HTML tags all around.
What I need is to extract every HTML content in these tags and put them in a string list, but I dont know how to do that.
Can you help me with this ?
Thanks for advance

If your rss is xml format, you will need dom4j.jar
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;
public class test {
public static void main(String[] args) throws Exception {
String rssUrl = ""; // paste url here
List<RssDocument> docList = new ArrayList<RssDocument>();
try
{
SAXReader saxReader = new SAXReader();
Document document = saxReader.read(rssUrl);
Element channel = (Element) document.getRootElement().element("channel");
for (Iterator i = channel.elementIterator("item"); i.hasNext();)
{
Element element = (Element) i.next();
String title = element.elementText("title");
String pubDate = element.elementText("pubDate");
String description = element.elementText("description");
RssDocument doc = new RssDocument(title, pubDate, description);
docList.add(doc);
}
}
catch (Exception e)
{
e.printStackTrace();
}
// do something with docList
}
public static class RssDocument {
String title;
String pubDate;
String description;
RssDocument(String title, String pubDate, String description) {
this.title = title;
this.pubDate = pubDate;
this.description = description;
}
}
}
Paste your rss url into variable "rssUrl", and run this main. You will get a list of RSS document, which contains title, published date and description.
If what you need is only the title and description of every rss item, use the following codes.
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;
public class test {
public static void main(String[] args) throws Exception {
String rssUrl = ""; // paste url here
List<String> strList = new ArrayList<String>();
try
{
SAXReader saxReader = new SAXReader();
Document document = saxReader.read(rssUrl);
Element channel = (Element) document.getRootElement().element("channel");
for (Iterator i = channel.elementIterator("item"); i.hasNext();)
{
Element element = (Element) i.next();
String title = element.elementText("title").replaceAll("\\<.*?>","");
String description = element.elementText("description").replaceAll("\\<.*?>","");
strList.add(title + " " + description);
}
}
catch (Exception e)
{
e.printStackTrace();
}
}
}
Then strList will be the list of string, which contains title and description.
For example:
{
"title1 description1"
"title2 description2"
"title3 description3"
}

Assume you have a html content called htmlString, you can clean that with regular expressions.
String htmlString = "<tr><td>12345</td></tr>";
String noHTMLString = htmlString.replaceAll("\\<.*?>","");

This should extract a list of all contents between html tags into the list called matches. You should modify the regex in brackets to match your content. The current version only matches text containing digits, letters, dots, commas, brackets, minuses and spaces.
Pattern pattern = Pattern.compile("<\\w+>([\\w\\s\\.,\\-\\(\\)]+)</\\w+>");
Matcher matcher = pattern.matcher(content);
List<String> matches = new ArrayList<String>();
while(matcher.find()){
matches.add(matcher.group(1));
}

jsoup java html parsing

I'm a new french user on stack and I have a problem ^^
I use an HTML parse Jsoup for parsing a html page. For that it's ok but I can't parse more url in same time.
This is my code:
first class for parsing a web page
package test2;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
public final class Utils {
public static Map<String, String> parse(String url){
Map<String, String> out = new HashMap<String, String>();
try
{
Document doc = Jsoup.connect(url).get();
doc.select("img").remove();
Elements denomination = doc.select(".AmmDenomination");
Elements composition = doc.select(".AmmComposition");
Elements corptexte = doc.select(".AmmCorpTexte");
for(int i = 0; i < denomination.size(); i++)
{
out.put("denomination" + i, denomination.get(i).text());
}
for(int i = 0; i < composition.size(); i++)
{
out.put("composition" + i, composition.get(i).text());
}
for(int i = 0; i < corptexte.size(); i++)
{
out.put("corptexte" + i, corptexte.get(i).text());
System.out.println(corptexte.get(i));
}
} catch(IOException e){
e.printStackTrace();
}
return out;
}//Fin Methode parse
public static void excelizer(int fileId, Map<String, String> values){
try
{
FileOutputStream out = new FileOutputStream("C:/Documents and Settings/c.bon/git/clinsearch/drugs/src/main/resources/META-INF/test/fichier2.xls" );
Workbook wb = new HSSFWorkbook();
Sheet mySheet = wb.createSheet();
Row row1 = mySheet.createRow(0);
Row row2 = mySheet.createRow(1);
String entete[] = {"CIS", "Denomination", "Composition", "Form pharma", "Indication therapeutiques", "Posologie", "Contre indication", "Mise en garde",
"Interraction", "Effet indesirable", "Surdosage", "Pharmacodinamie", "Liste excipients", "Incompatibilité", "Duree conservation",
"Conservation", "Emballage", "Utilisation Manipulation", "TitulaireAMM"};
for (int i = 0; i < entete.length; i++)
{
row1.createCell(i).setCellValue(entete[i]);
}
Set<String> set = values.keySet();
int rowIndexDenom = 1;
int rowIndexCompo = 1;
for(String key : set)
{
if(key.contains("denomination"))
{
mySheet.createRow(1).createCell(1).setCellValue(values.get(key));
rowIndexDenom++;
}
else if(key.contains("composition"))
{
row2.createCell(2).setCellValue(values.get(key));
rowIndexDenom++;
}
}
wb.write(out);
out.close();
}
catch(Exception e)
{
e.printStackTrace();
}
}
}
second class
package test2;
public final class Task extends Thread {
private static int fileId = 0;
private int id;
private String url;
public Task(String url)
{
this.url = url;
id = fileId;
fileId++;
}
#Override
public void run()
{
Utils.excelizer(id, Utils.parse(url));
}
}
the main class (entry point)
package test2;
import java.util.ArrayList;
public class Main {
public static void main(String[] args)
{
ArrayList<String> urls = new ArrayList<String>();
urls.add("http://base-donnees-publique.medicaments.gouv.fr/affichageDoc.php?specid=61266250&typedoc=R");
urls.add("http://base-donnees-publique.medicaments.gouv.fr/affichageDoc.php?specid=66207341&typedoc=R");
for(String url : urls)
{
new Task(url).run();
}
}
}
When the data was copied to my excel file, the second url doesn't work.
Can you help me solve my problem please?
Thanks

I think its because your main() exits before your second thread has a chance to do its job. You should wait for all spawned threads to complete using Thread.join(). Or better yet, create one of the ExecutorService's and use awaitTermination(...) to block until all URLs are parsed.
EDIT See some examples here http://www.javacodegeeks.com/2013/01/java-thread-pool-example-using-executors-and-threadpoolexecutor.html

How can i find APIs for a weather forcast websites?

I making a java application that give the user notification with the weather conditions.
i used the yahoo weather API provided by yahoo like that link :
http://weather.yahooapis.com/forecastrss?w=2502265
and all i have to do is to change the eight numbered code that is in the URL in order to change the city.
that's working perfect, but there are two problems facing me now:
the first one, i want to implement a lot of weather forecast sources in my application not just the yahoo weather and i can't find a similar service in any other weather forecast websites.
the second one, i want to obtain the codes of all the cities in yahoo weather as for sure i won't ask the user to enter his city code, but to enter his city name and i'll match it with the code.
and here is the code that works with me in java:
the code to return the XML file:
package search;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.URL;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import org.w3c.dom.Document;
public class Process {
public static void main(String[] args) throws IOException {
Display disp = new Display();
Document doc = generateXML("1940345");
disp.getConditions(doc);
}
public static Document generateXML(String code) throws IOException {
String url = null;
String XmlData = null;
// creating the URL
url = "http://weather.yahooapis.com/forecastrss?w=" + code;
URL xmlUrl = new URL(url);
InputStream in = xmlUrl.openStream();
// parsing the XmlUrl
Document doc = parse(in);
return doc;
}
public static Document parse(InputStream is) {
Document doc = null;
DocumentBuilderFactory domFactory;
DocumentBuilder builder;
try {
domFactory = DocumentBuilderFactory.newInstance();
domFactory.setValidating(false);
domFactory.setNamespaceAware(false);
builder = domFactory.newDocumentBuilder();
doc = builder.parse(is);
} catch (Exception ex) {
System.err.println("unable to load XML: " + ex);
}
return doc;
}
}
the code to display the temperature and humidity in that city :
package search;
import java.io.File;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
public class Display {
static void getConditions(Document doc) {
String city = null;
String unit = null;
try {
doc.getDocumentElement().normalize();
NodeList nList = doc.getElementsByTagName("rss");
for (int temp = 0; temp < nList.getLength(); temp++) {
Node nNode = nList.item(temp);
if (nNode.getNodeType() == Node.ELEMENT_NODE) {
Element eElement = (Element) nNode;
NodeList nl = eElement
.getElementsByTagName("yweather:location");
for (int tempr = 0; tempr < nl.getLength(); tempr++) {
Node n = nl.item(tempr);
if (nNode.getNodeType() == Node.ELEMENT_NODE) {
Element e = (Element) n;
city = e.getAttribute("city");
System.out.println("The City Is : " + city);
}
}
NodeList nl2 = eElement
.getElementsByTagName("yweather:units");
for (int tempr = 0; tempr < nl2.getLength(); tempr++) {
Node n2 = nl2.item(tempr);
if (nNode.getNodeType() == Node.ELEMENT_NODE) {
Element e2 = (Element) n2;
unit = e2.getAttribute("temperature");
}
}
NodeList nl3 = eElement
.getElementsByTagName("yweather:condition");
for (int tempr = 0; tempr < nl3.getLength(); tempr++) {
Node n3 = nl3.item(tempr);
if (nNode.getNodeType() == Node.ELEMENT_NODE) {
Element e3 = (Element) n3;
System.out.println("The Temperature In " + city
+ " Is : " + e3.getAttribute("temp") + " "
+ unit);
}
}
NodeList nl4 = eElement
.getElementsByTagName("yweather:atmosphere");
for (int tempr = 0; tempr < nl4.getLength(); tempr++) {
Node n4 = nl4.item(tempr);
if (nNode.getNodeType() == Node.ELEMENT_NODE) {
Element e4 = (Element) n4;
System.out.println("The Humidity In " + city
+ " Is : " + e4.getAttribute("humidity"));
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}

You can use Metwit weather api simply passing latitude and longitude.
If you can implement them client-side: 200 request/day (ip based throttling) no authentication required. Worldwide coverage, JSON and REST compliant. You can register for extra API calls for free and if you still need it to call them server side the basic plan is pretty cheap.
Full disclosure: I own this API.

Take a look on this discussion. It seems relevant:
https://stackoverflow.com/questions/4876800/is-there-an-international-weather-forecast-api-that-is-not-limited-for-non-comme
Additionally type "weather forecast api" in google. There are tons of references to APIs that support several weather services.

Here's a list of Weather APIs that are available via the Temboo Java SDK:
https://temboo.com/library/keyword/weather/

You can use YQL (yahoo query language) to find the WOEID by city name like
var lclqry = escape('select * from geo.places where text="OKLAHOMA CITY"')
var lclurl = "http://query.yahooapis.com/v1/public/yql?q=" + lclqry + "&format=json&callback=?";

I know this is an old question, but i found it and as Sai suggested i have written code in java that send YQL query and retrieve WOEID number. Than it uses it to get weather from yahoo-weather-java-api. It needs gson dependecy which you can get by adding dependency to maven. I hope this will help someone.
EDIT
If there is more than one WOEID number for given town name, than getWeather returns weather for town with first WOEID returned.
CODE
Weather.java:
import com.github.fedy2.weather.YahooWeatherService;
import com.github.fedy2.weather.data.Channel;
import com.github.fedy2.weather.data.unit.DegreeUnit;
import com.google.gson.*;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLEncoder;
import javax.xml.bind.JAXBException;
/**
*
* #author robert
*/
public class Weather
{
public Channel getWeather(String townName) throws CantFindWeatherException
{
try
{
String baseUrl = "http://query.yahooapis.com/v1/public/yql?q=";
String query =
"select woeid from geo.places where text=\"" +
townName + "\"";
String fullUrlStr = baseUrl + URLEncoder.encode(query, "UTF-8") +
"&format=json";
URL fullUrl = new URL(fullUrlStr);
ResultObject resultObject = null;
ResultArray resultArray = null;
try (InputStream is = fullUrl.openStream();
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr))
{
String result = "";
String read;
while ((read = br.readLine()) != null)
{
result += read;
}
Gson gson = new Gson();
try
{
resultObject = gson.fromJson(result, ResultObject.class);
}
catch (com.google.gson.JsonSyntaxException ex)
{
resultArray = gson.fromJson(result, ResultArray.class);
}
}
Integer woeid = null;
if (resultObject != null)
{
if (resultObject.query.results != null)
{
woeid = resultObject.query.results.place.woeid;
}
}
else if (resultArray != null)
{
woeid = resultArray.query.results.place[0].woeid;
}
if (woeid != null)
{
YahooWeatherService service = new YahooWeatherService();
Channel channel = service.getForecast(woeid.toString(),
DegreeUnit.CELSIUS);
return channel;
}
else
{
throw new CantFindWeatherException();
}
}
catch (IOException | JsonSyntaxException | JAXBException ex)
{
throw new CantFindWeatherException(ex);
}
}
private static class ResultObject
{
public QueryObject query;
}
private static class ResultArray
{
public QueryArray query;
}
private static class QueryObject
{
public int count;
public String created;
public String lang;
public ResultsObject results;
}
private static class QueryArray
{
public int count;
public String created;
public String lang;
public ResultsArray results;
}
private static class ResultsObject
{
public Place place;
}
private static class ResultsArray
{
public Place[] place;
}
private static class Place
{
public int woeid;
}
}
CantFindWeatherException.java:
/**
*
* #author robert
*/
public class CantFindWeatherException extends Exception
{
public CantFindWeatherException()
{
}
public CantFindWeatherException(String message)
{
super(message);
}
public CantFindWeatherException(String message, Throwable cause)
{
super(message, cause);
}
public CantFindWeatherException(Throwable cause)
{
super(cause);
}
}

As for the first question, I've build a website using forecast.io. It's pretty good. Good API and 1000 free calls/day. It uses latitute/longitude to find the weather of a place.
As for the second question, I would resolve what the user puts in with the Google Geocoding Api. So when they search for "New York", you check if you already have the relevant coordinates in your database, otherwise, you do an API call to Google Geocoding.