I am using twitter4j twitter Streaming API to get the tweets for the specific tag.
I am having number of keywords. I want to search the 100 tweets thats containing that tag
currently what i am doing is i wrote code for getting the tweets for single word
public class StreamAPI {
public static void main(String[] args) {
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setDebugEnabled(true);
cb.setOAuthConsumerKey("xxxx");
cb.setOAuthConsumerSecret("xxxxx");
cb.setOAuthAccessToken("xxxx");
cb.setOAuthAccessTokenSecret("xxxx");
cb.setUseSSL(true);
cb.setUserStreamRepliesAllEnabled(true);
TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
twitterStream.setOAuthAccessToken(accestoken);
StatusListener listener = new StatusListener() {
int countTweets = 0;
public void onStatus(Status status) {
System.out.println("#" + status.getUser().getScreenName() + " - " + status.getText());
countTweets++;
System.out.println(countTweets);
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
}
public void onScrubGeo(long userId, long upToStatusId) {
System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
}
#Override
public void onStallWarning(StallWarning stallWarning) {
//To change body of implemented methods use File | Settings | File Templates.
}
public void onException(Exception ex) {
ex.printStackTrace();
}
};
FilterQuery fq = new FilterQuery();
String keywords[] = {"ipl"};
fq.track(keywords);
twitterStream.addListener(listener);
twitterStream.filter(fq);
}
}
how would i stop the process after it reaches the count 100 and should return that 100 tweet as list.
Please help me.
see the below code maybe helpfull for you
String token= "Key Word";
Query query = new Query(token);
FileWriter outFile = new FileWriter(token.replaceAll("^#","").concat(".txt"), true);
int numberOfTweets = 1500;
long lastID = Long.MAX_VALUE;
ArrayList<Status> tweets = new ArrayList<Status>();
while (tweets.size () < numberOfTweets) {
if (numberOfTweets - tweets.size() > 100)
query.setCount(100);
else
query.setCount(numberOfTweets - tweets.size());
try {
QueryResult result = twitter.search(query);
tweets.addAll(result.getTweets());
System.out.println("Gathered " + tweets.size() + " tweets");
for (Status t: tweets)
if(t.getId() < lastID) lastID = t.getId(); }
catch (TwitterException te) {
System.out.println("Couldn't connect: " + te); };
query.setMaxId(lastID-1);
}
PrintWriter out1 = new PrintWriter(outFile);
for (int i = 0; i < tweets.size(); i++) {
Status t = (Status) tweets.get(i);
GeoLocation loc = t.getGeoLocation();
String user = t.getUser().getScreenName();
String msg = t.getText();
String time = "";
if (loc!=null) {
Double lat = t.getGeoLocation().getLatitude();
Double lon = t.getGeoLocation().getLongitude();
System.out.println(i + " USER: " + user + " wrote: " + msg + " located at " + lat + ", " + lon);}
else
// System.out.println(i + " USER: " + user + " wrote: " + msg.replaceAll("\n",""));
out1.append(i + " USER: " + user + " wrote: " +msg.replaceAll("\n"," ") );
out1.print("\n");
}
System.out.println("file write succefully");
Related
Currently, I'm am trying to parse from MealMaster files, but I am having an issue where Ingredients are being parsed as:
"Inch thick" due to the next line not having a quantity or unit, and carrying on from the previous
Also, I'm finding ingredients that are listed as "ingredient1 or ingredient2" and I'm not sure how to catagorise these in the parser
Here is an example of a file I'm parsing from and my code below
https://pastebin.com/fhkRczya
public void readIngredients() {
try {
Remover remover = new Remover();
ArrayList<Ingredient> ing = new ArrayList<Ingredient>();
while(!( "".equals(line.trim()))) {
parsedIngredients = line + "\n";
if(!line.contains("---") && !line.contains(":")) {
Ingredient currentIng = splitLine();
if(currentIng.getQuantity().length() == 0 && !ing.isEmpty()) {
Ingredient lastIng = ing.get(ing.size()-1);
if (currentIng.getName().toLowerCase().contains("inch") ) {
//System.out.println(currentIng.getName());
lastIng.setOther(lastIng.getOther() + "," + currentIng.getQuantity() + "," +currentIng.getName());
//System.out.println("OTher " + lastIng.getOther());
}else{
String lastIngName = lastIng.getName();
String addName = lastIngName + " " + currentIng.getName();
lastIng.setName(addName);
lastIng = remover.removeTo(unitWords,lastIng);
lastIng = remover.removeCustomWords(lastIng);
}
}else if (currentIng.getName().startsWith("-") || currentIng.getName().startsWith("For") ){
if(ing.size()>0) {
Ingredient lastIng = ing.get(ing.size()-1);
lastIng.setOther(currentIng.getQuantity() + " " + currentIng.getName());
}
}else {
currentIng = remover.removeTo(unitWords,currentIng);
currentIng = remover.removeCustomWords(currentIng);
//currentIng.setName(currentIng.getName().replace(",", ""));
System.out.println(currentIng.getName());
ing.add(currentIng);
}
}
line = reader.readLine();
}
for(int i = 0; i < ing.size();i++) {
removeCommaColon(ing.get(i));
}
for(int i = 0; i<ing.size();i++) {
ingredientsString = ingredientsString + ing.get(i).getName() + "|" + currentRecipe.getTitle() + " \n";
//ingredientsString = ingredientsString + currentRecipe.getTitle() + "\n";
}
currentRecipe.setIngredients(ing);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public List getPayslipsOfEmployees(String empids, Date paydate) {
initializeTransaction();
String strSQL = "SELECT "
+ " pd.* "
+ " FROM "
+ " payroll_d pd "
+ " INNER JOIN "
+ " payroll_h ph "
+ " ON pd.pay_uid = ph.pay_uid "
+ " WHERE pd.employee_id IN ("+empids+") "
+ " AND ph.pay_date = DATE('"+sql_dateformat.format(paydate)+"')";
SQLQuery q = session.createSQLQuery(strSQL);
q.addEntity("pd", PayrollD.class);
List pdlist = q.list();
commit();
List<Payslip> pslips = new ArrayList<Payslip>();
if (!pdlist.isEmpty()) {
Iterator it = pdlist.iterator();
while (it.hasNext()) {
try {
PayrollD payd = (PayrollD)it.next();
Payslip ps = new Payslip();
ps.setDepartment(payd.getEmployeeCatalog().getDepartmentCatalog().getDeptName());
ps.setEmployeeID(payd.getEmployeeCatalog().getEmployeeId());
ps.setEmployeeName(payd.getEmployeeCatalog().getEmpLastname(), payd.getEmployeeCatalog().getEmpFirstname(), payd.getEmployeeCatalog().getEmpMiddlename());
ps.setStartPayPeriod(payd.getPayrollH().getPayFromdate());
ps.setEndPayPeriod(payd.getPayrollH().getPayTodate());
ps.setDesignation(payd.getEmployeeCatalog().getJobDesignation());
EmployeeExemption exmptn = (EmployeeExemption)getCurrentExemption(payd.getEmployeeCatalog().getEmployeeId());
ps.setExemption((exmptn != null) ? exmptn.getExemptionCode() : SysConstants.DEFAULT_EXEMPTION);
List<PayslipEarning> earnings = new ArrayList<PayslipEarning>();
List<PayslipDeduction> deductions = new ArrayList<PayslipDeduction>();
ps.setNetPay(getPayslipNetPay(payd, earnings, deductions));
ps.setPrevTotalTaxableEarning(getPrevTotalTaxableEarning(payd));
ps.setTotalWithheld(getTotalWithheld(payd));
ps.setEarnings(earnings);
ps.setDeductions(deductions);
pslips.add(ps);
} catch (Exception e) {
JOptionPane.showMessageDialog(null, "Somethings wrong " + e.getMessage());
}
}
}
return pslips;
}
I am getting this function by this funtion
private void generatePaySummary() {
try {
Map params = new HashMap();
params = getOrganizationInfo(params);
params.put("rptsubtitle", "Payroll Date: "+date_formatter.format(tbpaydate.getDate()));
int i = cboDept.getSelectedIndex();
int deptno = 0;
if (i != -1) deptno = (Integer)deptnos.get(i);
ReportService srv = new ReportService();
List empids = srv.getEmployeesInPayroll(deptno, tbpaydate.getDate());
if (!empids.isEmpty()) {
PayslipService.setEmployees(empids);
PayslipService.setPayDate(tbpaydate.getDate());
RepGenService repsrv = new RepGenService();
JRBeanCollectionDataSource jbsrc = new JRBeanCollectionDataSource(PaySummaryFactory.getPaySummary());
repsrv.generateReport(false, "/orgpayroll/reports/jasper/payrollsummary.jasper", true, params, jbsrc);
}
else
SysUtils.messageBox("No employees in payroll on "+date_formatter.format(tbpaydate.getDate())+"!");
} catch (Exception e) {
JOptionPane.showMessageDialog(null, "Error" + e.getMessage());
}
}
When I try to run this function to get the payroll and employee information of the employee, it says Hibernate could not initialize proxy - no session. I can't find which one is causing the error. It works when I only process one employee of the same date, but when I process two employees on the same date, the error occurs.
I am trying to use I/O to give a report on the stock that I need (if the stock is below 8).
It tells me it requires an int for myShop.listLowStockToFile());; when I add a number it tells me that 'void is not allowed here'. How can I fix this?
public void listLowStockToFile(int threshhold)
{
System.out.println("****The Stock that is getting low is: " + " Minimum " +threshhold + " Report for Bob Shaw****\n");
for (Item nextItem : items)
{
if(nextItem.getNuminStock() < threshhold)
{
System.out.println(nextItem);
}
}
}
public class Report {
public static void main(String[] args) {
Shop myShop = new Shop();
CD cd1 = new CD("Abba Gold", "Abba", 15);
myShop.addItem(cd1);
Game game1 = new Game("Chess", 2, 39.95);
myShop.addItem(game1);
ElectronicGame eg1 = new ElectronicGame("Shrek", "PS2", 1, 79.50);
myShop.addItem(eg1);
ElectronicGame eg2 = new ElectronicGame("Doom", "PC", 2, 30.20);
myShop.addItem(eg2);
ElectronicGame eg3 = new ElectronicGame("AFL", "PS2", 2, 49.95);
myShop.addItem(eg3);
cd1.receiveStock(3);
game1.receiveStock(5);
eg1.receiveStock(10);
eg2.receiveStock(1);
cd1.receiveStock(7);
cd1.sellCopy(true);
cd1.sellCopy(true);
eg2.sellCopy(true);
myShop.listItems();
myShop.listLowStockToFile(8);
myShop.listGamesByPlatform("PS2");
myShop.calcTotalSales();
Game game2 = new Game("Chess", 2, 39.95);
myShop.addItem(game2);
eg2.sellCopy(false);
try {
BufferedWriter writer = new BufferedWriter(new FileWriter("LowStock.txt"));
writer.write("Report dated" + new Date() + "\n");
writer.write(myShop.listLowStockToFile()); // This line.
writer.close();
System.out.println("Report finished");
} catch (Exception ex) {
System.out.println("File I/O error" + ex);
}
}
}
You need listLowStockToFile to return a String:
public String listLowStockToFile(int threshhold) {
String rtn = "****The Stock that is getting low is: " + " Minimum " +threshhold + " Report for Bob Shaw****\n";
for (Item nextItem : items) {
if(nextItem.getNuminStock() < threshhold) {
rtn += nextItem.toString() + "\n";
}
}
System.out.print(rtn);
return rtn;
}
The reason is that BufferedWritter.write takes a String as an argument.
I created a button that allows to create a zip file., the function that zips the file works correctly, but when I call her via the button (in the JS file) it crashes and it gives a blank page (I think I do not manage the output stream)
would please an idea
here is my code:
Button
isc.ToolStripButton.create({
ID: "BooksApp_GetXmlImage_Button"
,autoDraw:false
,icon: getUIIcon("icon_xml_16")
,prompt: getUIMsg("book_report_get_xml",4)
,showHover:true
,hoverStyle:"book_hover_style"
,click : function () {
BooksApp_Action_loadFile("objx");
// isc.say("test");
}
});
function to call the zipfile() method:
function BooksApp_Action_loadFile(p_UsedFormat) {
var tmpBookID = BooksApp_Application.FP_BookID;
var tmpIDs = BooksApp_Application.FP_fct_getSelectedPOVIDs();
var tmpUsr_ID = FPIUser.FP_fct_getID();
var tmpFormat = p_UsedFormat;
var showInWindow=false;
books_objects.exportData(
{
r_book_idnum : tmpBookID
,sBook_ID : tmpBookID
,sPOV_IDs : tmpIDs
,sUser_ID : tmpUsr_ID
,sFormat : tmpFormat
}
,{ operationId: "customExport"
,exportDisplay: (showInWindow ? "window" : "download") }
,function (dsResponse, data, dsRequest) {
//Never called
BooksApp_Action_Log("BooksApp_Action_loadFile:"+data);
}
);
}
customExport() function
public static String customExport(RPCManager rpc,
HttpServletResponse response) throws Exception {
String sReturn = _Return_OK;
try {
// setting doCustomResponse() notifies the RPCManager that we'll
// bypass RPCManager.send
// and instead write directly to the servletResponse output stream
rpc.doCustomResponse();
RequestContext.setNoCacheHeaders(response);
writeServerDebug("customExport : start");
DSRequest req = rpc.getDSRequest();
List<?> results = req.execute().getDataList();
String sReqData = (String) req.getParameter("exportDisplay");
String sReqData_sBook_ID = "" + req.getCriteriaValue("sBook_ID");
String sReqData_sPOV_IDs = "" + req.getCriteriaValue("sPOV_IDs");
String sReqData_sUser_ID = "" + req.getCriteriaValue("sUser_ID");
String sReqData_sFormat = "" + req.getCriteriaValue("sFormat");
StringBuilder content = new StringBuilder("get (sReqData:"
+ sReqData + ",sBook_ID:" + sReqData_sBook_ID
+ ",sPOV_IDs:" + sReqData_sPOV_IDs + ",sUser_ID:"
+ sReqData_sUser_ID + ",sFormat:" + sReqData_sFormat + ")"
+ results.size() + " line(s):");
for (Iterator<?> i = results.iterator(); i.hasNext();) {
Map<?, ?> record = (Map<?, ?>) i.next();
content.append("\n" + Books.Column_IDNum + ":"
+ record.get(Books.Column_IDNum));
content.append("\n" + Books.Column_Name + ":"
+ record.get(Books.Column_Name));
}
writeServerDebug("The content is \n" + content.toString());
// Create the new Office Engine
OfficeEngine myOfficeEngine = new OfficeEngine();
boolean bIsConnected = myOfficeEngine._RepositoryBridge
.connectSourceDataBase(false);
if (bIsConnected) {
//Connected to the repository, so get the files
if (sReqData_sFormat.equalsIgnoreCase("pdf") || sReqData_sFormat.equalsIgnoreCase("pptx")) {
//The book end user format
String sReturnPptx = myOfficeEngine.performGeneratePptx(
req.getHttpServletRequest(), response,
sReqData_sBook_ID, sReqData_sPOV_IDs,
sReqData_sUser_ID, sReqData_sFormat);
writeServerDebug("customExport call performGeneratePptx, return is "
+ sReturnPptx);
}
else {
AppZip appZip = new AppZip();
appZip.ZipFile(" ", " ");
String r = "sReturn_OK";;
return r;
}
//Free the connection to repository
myOfficeEngine._RepositoryBridge.freeConnectionSource();
} else {
response.setContentType("text/plain");
response.addHeader("content-disposition",
"attachment; filename=book.txt");
ServletOutputStream os = response.getOutputStream();
os.print(content.toString());
os.flush();
}
} catch (Exception e) {
writeServerDebug("ERROR:" + e.getLocalizedMessage());
sReturn = Repository._Return_KO;
}
return sReturn;
}
Basics of this program;
Runs a webcrawler based on PerentUrl and Keyword specified by the user in Controller (main). If the Keyword is found in the page text, the Url is then saved to an array list;
ArrayList UrlHits = new ArrayList();
Once the crawl is complete the program will call methods from the WriteFile class in the main to write a html file containing all the UrlHits.
WriteFile f = new WriteFile();
f.openfile(Search);
f.StartHtml();
f.addUrl(UrlHits);
f.EndHtml();
f.closeFile();
All but f.addUrl work correctly, creating a html file with the correct name and directory. But none of the strings from the ArrayList output to the file.
public static void main(String[] args) throws Exception {
RobotstxtConfig robotstxtConfig2 = new RobotstxtConfig();
String crawlStorageFolder = "/Users/Jake/Documents/sem 2/FYP/Crawler/TestData";
int numberOfCrawlers = 1;
CrawlConfig config = new CrawlConfig();
config.setCrawlStorageFolder(crawlStorageFolder);
config.setMaxDepthOfCrawling(21);
config.setMaxPagesToFetch(24);
PageFetcher pageFetcher = new PageFetcher(config);
RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);
Scanner perentUrl = new Scanner(System.in);
System.out.println("Enter full perant Url... example. http://www.domain.co.uk/");
String Url = perentUrl.nextLine();
Scanner keyword = new Scanner(System.in);
System.out.println("Enter search term... example. Pies");
String Search = keyword.nextLine();
System.out.println("Searching domain :" + Url);
System.out.println("Keyword:" + Search);
ArrayList<String> DomainsToInv = new ArrayList<String>();
ArrayList<String> SearchTerms = new ArrayList<String>();
ArrayList<String> UrlHits = new ArrayList<String>();
DomainsToInv.add(Url);
SearchTerms.add(Search);
controller.addSeed(Url);
controller.setCustomData(DomainsToInv);
controller.setCustomData(SearchTerms);
controller.start(Crawler.class, numberOfCrawlers);
WriteFile f = new WriteFile();
f.openfile(Search);
f.StartHtml();
f.addUrl(UrlHits);
f.EndHtml();
f.closeFile();
}
}
public class Crawler extends WebCrawler {
#Override
public void visit(Page page) {
int docid = page.getWebURL().getDocid();
String url = page.getWebURL().getURL();
String domain = page.getWebURL().getDomain();
String path = page.getWebURL().getPath();
String subDomain = page.getWebURL().getSubDomain();
String parentUrl = page.getWebURL().getParentUrl();
String anchor = page.getWebURL().getAnchor();
System.out.println("Docid: " + docid);
System.out.println("URL: " + url);
System.out.println("Domain: '" + domain + "'");
System.out.println("Sub-domain: '" + subDomain + "'");
System.out.println("Path: '" + path + "'");
System.out.println("Parent page: " + parentUrl);
System.out.println("Anchor text: " + anchor);
if (page.getParseData() instanceof HtmlParseData) {
HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
String text = htmlParseData.getText();
String html = htmlParseData.getHtml();
List<WebURL> links = htmlParseData.getOutgoingUrls();
System.out.println("Text length: " + text.length());
System.out.println("Html length: " + html.length());
System.out.println("Number of outgoing links: " + links.size());
}
Header[] responseHeaders = page.getFetchResponseHeaders();
if (responseHeaders != null) {
System.out.println("Response headers:");
for (Header header : responseHeaders) {
System.out.println("\t" + header.getName() + ": " + header.getValue());
}
}
System.out.println("=============");
ArrayList<String> SearchTerms = (ArrayList<String>) this.getMyController().getCustomData();
ArrayList<String> UrlHits = (ArrayList<String>) this.getMyController().getCustomData();
for (String Keyword : SearchTerms) {
System.out.println("Searching Keyword: " + Keyword);
HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
int KeywordCounter = 0;
String pagetext = htmlParseData.getText();
Pattern pattern = Pattern.compile(Keyword);
Matcher match1 = pattern.matcher(pagetext);
if (match1.find()) {
while (match1.find()) {
KeywordCounter++;
}
System.out.println("FOUND " + Keyword + " in page text. KeywordCount: " + KeywordCounter);
UrlHits.add(url);
for (int i = 0; i < UrlHits.size(); i++) {
System.out.print(UrlHits.get(i) + "\n");
System.out.println("=============");
}
} else {
System.out.println("Keyword search was unsuccesful");
System.out.println("=============");
}
}
}
}
public class WriteFile {
private Formatter x;
public void openfile(String keyword) {
try {
x = new Formatter(keyword + ".html");
} catch (Exception e) {
System.out.println("ERROR");
}
}
public void StartHtml() {
x.format("%s %n %s %n %s %n %s %n %s %n ", "<html>", "<head>", "</head>", "<body>", "<center>");
}
public void addUrl(ArrayList<String> UrlHits) {
for (String list : UrlHits) {
x.format("%s%s%s%s%s%n%s%n", "", list, "", "<br>");
}
}
public void EndHtml() {
x.format("%s %n %s %n %s %n", "</center>", "</body>", "</html>");
}
public void closeFile() {
x.close();
}
}
Apologies for the class headers outside the code blocks its a little fiddly. I have tried a few different "for" statements to get the method to output the array list but it doesn't seem to be having it. The strings are being added to the array list as i can call them using a for loop in the main. But when i pass the array list to the method addUrl, it comes up with squat. is there an easier way to use arraylists using formatters and .format?
Thanks for you help