We Keep Coding

Process file (which sit on Server A) on Server B

Since Server A and server B doesn't have SFTP, and I am try to implement a web service on Server B, that takes the file on Server A and process it. I tried using Spring boot to do this, like save the file first, and the process. But this way it seems like async, which means when the code try to process the file, the file is not ready yet, (confirm when I print the file location, it return null). What is the good way to handle this?
Current Code I have for the controller is following:
package com.example.filedemo.controller;
import com.example.filedemo.payload.UploadFileResponse;
import com.example.filedemo.service.FileStorageService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.support.ServletUriComponentsBuilder;
import javax.servlet.http.HttpServletRequest;
import javax.xml.ws.Response;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
#RestController
public class FileController {
private static final Logger logger = LoggerFactory.getLogger(FileController.class);
private static final String PYTHON_FILE = "V:/speechRecognition/audio_transcribe.py";
#Autowired
private FileStorageService fileStorageService;
#PostMapping("/uploadFile")
public UploadFileResponse uploadFile(#RequestParam("file") MultipartFile file) {
String fileName = fileStorageService.storeFile(file);
String fileDownloadUri =
ServletUriComponentsBuilder.fromCurrentContextPath().path("/downloadFile/").path(fileName).toUriString();
return new UploadFileResponse(fileName, fileDownloadUri, file.getContentType(), file.getSize());
}
#PostMapping("/processFile")
public ResponseEntity<String> processFile(#RequestParam("file") MultipartFile file) throws IOException {
String filename = uploadFile(file).getFileName();
File actualFile = new File("E:\\Audio\\uploads\\" + filename);
String fetching = "python " + PYTHON_FILE + " " + actualFile.getAbsolutePath();
String[] command = new String[] {"cmd.exe", "/c", fetching};
Process p = Runtime.getRuntime().exec(command);
String pythonPath = System.getProperty("PYTHON_PATH");
System.out.println("pythonPath is " + pythonPath);
BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()));
String ret = in.readLine();
System.out.println(ret);
return new ResponseEntity<String>("Success", HttpStatus.OK);
}
}
FileStorageService.java
package com.example.filedemo.service;
import com.example.filedemo.exception.FileStorageException;
import com.example.filedemo.exception.MyFileNotFoundException;
import com.example.filedemo.property.FileStorageProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
import org.springframework.core.io.UrlResource;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.net.MalformedURLException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
#Service
public class FileStorageService {
private final Path fileStorageLocation;
#Autowired
public FileStorageService(FileStorageProperties fileStorageProperties) {
this.fileStorageLocation = Paths.get(fileStorageProperties.getUploadDir())
.toAbsolutePath().normalize();
try {
Files.createDirectories(this.fileStorageLocation);
} catch (Exception ex) {
throw new FileStorageException("Could not create the directory where the uploaded files will be stored.", ex);
}
}
public String storeFile(MultipartFile file) {
// Normalize file name
String fileName = StringUtils.cleanPath(file.getOriginalFilename());
try {
// Check if the file's name contains invalid characters
if(fileName.contains("..")) {
throw new FileStorageException("Sorry! Filename contains invalid path sequence " + fileName);
}
// Copy file to the target location (Replacing existing file with the same name)
Path targetLocation = this.fileStorageLocation.resolve(fileName);
long numberOfByte = Files.copy(file.getInputStream(), targetLocation, StandardCopyOption.REPLACE_EXISTING);
System.out.println("Copy byte " + numberOfByte);
return fileName;
} catch (IOException ex) {
throw new FileStorageException("Could not store file " + fileName + ". Please try again!", ex);
}
}
public Resource loadFileAsResource(String fileName) {
try {
Path filePath = this.fileStorageLocation.resolve(fileName).normalize();
Resource resource = new UrlResource(filePath.toUri());
if(resource.exists()) {
return resource;
} else {
throw new MyFileNotFoundException("File not found " + fileName);
}
} catch (MalformedURLException ex) {
throw new MyFileNotFoundException("File not found " + fileName, ex);
}
}
}
UploadFileResponse
package com.example.filedemo.payload;
public class UploadFileResponse {
private String fileName;
private String fileDownloadUri;
private String fileType;
private long size;
public UploadFileResponse(String fileName, String fileDownloadUri, String fileType, long size) {
this.fileName = fileName;
this.fileDownloadUri = fileDownloadUri;
this.fileType = fileType;
this.size = size;
}
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public String getFileDownloadUri() {
return fileDownloadUri;
}
public void setFileDownloadUri(String fileDownloadUri) {
this.fileDownloadUri = fileDownloadUri;
}
public String getFileType() {
return fileType;
}
public void setFileType(String fileType) {
this.fileType = fileType;
}
public long getSize() {
return size;
}
public void setSize(long size) {
this.size = size;
}
}

I would recommend storing the file in the database, then operating on it from there. Obviously, be careful the file size doesn't overrun the column size.
If your operations require an actual file on disk (rather than, say, a stream of bytes), then you could create a sync service that polls the database for new files and saves a copy on the local disk when found. Then your processing code can use the local disk file handle.

Yahoo Finance down loader stopped working

I have a JAVA program that i have been using for the last year to downlaod data from yahoo finance. A few days ago it stopped working and yahoo started to refuse the connection.
java.net.ConnectException: Connection refused: connect
Has anyone else experienced this?
import java.net.URL;
import java.net.URLConnection;
import java.util.Calendar;
import java.util.Scanner;
import java.util.GregorianCalendar;
import java.util.Calendar;
import java.util.ArrayList;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.Scanner;
public class C_GetStocksFromYahoo {
public Scanner input;
public void GetStocks(String symbol, GregorianCalendar start, GregorianCalendar end) {
String url = "http://real-chart.finance.yahoo.com/table.csv?s="
+ symbol
+ "&a=" + start.get(Calendar.MONTH)
+ "&b=" + start.get(Calendar.DAY_OF_MONTH)
+ "&c=" + start.get(Calendar.YEAR)
+ "&d=" + end.get(Calendar.MONTH)
+ "&e=" +end.get(Calendar.DAY_OF_MONTH)
+ "&f=" + end.get(Calendar.YEAR)
+ "&g=d&ignore=.csv"; //g=d means daily stock quotes
try {
URL yhoofin=new URL(url);//Create New URL data
URLConnection data=yhoofin.openConnection();//Open a connection
input = new Scanner(data.getInputStream());//get data from input stream
//System.out.println("This is the scanner " + input);
//System.out.println("This is the first line " + this.nextline());
//System.out.println("This is the first line " + this.nextline());
} catch (Exception e) {
System.out.println("This is the symbol i failed on " + symbol);
System.err.println(e);
}
}
//scanner value
public Scanner getScan() {
return input;
}
//determine if file has next entry
public boolean hasnext() {
return input.hasNext();
}
//contains next line
public String nextline() {
return input.nextLine();
}
public void closeConnection() {
input.close();
}
}

Unfortunately Yahoo discontinued their API for historical data, therefor your URL http://real-chart.finance.yahoo.com/table.csv? won't get you any data back anymore.
On https://forums.yahoo.net/t5/Yahoo-Finance-help/Is-Yahoo-Finance-API-broken/td-p/250503/page/3 Nixon (Official 'Hoo Staff) said:
Hi All - This feature was discontinued by the Finance team and they will not be reintroducing that functionality.
Taken from: Yahoo Finance URL not working

Jsoup Reddit Image Scraper Over 18 issue

I'm working on an image scraper that scrapes the first page of various subreddits using JSOUP. The issue that arises however is when attempting to scrape a NSFW subreddit, reddit redirects to an over 18 authentication page and the scraper scrapes the authentication page instead. I'm new to scraping and understand this is a noob question, but any help would be much appreciated as I am totally lost.
import java.io.IOException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.io.*;
import java.net.URL;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.io.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Attributes;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.net.URL;
import java.util.Scanner;
public class javascraper{
public static final String USER_AGENT = "<User-Agent: github.com/dabeermasood:v1.2.3 (by /u/swedenotswiss)>";
public static void main (String[]args) throws MalformedURLException
{
Scanner scan = new Scanner (System.in);
System.out.println("Where do you want to store the files?");
String folderpath = scan.next();
System.out.println("What subreddit do you want to scrape?");
String subreddit = scan.next();
subreddit = ("http://reddit.com/r/" + subreddit);
new File(folderpath + "/" + subreddit).mkdir();
//test
try{
//gets http protocol
Document doc = Jsoup.connect(subreddit).userAgent(USER_AGENT).timeout(0).get();
//get page title
String title = doc.title();
System.out.println("title : " + title);
//get all links
Elements links = doc.select("a[href]");
for(Element link : links){
//get value from href attribute
String checkLink = link.attr("href");
Elements images = doc.select("img[src~=(?i)\\.(png|jpe?g|gif)]");
if (imgCheck(checkLink)){ // checks to see if img link j
System.out.println("link : " + link.attr("href"));
downloadImages(checkLink, folderpath);
}
}
}
catch (IOException e){
e.printStackTrace();
}
}
public static boolean imgCheck(String http){
String png = ".png";
String jpg = ".jpg";
String jpeg = "jpeg"; // no period so checker will only check last four characaters
String gif = ".gif";
int length = http.length();
if (http.contains(png)|| http.contains("gfycat") || http.contains(jpg)|| http.contains(jpeg) || http.contains(gif)){
return true;
}
else{
return false;
}
}
private static void downloadImages(String src, String folderpath) throws IOException{
String folder = null;
//Exctract the name of the image from the src attribute
int indexname = src.lastIndexOf("/");
if (indexname == src.length()) {
src = src.substring(1, indexname);
}
indexname = src.lastIndexOf("/");
String name = src.substring(indexname, src.length());
System.out.println(name);
//Open a URL Stream
URLConnection connection = (new URL(src)).openConnection();
try {
Thread.sleep(2000);
} catch (InterruptedException e) {
e.printStackTrace();
} //Delay to comply with rate limiting
connection.setRequestProperty("User-Agent", USER_AGENT);
InputStream in = connection.getInputStream();
OutputStream out = new BufferedOutputStream(new FileOutputStream( folderpath+ name));
for (int b; (b = in.read()) != -1;) {
out.write(b);
}
out.close();
in.close();
}
}

I've posted an answer to authenticate against the server using Jsoup in this link. Basically you need to POST your login ID & password and other required data to the server using:
Connection.Response res = Jsoup.connect(url).data(...).method(Method.Post).execute();, then save the response cookie from the server to keep your session authenticated.

Jar ignoring all methods

I was making a pretty simple jar to unzip a zip and run the jar that was inside of it. The problem I've run into is that it doesn't do anything at all.
This is the main, and only class file for the jar. The manifest does point correctly to it, and it loads without errors.
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.FileWriter;
import java.io.BufferedWriter;
import java.io.BufferedOutputStream;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.io.InputStreamReader;
import static java.lang.Integer.parseInt;
import java.net.URLConnection;
import java.net.URL;
import java.util.zip.ZipFile;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.Enumeration;
import sign.signlink;
import java.nio.file.*;
import java.io.FileReader;
public class ClientUpdater {
private String fileToExtractNew = "/client.zip";
private String getJarDir() throws FileNotFoundException, IOException{
String linebuf="",verStr="";
FileInputStream fis = new FileInputStream("/runLocationURL.txt");
BufferedReader br= new BufferedReader(new InputStreamReader(fis));
while ((linebuf = br.readLine()) != null) {
verStr = linebuf;
}
return verStr;
}
public static void main(String[] args) {
System.out.println("start");
}
private void unZip() {
System.out.println("unzipping");
try {
ZipEntry zipEntry;
//client
BufferedInputStream bufferedInputStreamNew = new BufferedInputStream(new FileInputStream(this.fileToExtractNew));
ZipInputStream zipInputStreamNew = new ZipInputStream(bufferedInputStreamNew);
//client
while ((zipEntry = zipInputStreamNew.getNextEntry()) != null) {
String stringNew = zipEntry.getName();
File fileNew = new File(this.getJarDir() + File.separator + stringNew);
if (zipEntry.isDirectory()) {
new File(this.getJarDir() + zipEntry.getName()).mkdirs();
continue;
}
if (zipEntry.getName().equals(this.fileToExtractNew)) {
this.unzipNew(zipInputStreamNew, this.fileToExtractNew);
break;
}
new File(fileNew.getParent()).mkdirs();
this.unzipNew(zipInputStreamNew, this.getJarDir() + zipEntry.getName());
}
zipInputStreamNew.close();
}
catch (Exception var1_2) {
var1_2.printStackTrace();
}
}
private void unzipNew(ZipInputStream zipInputStreamNew, String stringNew) throws IOException {
System.out.println("unzipping new");
FileOutputStream fileOutputStreamNew = new FileOutputStream(stringNew);
byte[] arrby = new byte[4024];
int n = 0;
while ((n = zipInputStreamNew.read(arrby)) != -1) {
fileOutputStreamNew.write(arrby, 0, n);
}
fileOutputStreamNew.close();
Runtime.getRuntime().exec("java -jar " + getJarDir() + "/Project Pk Client.jar");
System.exit(0);
}
}
It shows the "Start" message, but not the other 2, so it never reaches those methods. Is it because they aren't being called? I'm still learning Java.

You actually have to call your other methods from main. Right now, all you are telling the computer to do is print start and then exit. Functions do not get called simply by existing.
It seems based on a quick glance that you just need to add unzip(); to your main function, right after the System.out.println line.
To do this, you need to say that those other methods are static, so you need to say private static void unZip() instead of private void unZip(). Do this for your other methods too.
import java.io.*;
import static java.lang.Integer.parseInt;
import java.net.URLConnection;
import java.net.URL;
import java.util.zip.ZipFile;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.Enumeration;
import sign.signlink;
import java.nio.file.*;
public class ClientUpdater {
private String fileToExtractNew = "/client.zip";
private static String getJarDir() throws FileNotFoundException, IOException{
String linebuf="",verStr="";
FileInputStream fis = new FileInputStream("/runLocationURL.txt");
BufferedReader br= new BufferedReader(new InputStreamReader(fis));
while ((linebuf = br.readLine()) != null) {
verStr = linebuf;
}
return verStr;
}
public static void main(String[] args) {
System.out.println("start");
unZip();
}
private static void unZip() {
System.out.println("unzipping");
try {
ZipEntry zipEntry;
//client
BufferedInputStream bufferedInputStreamNew = new BufferedInputStream(new FileInputStream(this.fileToExtractNew));
ZipInputStream zipInputStreamNew = new ZipInputStream(bufferedInputStreamNew);
//client
while ((zipEntry = zipInputStreamNew.getNextEntry()) != null) {
String stringNew = zipEntry.getName();
File fileNew = new File(this.getJarDir() + File.separator + stringNew);
if (zipEntry.isDirectory()) {
new File(this.getJarDir() + zipEntry.getName()).mkdirs();
continue;
}
if (zipEntry.getName().equals(this.fileToExtractNew)) {
this.unzipNew(zipInputStreamNew, this.fileToExtractNew);
break;
}
new File(fileNew.getParent()).mkdirs();
this.unzipNew(zipInputStreamNew, this.getJarDir() + zipEntry.getName());
}
zipInputStreamNew.close();
}
catch (Exception var1_2) {
var1_2.printStackTrace();
}
}
private static void unzipNew(ZipInputStream zipInputStreamNew, String stringNew) throws IOException {
System.out.println("unzipping new");
FileOutputStream fileOutputStreamNew = new FileOutputStream(stringNew);
byte[] arrby = new byte[4024];
int n = 0;
while ((n = zipInputStreamNew.read(arrby)) != -1) {
fileOutputStreamNew.write(arrby, 0, n);
}
fileOutputStreamNew.close();
Runtime.getRuntime().exec("java -jar " + getJarDir() + "/Project Pk Client.jar");
System.exit(0);
}
}

Client Server Program String leak?

My Server Program:-
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.ServerSocket;
import java.net.Socket;
import javax.print.PrintService;
import javax.print.PrintServiceLookup;
public class GetFileServeredit implements Runnable {
public static final int SERVERPORT = 4747;
public String FileName=null;
public void run() {
try {
ServerSocket svr=new ServerSocket(SERVERPORT);
while(true){
System.out.println("S: Waiting...");
Socket sktClient=svr.accept();
System.out.println("S: Receiving...");
try{
PrintService services[] = PrintServiceLookup.lookupPrintServices(null, null);
PrintWriter out2 = new PrintWriter(new BufferedWriter(new OutputStreamWriter(sktClient.getOutputStream())),true);
/*for(int z=0;z<services.length;z++){
out2.println(services[z]);
}*/
out2.println("aaa 12212");
out2.flush();
out2.close();
sktClient.close();
System.out.println("Transfer complete.");
}
catch(Exception e){
e.printStackTrace();
}
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static void main(String [] args){
Thread servThread =new Thread(new GetFileServeredit());
servThread.start();
}
}
My Client Program
:-
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.Socket;
import java.util.ArrayList;
public class ClientPrinters implements Runnable {
static final int PORT = 4747; //Change this to the relevant port
static final String HOST = "192.***.*.***"; //Change this to the relevant HOST,//(where Server.java is running)
public void run() {
try {
System.out.print("Sending data...\n");
Socket skt = new Socket(HOST, PORT);
ArrayList Printers =new ArrayList();
InputStream inStream = skt.getInputStream();
BufferedReader inm = new BufferedReader(new InputStreamReader(inStream));
while ((inm.read()) != -1) {
Printers.add(inm.readLine());
}
inm.close();
inStream.close();
skt.close();
}
catch( Exception e ) {
System.out.print("Error! It didn't work! " + e + "\n");
}
}
public static void main(String [] args){
Thread cThread =new Thread(new ClientPrinters());
cThread.start();
}
}
The out that i am sending form the server i.e
out2.println("aaa 12212");
becomes 12212 only at the client side,why? where is the other text??

while ((inm.read()) != -1) {
Printers.add(inm.readLine());
}
This line first reads a single and if that succeeds, it tries to read a line. This swallows (and ignores) one character per line.
Also: you don't specify any character encodings in your server and client: This will work (with some restrictions) as long as client and server run using the same locale. But it will break once they use different default encodings.
It's probably best to just specify the encoding to use "on the wire". A great candidate for this is UTF-8:
// server
PrintWriter out2 = new PrintWriter(new BufferedWriter(new OutputStreamWriter(sktClient.getOutputStream(), "UTF-8")),true);
// client
BufferedReader inm = new BufferedReader(new InputStreamReader(inStream, "UTF-8"));

I was not running your code,but.
while ((inm.read()) != -1) {
}
I think .read() consumed some byte.
you should use some buffer(byte[]) and call .read(buffer)!=-1

Perhaps what you intended was
List<String> printers =new ArrayList<String>();
String line;
while ((line = inm.readLine()) != null)
printers.add(line);
Note: use camelCase for field/variable names.