We Keep Coding

What is the fastest way to download and re-uploading a file in Java?

I want to download and re-upload a file (close to one GB in size) using standard Java API. The current implementation is:
private void transfer(String sourceUrl, String targetUrl, long size) throws IOException {
logger.info("Copying " + sourceUrl + " to " + targetUrl);
HttpsURLConnection conn = (HttpsURLConnection) new URL(targetUrl).openConnection();
conn.setDoOutput(true);
conn.setRequestMethod("POST");
conn.setRequestProperty("Content-Length", Long.toString(size));
try (InputStream inputStream = new URL(sourceUrl).openStream();
OutputStream outputStream = conn.getOutputStream()) {
byte[] buf = new byte[1 << 20];
int read;
long total = 0;
while ((read = inputStream.read(buf)) != -1) {
outputStream.write(buf, 0, read);
total += read;
logger.info(total + "/" + size + " " + (total * 100 / size) + "%");
}
outputStream.flush();
}
}
However, it is running slow in comparison with a native application. I tried to increase the buffer size. But it didn't help. What can I do to increase the performance of this method?
Can anyone please help me with a multithreaded example or an NIO example which can improve the performance?

How to resume downloading files and show the progress in java

I want to code a way to resume the download file in Java and show the progress if possible.
The following code was used to subtract the total size of the downloaded file (totalSize - downloaded) instead of completing the download.
URL url = new URL(urlFile);
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
File SDCardRoot = Environment.getExternalStorageDirectory();
file = new File(SDCardRoot,"/MySchool/"+Folder+"/"+nameBook.getText().toString()+".pdf");
urlConnection.setRequestProperty("Range", "bytes=" + file.length() + "-");
urlConnection.setDoOutput(true);
urlConnection.connect();
outputStream = new FileOutputStream(file);
InputStream inputStream = urlConnection.getInputStream();
//this is the total size of the file which we are downloading
totalSize = urlConnection.getContentLength();
byte[] buffer = new byte[1024];
int bufferLength = 0;
while ( (bufferLength = inputStream.read(buffer)) > 0 ) {
outputStream.write(buffer, 0, bufferLength);
downloadedSize += bufferLength;
}
outputStream.close();

This might be useful for pause and resume but please specify exact problem.
if (outputFileCache.exists())
{
connection.setAllowUserInteraction(true);
connection.setRequestProperty("Range", "bytes=" + outputFileCache.length() + "-");
}
connection.setConnectTimeout(14000);
connection.setReadTimeout(20000);
connection.connect();
if (connection.getResponseCode() / 100 != 2)
throw new Exception("Invalid response code!");
else
{
String connectionField = connection.getHeaderField("content-range");
if (connectionField != null)
{
String[] connectionRanges = connectionField.substring("bytes=".length()).split("-");
downloadedSize = Long.valueOf(connectionRanges[0]);
}
if (connectionField == null && outputFileCache.exists())
outputFileCache.delete();
fileLength = connection.getContentLength() + downloadedSize;
input = new BufferedInputStream(connection.getInputStream());
output = new RandomAccessFile(outputFileCache, "rw");
output.seek(downloadedSize);
byte data[] = new byte[1024];
int count = 0;
int __progress = 0;
while ((count = input.read(data, 0, 1024)) != -1
&& __progress != 100)
{
downloadedSize += count;
output.write(data, 0, count);
__progress = (int) ((downloadedSize * 100) / fileLength);
}
output.close();
input.close();
}

Java HTTPConnection retrieve file size before downloading

I am trying to download a large (11MB) JSON file from a web service using this code:
public static void downloadBigFile(final String serverUrl,
final String fileName) throws MalformedURLException, IOException {
System.out.println("Downloading " + serverUrl + " (" + fileName + ")");
URL url = new URL(serverUrl);
URLConnection con = url.openConnection();
con.setConnectTimeout(10000);
con.setReadTimeout(2 * 60 * 1000);
int totalFileSize = con.getContentLength();
System.out.println("Total file size: " + totalFileSize);
InputStream inputStream = con.getInputStream();
FileOutputStream outputStream = new FileOutputStream(fileName);
// Used only for knowing the amount of bytes downloaded.
int downloaded = 0;
final byte[] buffer = new byte[1024 * 8];
int bytesRead;
bytesRead = inputStream.read(buffer);
while (bytesRead != -1) {
downloaded += bytesRead;
outputStream.write(buffer, 0, bytesRead);
bytesRead = inputStream.read(buffer);
System.out.println(String.format("%d/%d (%.2f%%)", downloaded,
totalFileSize,
(downloaded * 1.0 / totalFileSize * 1.0) * 100));
}
System.out
.println(fileName + " downloaded! (" + downloaded + " bytes)");
inputStream.close();
outputStream.close();
}
However the call to con.getContentLength() blocks the thread for several minutes, while it downloads the whole file I presume.
The problem is I need a quick way to discover file size before the download starts so I can notify the user accordingly.
Note: already tried to call con.connect() and con.getHeaderField("Content-Length").

If the server does not specify the Content-Length header, the only way to get the content length is to download the whole file and see how big it is.

HTTP Async Downloader with auto-resume on error

I have an async downloader in my app, but sometimes the connection is lost, especially when I'm on a mobile connection and if the file is a large one (>10 MB).
Is there a way to catch when a download stops and then force it to resume with the result of completing the download?
This is the async task doInBackground:
protected String doInBackground(String... aurl) {
int count;
try {
URL url = new URL(aurl[0]);
URLConnection conexion = url.openConnection();
// conexion.setRequestProperty("Range", "bytes=" + downloaded + "-");
conexion.connect();
int lenghtOfFile = conexion.getContentLength();
pesoVideo = lenghtOfFile / 1048576;
output = "/sdcard/" + folderString + "/" + nomeFile + ".mp3";
InputStream input = new BufferedInputStream(url.openStream());
OutputStream output = new FileOutputStream(
VideoDownloaderBrowserActivity.this.output);
byte data[] = new byte[1024];
long total = 0;
while ((count = input.read(data)) != -1) {
total += count;
publishProgress("" + (int) ((total * 100) / lenghtOfFile));
output.write(data, 0, count);
}
output.flush();
output.close();
input.close();
} catch (Exception e) {
}
return null;
}
This is the onProgressUpdate:
protected void onProgressUpdate(String... progress) {
if (Integer.parseInt(progress[0]) > progresso) {
...
}
}

Here is a thread discussing resumable downloading in Android below API 9. Otherwise DownloadManager is a good option too for newer versions.
Basically you need to enable byte serving on your server to allow for the resumable downloading.

Android:"Unexpected end of stream" exception downloading large files

I am building an Android Application and I need to download a file from a url, which is 33 MB large.
Here the download task:
try {
int MAX_BUFFER_SIZE = 4096;
URL mUrl = new URL(params[0]);
HttpURLConnection connection = (HttpURLConnection) mUrl.openConnection();
connection.setRequestMethod("GET");
long length = connection.getContentLength(), downloaded = 0;
int read;
byte [] buffer = new byte[(((int)length) > MAX_BUFFER_SIZE) ? MAX_BUFFER_SIZE : (int)length];
String filename = getFilename(mUrl);
File file = new File (SDCARD_ROOT);
if (!file.exists() || !file.isDirectory()){
file.mkdir();
}
this.filename = filename;
file = new File (SDCARD_ROOT + this.filename);
FileOutputStream fos = new FileOutputStream (file);
//Start downloading
InputStream stream = connection.getInputStream();
while ((read=stream.read(buffer)) > -1){
fos.write(buffer, 0, read);
downloaded += read;
publishProgress((int) ((float) downloaded/length * 100));
}
fos.close();
return 1;
} catch (Exception e){
Log.e("REV-PARTS", "Revolver parts error in DownloadTask: " + e.getMessage());
return 2;
}
It works right with small files (1-15 MB), but it will return a "unexpected end of stream" exception with large files.

Setting a chunk size seemed to work for me.
connection.setChunkedStreamingMode(1048576);

For large files you need to set the connection time out manually by using the following code.
I have set the time out to 3 minutes
connection.setConnectTimeout(180000);
connection.setReadTimeout(180000);

While you catch the exception, I try the method downContinue(). I can show my code:
private void downloadApk() {
thread1 = new Thread() {
public void run() {
File oFile = null;
try {
URL url = new URL(PQGLApplication.resrootURL + "apk/PQGLMap.apk");
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
ReadableByteChannel channel =
Channels.newChannel(urlConnection.getInputStream());
oFile =
new File(Environment.getExternalStorageDirectory().getAbsolutePath()
+ "/" + "hy_ht_new/" + "test2" + ".apk");
oFile.setWritable(true);
oFile.setReadable(true);
if (oFile.exists()) {
oFile.delete();
}
FileOutputStream fos = new FileOutputStream(oFile);
fileSize = urlConnection.getContentLength();
ByteBuffer buffer = ByteBuffer.allocate(1024);
int noOfBytes = 0;
byte[] data = null;
sendApkMessage(0, 0);
while ((noOfBytes = channel.read(buffer)) > 0) {
data = new byte[noOfBytes];
System.arraycopy(buffer.array(), 0, data, 0, noOfBytes);
buffer.clear();
fos.write(data, 0, noOfBytes);
downLoadFileSize += noOfBytes;
sendApkMessage(1, downLoadFileSize);
}
fos.flush();
fos.close();
channel.close();
sendApkMessage(2, oFile.getAbsolutePath());
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
downContinue();
}
};
};
thread1.start();
}
private void downContinue() {
continueTime++;
try {
if (continueTime == 3) {
continueTime = 0;
sendApkMessage(4, 0);
Log.e("what is the continuetime", "continueTime" + continueTime);
} else {
URL url = new URL(PQGLApplication.resrootURL + "apk/PQGLMap.apk");
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
File oFile =
new File(Environment.getExternalStorageDirectory().getAbsolutePath() + "/"
+ "hy_ht_new/" + "test2" + ".apk");
RandomAccessFile oSavedFile = new RandomAccessFile(oFile, "rw");
FileOutputStream fos = new FileOutputStream(oFile);
ReadableByteChannel channel = Channels.newChannel(urlConnection.getInputStream());
// oSavedFile.seek(nPos);
ByteBuffer buffer = ByteBuffer.allocate(1024);
byte[] data = null;
int temp = 0;
sendApkMessage(3, oFile.getAbsolutePath());
while ((temp = channel.read(buffer)) > 0) {
data = new byte[temp];
System.arraycopy(buffer.array(), 0, data, 0, temp);
buffer.clear();
fos.write(data, 0, temp);
}
fos.flush();
fos.close();
oSavedFile.close();
sendApkMessage(2, oFile.getAbsolutePath());
continueTime = 0;
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
Log.e("what is the exception", e.toString() + continueTime);
downContinue();
}
}
This downContinue method is used to solve this problem. At least, the file is downloaded successfully!