We Keep Coding

Combining methods to TAR + LZ4 in a single run in Java

I have two methods that separately TAR a set of files and then another that will compress it with LZ4. They both work fine, but I'm wondering if combining them together would be more efficient or save some time? Also I'm not really sure how I would combine them. Any suggestions would be useful. As you can see in my code below, I'm trying to have fine access to the data so I can give a good % complete to the user.
public static boolean createTarFile(List<Path> paths, Path output, int machineId)
{
boolean success = false;
boolean failure = false;
try (OutputStream fOut = Files.newOutputStream(output, StandardOpenOption.APPEND);
BufferedOutputStream buffOut = new BufferedOutputStream(fOut);
TarArchiveOutputStream tOut = new TarArchiveOutputStream(buffOut))
{
float index = 1;
for (Path path : paths)
{
TarArchiveEntry tarEntry = new TarArchiveEntry(
path.toFile(),
path.getFileName().toString());
tarEntry.setSize(path.toFile().length());
tOut.putArchiveEntry(tarEntry);
// copy file to TarArchiveOutputStream
Files.copy(path, tOut);
tOut.closeArchiveEntry();
tarPercentComplete = (index / (float) paths.size()) * 100;
index++;
if (abort)
{
break;
}
}
tOut.finish();
}
catch (Exception e)
{
LOG.error("Tarring file failed ", e);
failure = true;
}
}
return success;
}
/**
* Zip a tar file using LZ4
*
* #param fileToZip
* #param outputFileName
* #return
*/
public boolean zipFile(File fileToZip, File outputFileName)
{
boolean success = false;
boolean failure = false;
try (FileOutputStream fos = new FileOutputStream(outputFileName);
LZ4FrameOutputStream lz4fos = new LZ4FrameOutputStream(fos);)
{
try (FileInputStream fis = new FileInputStream(fileToZip))
{
byte[] buf = new byte[bufferSizeZip];
int length;
long count = 0;
while ((length = fis.read(buf)) > 0)
{
lz4fos.write(buf, 0, length);
if (count % 50 == 0)
{
zipPercentComplete = ((bufferSizeZip * count) / (float) fileToZip.length()) * 100;
}
count++;
if (abort)
{
break;
}
}
}
}
catch (Exception e)
{
LOG.error("Zipping file failed ", e);
failure = true;
}
}
return success;
}

Just chain them.
try (OutputStream fOut = Files.newOutputStream(output, StandardOpenOption.APPEND);
BufferedOutputStream buffOut = new BufferedOutputStream(fOut);
LZ4FrameOutputStream lz4fos = new LZ4FrameOutputStream(buffOut);)
TarArchiveOutputStream tOut = new TarArchiveOutputStream(lz4fos)) {
No need to name them all.
try (TarArchiveOutputStream tOut = new TarArchiveOutputStream(
new LZ4FrameOutputStream(
new BufferedOutputStream(
Files.newOutputStream(output, StandardOpenOption.APPEND))))) {

Combining tar + lz4 in a single passs will definitely be more efficient,
because it will avoid moving data to/from storage
for a content which is inherently temporary.
On a shell (command line), one would do something like that :
tar cvf - DIR | lz4 > DIR.tar.lz4
which uses stdout / stdin as the intermediate interface (instead of storage file I/O).
However, as you are not using shell in your code,
prefer following suggestions from #Andreas for a Java example.
The main idea is the same, but the implementation is definitely different.

I need my byte compression looked, I don't think it is compressing to where I wanted

I'm trying to compress the byte data from video, the app crashes when the file is over 10mb due to the limitations of Parse servers. So I'm trying to compress at 10mb. The code does well on older phones but on the newer phones the the video bytes easily go over 10mb but the code won't compress the bytes down to 10 mb. Is there something I can add or something?
private void uploadVideo() {
Intent videoIntent = new Intent (MediaStore.ACTION_VIDEO_CAPTURE);
mMediaUri = getOutputMediaFileUri (MEDIA_TYPE_VIDEO);
if (mMediaUri == null) {
// display an error
Toast.makeText (MemberActivity.this, R.string.error_external_storage,
Toast.LENGTH_LONG).show ();
} else {
videoIntent.putExtra (MediaStore.EXTRA_OUTPUT, mMediaUri);
videoIntent.putExtra (MediaStore.EXTRA_DURATION_LIMIT, 10);
videoIntent.putExtra (MediaStore.EXTRA_SIZE_LIMIT, 1024*1024*10);
videoIntent.putExtra (MediaStore.EXTRA_VIDEO_QUALITY, 0); // 0 = lowest res
startActivityForResult (videoIntent, CAMERA_CAPTURE_VIDEO_REQUEST_CODE);
}
String mediaBytes = mMediaUri.getEncodedPath ();
//Set up video file and data
File iFile = new File (mediaBytes);
byte[] byteArray = new byte[(int) iFile.length ()];
try {
FileInputStream fileInputStream = new FileInputStream (iFile);
fileInputStream.read (byteArray);
for (byte aByteArray : byteArray) {
System.out.print ((char) aByteArray);
}
} catch (FileNotFoundException e) {
System.out.println ("File Not Found.");
e.printStackTrace ();
} catch (IOException e1) {
System.out.println ("Error Reading The File.");
e1.printStackTrace ();
}
final ParseFile file = new ParseFile ("HipkuVideo.mp4", byteArray);
file.saveInBackground (new SaveCallback () {
#Override
public void done(ParseException e) {
if (e == null) {
//addPhotoTransferToUserUpload (file);
// Show a simple toast message
Toast.makeText (MemberActivity.this, "Saving to SD Card",
Toast.LENGTH_SHORT).show ();
} else {
// Show a simple toast message
Toast.makeText (MemberActivity.this, "Error saving:" + e.getMessage (),
Toast.LENGTH_SHORT).show ();
}
and here is the compress code:
public static byte[] getByteArrayFromFile(Context context, Uri uri) {
byte[] fileBytes = null;
InputStream inStream = null;
ByteArrayOutputStream outStream = null;
if (uri.getScheme().equals("content")) {
try {
inStream = context.getContentResolver().openInputStream(uri);
outStream = new ByteArrayOutputStream();
// Compressor with highest level of compression
Deflater compressor = new Deflater();
compressor.setLevel(Deflater.BEST_COMPRESSION);
// Give the compressor the data to compress
compressor.setInput(fileBytes);
compressor.finish();
// Create an expandable byte array to hold the compressed data.
// It is not necessary that the compressed data will be smaller than
// the uncompressed data.
outStream = new ByteArrayOutputStream(fileBytes.length);
// Compress the data
byte[] buf = new byte[1024*1024*10];
while (!compressor.finished()) {
int count = compressor.deflate(buf);
outStream.write(buf, 0, count);
}
try {
outStream.close();
} catch (IOException e) {
}
// Get the compressed data
byte[] compressedData = outStream.toByteArray();
byte[] bytesFromFile = new byte[1024]; // buffer size (1 MB)
int bytesRead = inStream.read(compressedData);
while (bytesRead != -1) {
outStream.write(compressedData, 0, bytesRead);
bytesRead = inStream.read(compressedData);
}
fileBytes = outStream.toByteArray();
}
catch (IOException e) {
Log.e(TAG, e.getMessage());
}
finally {
try {
inStream.close();
outStream.close();
}
catch (IOException e) { /*( Intentionally blank */ }
}
}
else {
try {
File file = new File(uri.getPath());
FileInputStream fileInput = new FileInputStream(file);
fileBytes = IOUtils.toByteArray(fileInput);
}
catch (IOException e) {
Log.e(TAG, e.getMessage());
}
}
return fileBytes;
}

If you need to compress further than what's provided by ZLIB, take a look at Apache Commons Compress. Specifically, the section on XZ since that uses the LZMA compression algorithm which at it's highest level gives some of the best compression ratios around. Be sure to thoroughly test performance however since it can get a little slow. If you're worried about space you can cut out the Apache middleman and go straight to the Tukaani java library for the same.

Downloading images overwrites the previous one

void downloadFile(){
try {
Bundle bundle = getIntent().getExtras();
String[] imageUrls = bundle.getStringArray(Extra.IMAGES);
// Log.v(imageUrls, "Creating view...");
int pagerPosition = bundle.getInt(Extra.IMAGE_POSITION, 0);
URL url = new URL(imageUrls[pagerPosition]);
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("GET");
urlConnection.setDoOutput(true);
//connect
urlConnection.connect();
//set the path where we want to save the file
File SDCardRoot = Environment.getExternalStorageDirectory();
//create a new file, to save the downloaded file
File file = new File(SDCardRoot,"image.png");
FileOutputStream fileOutput = new FileOutputStream(file);
//Stream used for reading the data from the internet
InputStream inputStream = urlConnection.getInputStream();
//this is the total size of the file which we are downloading
totalSize = urlConnection.getContentLength();
runOnUiThread(new Runnable() {
public void run() {
pb.setMax(totalSize);
}
});
//create a buffer...
byte[] buffer = new byte[1024];
int bufferLength = 0;
while ( (bufferLength = inputStream.read(buffer)) > 0 ) {
fileOutput.write(buffer, 0, bufferLength);
downloadedSize += bufferLength;
// update the progressbar //
runOnUiThread(new Runnable() {
public void run() {
pb.setProgress(downloadedSize);
float per = ((float)downloadedSize/totalSize) * 100;
cur_val.setText("Downloaded " + downloadedSize + "KB / " + totalSize + "KB (" + (int)per + "%)" );
}
});
}
//close the output stream when complete //
fileOutput.close();
runOnUiThread(new Runnable() {
public void run() {
// pb.dismiss(); // if you want close it..
}
});
} catch (final MalformedURLException e) {
showError("Error : MalformedURLException " + e);
e.printStackTrace();
} catch (final IOException e) {
showError("Error : IOException " + e);
e.printStackTrace();
}
catch (final Exception e) {
showError("Error : Please check your internet connection " + e);
}
}
Hi, I am using this function for downloading images from URLs which is working fine. But it overwrites the previously downloaded image..
File file = new File(SDCardRoot,"image.png");
How can I change the name with every new download in the above line.. its not accepting anything which is not a string.
Any help would be great.

Its because you always save images with same name image.png ,Change name using current datetime as
So change this
File file = new File(SDCardRoot,"image.png");
to
SimpleDateFormat formatter = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss");
Date now = new Date();
String fileName = "image" + formatter.format(now) + ".png";
File file = new File(SDCardRoot,fileName);
Edit
You may also use from Calendar as
Calendar calendar = Calendar.getInstance();
SimpleDateFormat formatter = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss");
String fileName = "image" + formatter.format(calendar.getTime()) + ".png";
File file = new File(SDCardRoot,fileName);

You can Give by this:
Calender calendar=Calender.getInstance();
String fileName = "image_" + calendar.getTimeInMillis() + ".png";
File file = new File(SDCardRoot,fileName);

Android : How to read file in bytes?

I am trying to get file content in bytes in Android application. I have get the file in SD card now want to get the selected file in bytes. I googled but no such success. Please help
Below is the code to get files with extension. Through this i get files and show in spinner. On file selection I want to get file in bytes.
private List<String> getListOfFiles(String path) {
File files = new File(path);
FileFilter filter = new FileFilter() {
private final List<String> exts = Arrays.asList("jpeg", "jpg", "png", "bmp", "gif","mp3");
public boolean accept(File pathname) {
String ext;
String path = pathname.getPath();
ext = path.substring(path.lastIndexOf(".") + 1);
return exts.contains(ext);
}
};
final File [] filesFound = files.listFiles(filter);
List<String> list = new ArrayList<String>();
if (filesFound != null && filesFound.length > 0) {
for (File file : filesFound) {
list.add(file.getName());
}
}
return list;
}

here it's a simple:
File file = new File(path);
int size = (int) file.length();
byte[] bytes = new byte[size];
try {
BufferedInputStream buf = new BufferedInputStream(new FileInputStream(file));
buf.read(bytes, 0, bytes.length);
buf.close();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
Add permission in manifest.xml:
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />

The easiest solution today is to used Apache common io :
http://commons.apache.org/proper/commons-io/javadocs/api-release/org/apache/commons/io/FileUtils.html#readFileToByteArray(java.io.File)
byte bytes[] = FileUtils.readFileToByteArray(photoFile)
The only drawback is to add this dependency in your build.gradle app :
implementation 'commons-io:commons-io:2.5'
+ 1562 Methods count

Since the accepted BufferedInputStream#read isn't guaranteed to read everything, rather than keeping track of the buffer sizes myself, I used this approach:
byte bytes[] = new byte[(int) file.length()];
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file));
DataInputStream dis = new DataInputStream(bis);
dis.readFully(bytes);
Blocks until a full read is complete, and doesn't require extra imports.

Here is a solution that guarantees entire file will be read, that requires no libraries and is efficient:
byte[] fullyReadFileToBytes(File f) throws IOException {
int size = (int) f.length();
byte bytes[] = new byte[size];
byte tmpBuff[] = new byte[size];
FileInputStream fis= new FileInputStream(f);;
try {
int read = fis.read(bytes, 0, size);
if (read < size) {
int remain = size - read;
while (remain > 0) {
read = fis.read(tmpBuff, 0, remain);
System.arraycopy(tmpBuff, 0, bytes, size - remain, read);
remain -= read;
}
}
} catch (IOException e){
throw e;
} finally {
fis.close();
}
return bytes;
}
NOTE: it assumes file size is less than MAX_INT bytes, you can add handling for that if you want.

If you want to use a the openFileInput method from a Context for this, you can use the following code.
This will create a BufferArrayOutputStream and append each byte as it's read from the file to it.
/**
* <p>
* Creates a InputStream for a file using the specified Context
* and returns the Bytes read from the file.
* </p>
*
* #param context The context to use.
* #param file The file to read from.
* #return The array of bytes read from the file, or null if no file was found.
*/
public static byte[] read(Context context, String file) throws IOException {
byte[] ret = null;
if (context != null) {
try {
InputStream inputStream = context.openFileInput(file);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
int nextByte = inputStream.read();
while (nextByte != -1) {
outputStream.write(nextByte);
nextByte = inputStream.read();
}
ret = outputStream.toByteArray();
} catch (FileNotFoundException ignored) { }
}
return ret;
}

In Kotlin you can simply use:
File(path).readBytes()

You can also do it this way:
byte[] getBytes (File file)
{
FileInputStream input = null;
if (file.exists()) try
{
input = new FileInputStream (file);
int len = (int) file.length();
byte[] data = new byte[len];
int count, total = 0;
while ((count = input.read (data, total, len - total)) > 0) total += count;
return data;
}
catch (Exception ex)
{
ex.printStackTrace();
}
finally
{
if (input != null) try
{
input.close();
}
catch (Exception ex)
{
ex.printStackTrace();
}
}
return null;
}

A simple InputStream will do
byte[] fileToBytes(File file){
byte[] bytes = new byte[0];
try(FileInputStream inputStream = new FileInputStream(file)) {
bytes = new byte[inputStream.available()];
//noinspection ResultOfMethodCallIgnored
inputStream.read(bytes);
} catch (IOException e) {
e.printStackTrace();
}
return bytes;
}

Following is the working solution to read the entire file in chunks and its efficient solution to read the large files using a scanner class.
try {
FileInputStream fiStream = new FileInputStream(inputFile_name);
Scanner sc = null;
try {
sc = new Scanner(fiStream);
while (sc.hasNextLine()) {
String line = sc.nextLine();
byte[] buf = line.getBytes();
}
} finally {
if (fiStream != null) {
fiStream.close();
}
if (sc != null) {
sc.close();
}
}
}catch (Exception e){
Log.e(TAG, "Exception: " + e.toString());
}

To read a file in bytes, often used to read binary files, such as pictures, sounds, images, etc.
Use the method below.
public static byte[] readFileByBytes(File file) {
byte[] tempBuf = new byte[100];
int byteRead;
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
try {
BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(file));
while ((byteRead = bufferedInputStream.read(tempBuf)) != -1) {
byteArrayOutputStream.write(tempBuf, 0, byteRead);
}
bufferedInputStream.close();
return byteArrayOutputStream.toByteArray();
} catch (Exception e) {
e.printStackTrace();
return null;
}
}

how to add ProgressMonitorInputStream to ftp upload?

Can anybody see what is wrong with this code. it does not show up progress-bar but uploades all the files.
I did checkout sun tutorial and swingworkers also but i couldn't fix it yet.
private static boolean putFile(String m_sLocalFile, FtpClient m_client) {
boolean success = false;
int BUFFER_SIZE = 10240;
if (m_sLocalFile.length() == 0) {
System.out.println("Please enter file name");
}
byte[] buffer = new byte[BUFFER_SIZE];
try {
File f = new File(m_sLocalFile);
int size = (int) f.length();
System.out.println("File " + m_sLocalFile + ": " + size + " bytes");
System.out.println(size);
FileInputStream in = new FileInputStream(m_sLocalFile);
//test
InputStream inputStream = new BufferedInputStream(
new ProgressMonitorInputStream(null,"Uploading " + f.getName(),in));
//test
OutputStream out = m_client.put(f.getName());
int counter = 0;
while (true) {
int bytes = inputStream.read(buffer); //in
if (bytes < 0)
break;
out.write(buffer, 0, bytes);
counter += bytes;
System.out.println(counter);
}
out.close();
in.close();
inputStream.close();
success =true;
} catch (Exception ex) {
System.out.println("Error: " + ex.toString());
}
return true;
}

I think your code is fine.
Maybe the task isn't taking long enough for the progress bar to be needed?
Here's a modified version of your code which reads from a local file and writes to another local file.
I have also added a delay to the write so that it gives the progress bar time to kick in.
This works fine on my system with a sample 12MB PDF file, and shows the progress bar.
If you have a smaller file then just increase the sleep from 5 milliseconds to 100 or something - you would need to experiment.
And I didn't even know that the ProgressMonitorInputStream class existed, so I've learnt something myself ;].
/**
* main
*/
public static void main(String[] args) {
try {
System.out.println("start");
final String inf = "d:/testfile.pdf";
final String outf = "d:/testfile.tmp.pdf";
final FileOutputStream out = new FileOutputStream(outf) {
#Override
public void write(byte[] b, int off, int len) throws IOException {
super.write(b, off, len);
try {
// We delay the write by a few millis to give the progress bar time to kick in
Thread.sleep(5);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
};
putFile(inf, out);
System.out.println("end");
} catch (FileNotFoundException e) {
e.printStackTrace();
}
}
private static boolean putFile(String m_sLocalFile, OutputStream out /*FtpClient m_client*/) {
boolean success = false;
int BUFFER_SIZE = 10240;
if (m_sLocalFile.length() == 0) {
System.out.println("Please enter file name");
}
byte[] buffer = new byte[BUFFER_SIZE];
try {
File f = new File(m_sLocalFile);
int size = (int) f.length();
System.out.println("File " + m_sLocalFile + ": " + size + " bytes");
System.out.println(size);
FileInputStream in = new FileInputStream(m_sLocalFile);
//test
InputStream inputStream = new BufferedInputStream(
new ProgressMonitorInputStream(null,"Uploading " + f.getName(),in));
//test
//OutputStream out = m_client.put(f.getName());
int counter = 0;
while (true) {
int bytes = inputStream.read(buffer); //in
if (bytes < 0)
break;
out.write(buffer, 0, bytes);
counter += bytes;
System.out.println(counter);
}
out.close();
in.close();
inputStream.close();
success =true;
} catch (Exception ex) {
System.out.println("Error: " + ex.toString());
}
return true;
}