We Keep Coding

Java - Adding File to the Archive without deleting other files

So I have this code, it pretty much does what it should but my archive ends up broken and it doesn't save the files. Of course I have to achieve these without using FileSystem, no TempFiles or anything.
public static void main(String[] args) throws IOException {
// Path fileName = Paths.get(args[0]);
// String pathZip = args[1];
Path fileName = Paths.get("C:\\Users\\dell\\Desktop\\addfile.txt");
String pathZip = "C:\\Users\\dell\\Desktop\\test.zip";
Map<String, byte[]> zipEntryMap = addFilesInMap(pathZip);
zipEntryMap.forEach((zipEntryName, bytes) -> {
System.out.println(zipEntryName+" "+bytes.toString());
try {
containAndSaveSameFiles(pathZip, bytes, zipEntryName);
} catch (Exception e) {
e.printStackTrace();
}
});
// saveFileInArchive(fileName, pathZip);
}
private static Map<String, byte[]> addFilesInMap(String pathZip) throws IOException {
Map<String, byte[]> zipEntryMap = new HashMap<>();
FileInputStream fileInputStream = new FileInputStream(pathZip);
ZipInputStream zipInputStream = new ZipInputStream(fileInputStream);
ZipEntry zipEntry;
while((zipEntry = zipInputStream.getNextEntry())!= null){
byte[] buffer = new byte[1024];
ByteArrayOutputStream builder = new ByteArrayOutputStream();
int end;
while((end = zipInputStream.read(buffer)) > 0){
builder.write(buffer, 0, end);
}
zipEntryMap.put(zipEntry.getName(), builder.toByteArray());
}
return zipEntryMap;
}
private static void containAndSaveSameFiles(String pathZip, byte[] bytes, String zipEntryName) throws Exception{
ByteArrayOutputStream readBytes = new ByteArrayOutputStream();
FileOutputStream fileOutputStream = new FileOutputStream(pathZip);
ZipOutputStream outputStream = new ZipOutputStream(readBytes);
ZipEntry zipEntry2 = new ZipEntry(zipEntryName);
zipEntry2.setSize(bytes.length);
outputStream.putNextEntry(new ZipEntry(zipEntryName));
outputStream.write(bytes);
}
private static void saveFileInArchive(Path fileToBeAdded, String pathToArchive) throws IOException {
FileOutputStream fileOutputStream = new FileOutputStream(pathToArchive);
ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream);
zipOutputStream.putNextEntry(new ZipEntry("new/"+fileToBeAdded.getFileName()));
Files.copy(fileToBeAdded, zipOutputStream);
zipOutputStream.close();
fileOutputStream.close();
}
I tried a few ways, and look up on the internet but can't find any good answer.
Thank you for help.

Your code is almost correct.
Bug No:1 in containAndSaveSameFiles
Using readBytes instead of fileOutputStream.
Bug No:2 in saveFileInArchive Rewriting OutputStream by reopening it again.
Complete code after review:
public static void main(String[] args) throws IOException {
// Path fileName = Paths.get(args[0]);
// String pathZip = args[1];
Path fileName = Paths.get("C:\\Users\\dell\\Desktop\\addfile.txt");
String pathZip = "C:\\Users\\dell\\Desktop\\test.zip";
Map<String, byte[]> zipEntryMap = addFilesInMap(pathZip);
FileOutputStream fileOutputStream = new FileOutputStream(pathZip);
ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream);
zipEntryMap.forEach((zipEntryName, bytes) -> {
System.out.println(zipEntryName+" "+bytes.toString());
try {
containAndSaveSameFiles(pathZip, bytes, zipEntryName, zipOutputStream);
} catch (Exception e) {
e.printStackTrace();
}
});
saveFileInArchive(fileName, pathZip,zipOutputStream);
zipOutputStream.close();
fileOutputStream.close();
}
private static Map<String, byte[]> addFilesInMap(String pathZip) throws IOException {
Map<String, byte[]> zipEntryMap = new HashMap<>();
FileInputStream fileInputStream = new FileInputStream(pathZip);
ZipInputStream zipInputStream = new ZipInputStream(fileInputStream);
ZipEntry zipEntry;
while((zipEntry = zipInputStream.getNextEntry())!= null){
byte[] buffer = new byte[1024];
ByteArrayOutputStream builder = new ByteArrayOutputStream();
int end;
while((end = zipInputStream.read(buffer)) > 0){
builder.write(buffer, 0, end);
}
zipEntryMap.put(zipEntry.getName(), builder.toByteArray());
}
return zipEntryMap;
}
private static void containAndSaveSameFiles(String pathZip, byte[] bytes, String zipEntryName, ZipOutputStream zipOutputStream) throws Exception{
// ByteArrayOutputStream readBytes = new ByteArrayOutputStream();
ZipEntry zipEntry2 = new ZipEntry(zipEntryName);
zipEntry2.setSize(bytes.length);
zipOutputStream.putNextEntry(new ZipEntry(zipEntryName));
zipOutputStream.write(bytes);
}
private static void saveFileInArchive(Path fileToBeAdded, String pathToArchive, ZipOutputStream zipOutputStream) throws IOException, IOException {
zipOutputStream.putNextEntry(new ZipEntry("new/"+fileToBeAdded.getFileName()));
Files.copy(fileToBeAdded, zipOutputStream);
}

Java zip files from streams instantly without using byte[]

I want to compress multiples files into a zip files, I'm dealing with big files, and then download them into the client, for the moment I'm using this:
#RequestMapping(value = "/download", method = RequestMethod.GET, produces = "application/zip")
public ResponseEntity <StreamingResponseBody> getFile() throws Exception {
File zippedFile = new File("test.zip");
FileOutputStream fos = new FileOutputStream(zippedFile);
ZipOutputStream zos = new ZipOutputStream(fos);
InputStream[] streams = getStreamsFromAzure();
for (InputStream stream: streams) {
addToZipFile(zos, stream);
}
final InputStream fecFile = new FileInputStream(zippedFile);
Long fileLength = zippedFile.length();
StreamingResponseBody stream = outputStream - >
readAndWrite(fecFile, outputStream);
return ResponseEntity.ok()
.header(HttpHeaders.ACCESS_CONTROL_EXPOSE_HEADERS, HttpHeaders.CONTENT_DISPOSITION)
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + "download.zip")
.contentLength(fileLength)
.contentType(MediaType.parseMediaType("application/zip"))
.body(stream);
}
private void addToZipFile(ZipOutputStream zos, InputStream fis) throws IOException {
ZipEntry zipEntry = new ZipEntry(generateFileName());
zos.putNextEntry(zipEntry);
byte[] bytes = new byte[1024];
int length;
while ((length = fis.read(bytes)) >= 0) {
zos.write(bytes, 0, length);
}
zos.closeEntry();
fis.close();
}
This take a lot of time before all files are zipped and then the downloading start, and for large files this kan take a lot of time, this is the line responsible for the delay:
while ((length = fis.read(bytes)) >= 0) {
zos.write(bytes, 0, length);
}
So is there a way to download files immediately while their being zipped ?

Try this instead. Rather than using the ZipOutputStream to wrap a FileOutputStream, writing your zip to a file, then copying it to the client output stream, instead just use the ZipOutputStream to wrap the client output stream so that when you add zip entries and data it goes directly to the client. If you want to also store it to a file on the server then you can make your ZipOutputStream write to a split output stream, to write both locations at once.
#RequestMapping(value = "/download", method = RequestMethod.GET, produces = "application/zip")
public ResponseEntity<StreamingResponseBody> getFile() throws Exception {
InputStream[] streamsToZip = getStreamsFromAzure();
// You could cache already created zip files, maybe something like this:
// String[] pathsOfResourcesToZip = getPathsFromAzure();
// String zipId = getZipId(pathsOfResourcesToZip);
// if(isZipExist(zipId))
// // return that zip file
// else do the following
StreamingResponseBody streamResponse = clientOut -> {
FileOutputStream zipFileOut = new FileOutputStream("test.zip");
ZipOutputStream zos = new ZipOutputStream(new SplitOutputStream(clientOut, zipFileOut));
for (InputStream in : streamsToZip) {
addToZipFile(zos, in);
}
};
return ResponseEntity.ok()
.header(HttpHeaders.ACCESS_CONTROL_EXPOSE_HEADERS, HttpHeaders.CONTENT_DISPOSITION)
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + "download.zip")
.contentType(MediaType.parseMediaType("application/zip")).body(streamResponse);
}
private void addToZipFile(ZipOutputStream zos, InputStream fis) throws IOException {
ZipEntry zipEntry = new ZipEntry(generateFileName());
zos.putNextEntry(zipEntry);
byte[] bytes = new byte[1024];
int length;
while ((length = fis.read(bytes)) >= 0) {
zos.write(bytes, 0, length);
}
zos.closeEntry();
fis.close();
}
public static class SplitOutputStream extends OutputStream {
private final OutputStream out1;
private final OutputStream out2;
public SplitOutputStream(OutputStream out1, OutputStream out2) {
this.out1 = out1;
this.out2 = out2;
}
#Override public void write(int b) throws IOException {
out1.write(b);
out2.write(b);
}
#Override public void write(byte b[]) throws IOException {
out1.write(b);
out2.write(b);
}
#Override public void write(byte b[], int off, int len) throws IOException {
out1.write(b, off, len);
out2.write(b, off, len);
}
#Override public void flush() throws IOException {
out1.flush();
out2.flush();
}
/** Closes all the streams. If there was an IOException this throws the first one. */
#Override public void close() throws IOException {
IOException ioException = null;
for (OutputStream o : new OutputStream[] {
out1,
out2 }) {
try {
o.close();
} catch (IOException e) {
if (ioException == null) {
ioException = e;
}
}
}
if (ioException != null) {
throw ioException;
}
}
}
For the first request for a set of resources to be zipped you wont know the size that the resulting zip file will be so you can't send the length along with the response since you are streaming the file as it is zipped.
But if you expect there to be repeated requests for the same set of resources to be zipped, then you can cache your zip files and simply return them on any subsequent requests; You will also know the length of the cached zip file so you can send that in the response as well.
If you want to do this then you will have to be able to consistently create the same identifier for each combination of the resources to be zipped, so that you can check if those resources were already zipped and return the cached file if they were. You might be able to could sort the ids (maybe full paths) of the resources that will be zipped and concatenate them to create an id for the zip file.

Spring boot: Download zipped large files while they are being processed on the server

In my app I'm zipping and then downloading larges files, the files are located in azure, so I read the files from a stream and then zip them one after another, so I can dowload the zip file after all files has been zipped, here's my code:
#RequestMapping(value = "{analyseId}/download", method = RequestMethod.GET, produces = "application/zip")
public ResponseEntity<Resource> download(#PathVariable List<String> paths) throws IOException {
String zipFileName = "zipFiles.zip";
File zipFile = new File(zipFileName);
FileOutputStream fos = new FileOutputStream(zipFile);
ZipOutputStream zos = new ZipOutputStream(fos);
for (String path : paths) {
InputStream fis = azureDataLakeStoreService.readFile(path);
addToZipFile(path , zos, fis);
}
zos.close();
fos.close();
BufferedInputStream zipFileInputStream = new BufferedInputStream(new FileInputStream(zipFile.getAbsolutePath()));
InputStreamResource resource = new InputStreamResource(zipFileInputStream);
zipFile.delete();
return ResponseEntity.ok()
.header(HttpHeaders.ACCESS_CONTROL_EXPOSE_HEADERS, HttpHeaders.CONTENT_DISPOSITION)
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + zipFileName)
.contentType(MediaType.parseMediaType("application/octet-stream"))
.body(resource);
}
private static void addToZipFile(String path, ZipOutputStream zos, InputStream fis) throws IOException {
ZipEntry zipEntry = new ZipEntry(FilenameUtils.getName(path));
zos.putNextEntry(zipEntry);
byte[] bytes = new byte[1024];
int length;
while ((length = fis.read(bytes)) >= 0) {
zos.write(bytes, 0, length);
}
zos.closeEntry();
fis.close();
}
However on azure the request time out is set to 230 sec, and cannot be changed, however for big files it takes more than that to load and then zip the files on the server, so the connection with the client will be lost meanwhile.
So my question is since I'm getting the data from a stream, can we do all these operations simultaneously, means getting the stream and download it as the same time and not waiting till getting the whole file, or if there any other idea can any body share it here please.
Thanks.

The answer is to not download the file to the server and then send it to the client but streaming it to the client directly here's the code
#RequestMapping(value = "/download", method = RequestMethod.GET)
public StreamingResponseBody download(#PathVariable String path) throws IOException {
final InputStream fecFile = azureDataLakeStoreService.readFile(path);
return (os) -> {
readAndWrite(fecFile, os);
};
}
private void readAndWrite(final InputStream is, OutputStream os)
throws IOException {
byte[] data = new byte[2048];
int read = 0;
while ((read = is.read(data)) >= 0) {
os.write(data, 0, read);
}
os.flush();
}
I also added this configuration to ApplicationInit:
#Configuration
public static class WebConfig extends WebMvcConfigurerAdapter {
#Override
public void configureAsyncSupport(AsyncSupportConfigurer configurer) {
configurer.setDefaultTimeout(-1);
configurer.setTaskExecutor(asyncTaskExecutor());
}
#Bean
public AsyncTaskExecutor asyncTaskExecutor() {
return new SimpleAsyncTaskExecutor("async");
}
}

Java create InputStream from ZipInputStream entry

I would like to write a method that read several XML files inside a ZIP, from a single InputStream.
The method would open a ZipInputStream, and on each xml file, get the corresponding InputStream, and give it to my XML parser. Here is the skeleton of the method :
private void readZip(InputStream is) throws IOException {
ZipInputStream zis = new ZipInputStream(is);
ZipEntry entry = zis.getNextEntry();
while (entry != null) {
if (entry.getName().endsWith(".xml")) {
// READ THE STREAM
}
entry = zis.getNextEntry();
}
}
The problematic part is the "// READ THE STREAM". I have a working solution, which consist to create a ByteArrayInputStream, and feed my parser with it. But it uses a buffer, and for large files I get an OutOfMemoryError. Here is the code, if someone is still interested :
int count;
byte buffer[] = new byte[2048];
ByteArrayOutputStream out = new ByteArrayOutputStream();
while ((count = zis.read(buffer)) != -1) { out.write(buffer, 0, count); }
InputStream is = new ByteArrayInputStream(out.toByteArray());
The ideal solution would be to feed the parser with the original ZipInputStream. It should works, because it works if I just print the entry content with a Scanner :
Scanner sc = new Scanner(zis);
while (sc.hasNextLine())
{
System.out.println(sc.nextLine());
}
But... The parser I'm currently using (jdom2, but I also tried with javax.xml.parsers.DocumentBuilderFactory) closes the stream after parsing the data :/ . So I'm unable to get the next entry and continue.
So finally the question is :
Does anybody know a DOM parser that doesn't close its stream ?
Is there another way to have an InputStream from a ZipEntry ?
Thanks.

A small improvement on Tim's solution: The problem with having to call allowToBeClosed() before close() is that it makes closing the ZipInputStream properly when handling exceptions tricky and will break Java 7's try-with-resources statement.
I suggest creating a wrapper class as follows:
public class UncloseableInputStream extends InputStream {
private final InputStream input;
public UncloseableInputStream(InputStream input) {
this.input = input;
}
#Override
public void close() throws IOException {} // do not close the wrapped stream
#Override
public int read() throws IOException {
return input.read();
}
// delegate all other InputStream methods as with read above
}
which can then safely be used as follows:
try (ZipInputStream zipIn = new ZipInputStream(...))
{
DocumentBuilder db = DocumentBuilderFactory.newInstance().newDocumentBuilder();
ZipEntry entry;
while (null != (entry = zipIn.getNextEntry()))
{
if ("file.xml".equals(entry.getName())
{
Document doc = db.parse(new UncloseableInputStream(zipIn));
}
}
}

Thanks to halfbit, I ended up with my own ZipInputStream class, which overrides the close method :
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.ZipInputStream;
public class CustomZipInputStream extends ZipInputStream {
private boolean _canBeClosed = false;
public CustomZipInputStream(InputStream is) {
super(is);
}
#Override
public void close() throws IOException {
if(_canBeClosed) super.close();
}
public void allowToBeClosed() { _canBeClosed = true; }
}

You could wrap the ZipInputStream and intercept the call to close().

If you don't mind external dependencies, Apache Commons IO provides a convenience class named CloseShieldInputStream for blocking the close() call.
private void readZip(InputStream is) throws IOException {
ZipInputStream zis = new ZipInputStream(is);
ZipEntry entry = zis.getNextEntry();
while (entry != null) {
if (entry.getName().endsWith(".xml")) {
//commons-io 2.9 and later
InputStream tempIs = CloseShieldInputStream.wrap(zis);
//commons-io < 2.9
//InputStream tempIs = new CloseShieldInputStream(zis);
// READ THE STREAM
}
entry = zis.getNextEntry();
}
}

Copy and validate a Zip File via Java

After some research:
How to create a Zip File
and some google research i came up with this java function:
static void copyFile(File zipFile, File newFile) throws IOException {
ZipFile zipSrc = new ZipFile(zipFile);
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(newFile));
Enumeration srcEntries = zipSrc.entries();
while (srcEntries.hasMoreElements()) {
ZipEntry entry = (ZipEntry) srcEntries.nextElement();
ZipEntry newEntry = new ZipEntry(entry.getName());
zos.putNextEntry(newEntry);
BufferedInputStream bis = new BufferedInputStream(zipSrc
.getInputStream(entry));
while (bis.available() > 0) {
zos.write(bis.read());
}
zos.closeEntry();
bis.close();
}
zos.finish();
zos.close();
zipSrc.close();
}
This code is working...but it is not nice and clean at all...anyone got a nice idea or an example?
Edit:
I want to able to add some type of validation if the zip archive got the right structure...so copying it like an normal file without regarding its content is not working for me...or would you prefer checking it afterwards...i am not sure about this one

You just want to copy the complete zip file? Than it is not needed to open and read the zip file... Just copy it like you would copy every other file.
public final static int BUF_SIZE = 1024; //can be much bigger, see comment below
public static void copyFile(File in, File out) throws Exception {
FileInputStream fis = new FileInputStream(in);
FileOutputStream fos = new FileOutputStream(out);
try {
byte[] buf = new byte[BUF_SIZE];
int i = 0;
while ((i = fis.read(buf)) != -1) {
fos.write(buf, 0, i);
}
}
catch (Exception e) {
throw e;
}
finally {
if (fis != null) fis.close();
if (fos != null) fos.close();
}
}

Try: http://commons.apache.org/io/api-release/org/apache/commons/io/FileUtils.html#copyFile
Apache Commons FileUtils#copyFile

My solution:
import java.io.*;
import javax.swing.*;
public class MovingFile
{
public static void copyStreamToFile() throws IOException
{
FileOutputStream foutOutput = null;
String oldDir = "F:/UPLOADT.zip";
System.out.println(oldDir);
String newDir = "F:/NewFolder/UPLOADT.zip"; // name as the destination file name to be done
File f = new File(oldDir);
f.renameTo(new File(newDir));
}
public static void main(String[] args) throws IOException
{
copyStreamToFile();
}
}

I have updated your code to Java 9+, FWIW
try (ZipFile srcFile = new ZipFile(inputName)) {
try (ZipOutputStream destFile = new ZipOutputStream(
Files.newOutputStream(Paths.get(new File(outputName).toURI())))) {
Enumeration<? extends ZipEntry> entries = srcFile.entries();
while (entries.hasMoreElements()) {
ZipEntry src = entries.nextElement();
ZipEntry dest = new ZipEntry(src.getName());
destFile.putNextEntry(dest);
try (InputStream content = srcFile.getInputStream(src)) {
content.transferTo(destFile);
}
destFile.closeEntry();
}
destFile.finish();
}
}