We Keep Coding

Kotlin gzip uncompress fail

I try to simplify my java gzip uncompress code to kotlin. But after I changed, it sames broken.
Here is the java code
public static byte[] uncompress(byte[] compressedBytes) {
if (null == compressedBytes || compressedBytes.length == 0) {
return null;
}
ByteArrayOutputStream out = null;
ByteArrayInputStream in = null;
GZIPInputStream gzipInputStream = null;
try {
out = new ByteArrayOutputStream();
in = new ByteArrayInputStream(compressedBytes);
gzipInputStream = new GZIPInputStream(in);
byte[] buffer = new byte[256];
int n = 0;
while ((n = gzipInputStream.read(buffer)) >= 0) {
out.write(buffer, 0, n);
}
return out.toByteArray();
} catch (IOException ignore) {
} finally {
CloseableUtils.closeQuietly(gzipInputStream);
CloseableUtils.closeQuietly(in);
CloseableUtils.closeQuietly(out);
}
return null;
}
This is my kotlin code.
payload = GZIPInputStream(payload.inputStream())
.bufferedReader()
.use { it.readText() }
.toByteArray()
And I got this error.
com.google.protobuf.nano.InvalidProtocolBufferNanoException: While parsing a protocol message, the input ended unexpectedly in the middle of a field. This could mean either than the input has been truncated or that an embedded message misreported its own length.
It seems that the decompression process was interrupted by reader?

The readText(charset: Charset = Charsets.UTF_8) decodes the bytes into UTF-8 character set, which is why it says "This could mean either than the input has been truncated" it probably have tried to convert 8-bits into a Char and build a String out of it.
Use the readBytes() to get ByteArray which is represented same as byte[] in JVM platform.
Example:
GZIPInputStream(payload.inputStream())
.bufferedReader()
.use { it.readBytes() }
Edit:
For reading bytes, you shouldn't be using the Reader, it is meant for reading the Text in UTF-8 format as defined in the Kotlin's InputStream.bufferedReader:
public inline fun InputStream.bufferedReader(charset: Charset = Charsets.UTF_8): BufferedReader = reader(charset).buffered()
The InputStream.readBytes() will read the bytes at a buffer of 8KB itself.
public fun InputStream.readBytes(): ByteArray {
val buffer = ByteArrayOutputStream(maxOf(DEFAULT_BUFFER_SIZE, this.available()))
copyTo(buffer)
return buffer.toByteArray()
}
// This copies with 8KB buffer automatically
// DEFAULT_BUFFER_SIZE = 8 * 1024
public fun InputStream.copyTo(out: OutputStream, bufferSize: Int = DEFAULT_BUFFER_SIZE): Long {
var bytesCopied: Long = 0
val buffer = ByteArray(bufferSize)
var bytes = read(buffer)
while (bytes >= 0) {
out.write(buffer, 0, bytes)
bytesCopied += bytes
bytes = read(buffer)
}
return bytesCopied
}
So you just have to do:
GZIPInputStream(payload.inputStream()).use { it.readBytes() }

use the following function:
fun File.unzip(unzipLocationRoot: File? = null) {
val rootFolder = unzipLocationRoot
?: File(parentFile.absolutePath + File.separator + nameWithoutExtension)
if (!rootFolder.exists()) {
rootFolder.mkdirs()
}
ZipFile(this).use { zip ->
zip
.entries()
.asSequence()
.map {
val outputFile = File(rootFolder.absolutePath + File.separator + it.name)
ZipIO(it, outputFile)
}
.map {
it.output.parentFile?.run {
if (!exists()) mkdirs()
}
it
}
.filter { !it.entry.isDirectory }
.forEach { (entry, output) ->
zip.getInputStream(entry).use { input ->
output.outputStream().use { output ->
input.copyTo(output)
}
}
}
}
}
Pass the file as a parameter as follow:
val zipFile = File(your file directory, your file name)
zipFile.unzip()
Hope this would help 🙏🏼

How to zip folder and sub-folder using zip4j and outputstream

In the program that I am writing, I take the uri of the location to save the zip file from a user. Then, I try to zip the files and folders using zip4j library and outpustream in Android. I modified the code in this Stackoverflow answer and used zip4j instead. My modified code produces the zip file, however it is corrupted.
this is my code written in Kotlin:
class ZipBuilder {
private fun buildZipParameters(compressionMethod: CompressionMethod, compressionLevel: CompressionLevel,
encrypt: Boolean,
encryptionMethod: EncryptionMethod?,
aesKeyStrength: AesKeyStrength?
): ZipParameters? {
val zipParameters = ZipParameters()
zipParameters.compressionMethod = compressionMethod
zipParameters.compressionLevel = compressionLevel
return zipParameters
}
fun zipFileAtPath(sourcePath: String?, toLocation: ParcelFileDescriptor?): Boolean {
println("zipFileAtPath is called")
val BUFFER = 2048
val sourceFile = File(sourcePath!!)
val zipParameters = buildZipParameters(CompressionMethod.DEFLATE, CompressionLevel.NORMAL,
false, null, null)
try {
var origin: BufferedInputStream? = null
val desc = toLocation
val dest = FileOutputStream(desc!!.fileDescriptor)
val out = ZipOutputStream(BufferedOutputStream(dest))
if (sourceFile.isDirectory) {
zipParameters.rootFolderNameInZip = sourcePath
zipSubFolder(out, sourceFile, sourceFile.parent!!.length, zipParameters!!)
} else {
val data = ByteArray(BUFFER)
val fi = FileInputStream(sourcePath)
origin = BufferedInputStream(fi, BUFFER)
zipParameters!!.fileNameInZip = getLastPathComponent(sourcePath)
zipParameters.lastModifiedFileTime = sourceFile.lastModified()
out.putNextEntry(zipParameters)
var count: Int = 0
while (fi.read(data).also({ count = it }) != -1) {
out.write(data, 0, count)
}
}
out.close()
} catch (e: java.lang.Exception) {
e.printStackTrace()
return false
}
return true
}
#Throws(IOException::class)
private fun zipSubFolder(
out: ZipOutputStream, folder: File, basePathLength: Int, zipParameters: ZipParameters
) {
val BUFFER = 2048
val fileList = folder.listFiles()
var origin: BufferedInputStream
fileList?.forEach { file ->
if (file.isDirectory) {
zipSubFolder(out, file, basePathLength, zipParameters)
} else {
val data = ByteArray(BUFFER)
val unmodifiedFilePath = file.path
val relativePath = unmodifiedFilePath
.substring(basePathLength)
val fi = FileInputStream(unmodifiedFilePath)
origin = BufferedInputStream(fi, BUFFER)
zipParameters.fileNameInZip = relativePath
zipParameters.lastModifiedFileTime = file.lastModified()
out.putNextEntry(zipParameters)
var count: Int = 0
while (fi.read(data).also({ count = it }) != -1) {
out.write(data, 0, count)
}
origin.close()
}
}
}
fun getLastPathComponent(filePath: String): String? {
val segments = filePath.split("/").toTypedArray()
return if (segments.size == 0) "" else segments[segments.size - 1]
}
}
I would appreciate it if someone could tell me what could be the problem.

I guess you forgot to closeEntry() after writing date for that particular entry. You need to do out.closeEntry() after the while loop of writing data. Have a look at the example here.
while (fi.read(data).also({ count = it }) != -1) {
out.write(data, 0, count)
}
out.closeEntry() // --> close entry after writing data

Chat with option to send file

I would like to create a simple TCP chat with option to send files.
Sending messages to other clients works, but sending files doesn't work. My chat sends only few bytes of file.
Chat works like: Server starts up and wait for clients, Clients connect to server and they can send messages to other clients through the server. I would like to allow to the same with files.
Server is wrote in C, and Client is wrote in Java (I had guidelines like that).
Server:
for (i = 0; i < max_clients; i++) {
sd = client_socket[i];
memset(buffer, 0, 10000);
if (FD_ISSET( sd , &readfds)) {
if ((valread = read( sd , buffer, 1024)) == 0) {
getpeername(sd, (struct sockaddr*)&address, (socklen_t*)&addrlen);
printf("Host disconnected , ip %s , port %d \n" ,
inet_ntoa(address.sin_addr) , ntohs(address.sin_port));
close( sd );
client_socket[i] = 0;
}
else {
// When message "start" arrived download the file and send it back to other clients
if (strcmp(buffer, "start") == 0) {
uint8_t buff[10000];
// Read chunks of file
while (read( sd , buff, sizeof(buff)) > 0) {
// Sending chunks of file to other clients
for(j=0; j<max_clients; j++) {
int outSock = client_socket[j];
if(outSock != master_socket && outSock != sd) {
send(outSock , buff , sizeof(buff) , 0 );
}
}
}
} else {
buffer[valread] = '\0';
for(j=0; j<max_clients; j++) {
int outSock = client_socket[j];
if(outSock != master_socket && outSock != sd) {
send(outSock , buffer , strlen(buffer) , 0 );
}
}
}
}
}
}
Client:
#FXML
void sendFile(ActionEvent event) {
FileChooser fileChooser = new FileChooser();
File file = fileChooser.showOpenDialog(null);
// Send "start" message to let server know that I'm going to send a file
out.println("start");
out.flush();
try {
FileInputStream fis = new FileInputStream(file);
BufferedInputStream bis = new BufferedInputStream(fis);
//Get socket's output stream
OutputStream os = clientSocket.getOutputStream();
//Read File Contents into contents array
byte[] contents;
long fileLength = file.length();
long current = 0;
while(current!=fileLength){
int size = 10000;
if(fileLength - current >= size)
current += size;
else{
size = (int)(fileLength - current);
current = fileLength;
}
contents = new byte[size];
bis.read(contents, 0, size);
os.write(contents);
System.out.print("Sending file ... "+(current*100)/fileLength+"% complete!");
}
os.flush();
System.out.println("File sent successfully!");
} catch(Exception e) {
}
}
public ChatWindowController() {
try {
clientSocket = new Socket("127.0.0.1", 54000);
outToServer = new DataOutputStream(clientSocket.getOutputStream());
inFromServer = new BufferedReader(new InputStreamReader(clientSocket.getInputStream()));
out = new PrintWriter(clientSocket.getOutputStream(), true);
thread = new Thread() {
#Override
public void run() {
try {
while(isRunning) {
if (ta_display != null) {
String message = inFromServer.readLine();
if (!isDownloadingFile) {
System.out.println(message);
ta_display.appendText(message + '\n');
if (message.equals("start")) {
isDownloadingFile = true;
}
} else {
byte[] contents = new byte[10000];
//Initialize the FileOutputStream to the output file's full path.
FileOutputStream fos = new FileOutputStream("/example/test.png");
BufferedOutputStream bos = new BufferedOutputStream(fos);
InputStream is = clientSocket.getInputStream();
//No of bytes read in one read() call
int bytesRead = 0;
while((bytesRead=is.read(contents))!=-1)
bos.write(contents, 0, bytesRead);
bos.flush();
System.out.println("File saved successfully!");
}
}
}
} catch(Exception e) {
e.printStackTrace();
}
}
};
thread.start();
} catch(Exception e) {
e.printStackTrace();
}
}
When I click button sendFile method is called, then I choose file and I want to send this file, and with other clients download it.

I have written a similar example for chatting at https://github.com/foreverpersist/socket/blob/master/chatroom.c.
File transferring is implemented by P2P
Server tells <IP:PORT>s of sender and receiver to each other.
Sender and receiver connect to each other directly.
Sender sends file to receiver.
Sender and receiver close the connection.
As what happens in your project, when transferring files with path Sender -> Server -> Receiver, the content of large files may be incomplete if you don't deal carefully. So, I just transfer files with path Sender -> Receiver.

Ok, there is a bug on the server when receiving the file:
//client sends 10000 bytes of data in chunks of unknown size
//receive one chunk of unknown size
while (read( sd , buff, sizeof(buff)) > 0) {
for(j=0; j<max_clients; j++) {
int outSock = client_socket[j];
if(outSock != master_socket && outSock != sd) {
// Send 10000 bytes aka the received chunk and whatever else is in the buffer to all clients
send(outSock , buff , sizeof(buff) , 0 );
}
}
You need to either receive all the data first and then send it to the clients, or only send the size of the chunk many bytes, like in the File writer in the client or as follows:
int chunk = 0;
while ((chunk = read( sd , buff, sizeof(buff))) > 0) {
...
send(outSock , buff , chunk , 0);
Also Im not certain if the Client file writing code works, you should ensure that no following messages are written to file accidently.

Split large file into chunks

I have a method which accept file and size of chunks and return list of chunked files. But the main problem that my line in file could be broken, for example in main file I have next lines:
|1|aaa|bbb|ccc|
|2|ggg|ddd|eee|
After split I could have in one file:
|1|aaa|bbb
In another file:
|ccc|2|
|ggg|ddd|eee|
Here is the code:
public static List<File> splitFile(File file, int sizeOfFileInMB) throws IOException {
int counter = 1;
List<File> files = new ArrayList<>();
int sizeOfChunk = 1024 * 1024 * sizeOfFileInMB;
byte[] buffer = new byte[sizeOfChunk];
try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file))) {
String name = file.getName();
int tmp = 0;
while ((tmp = bis.read(buffer)) > 0) {
File newFile = new File(file.getParent(), name + "."
+ String.format("%03d", counter++));
try (FileOutputStream out = new FileOutputStream(newFile)) {
out.write(buffer, 0, tmp);
}
files.add(newFile);
}
}
return files;
}
Should I use RandomAccessFile class for above purposes (main file is really big - more then 5 Gb)?

If you don't mind to have chunks of different lengths (<=sizeOfChunk but closest to it) then here is the code:
public static List<File> splitFile(File file, int sizeOfFileInMB) throws IOException {
int counter = 1;
List<File> files = new ArrayList<File>();
int sizeOfChunk = 1024 * 1024 * sizeOfFileInMB;
String eof = System.lineSeparator();
try (BufferedReader br = new BufferedReader(new FileReader(file))) {
String name = file.getName();
String line = br.readLine();
while (line != null) {
File newFile = new File(file.getParent(), name + "."
+ String.format("%03d", counter++));
try (OutputStream out = new BufferedOutputStream(new FileOutputStream(newFile))) {
int fileSize = 0;
while (line != null) {
byte[] bytes = (line + eof).getBytes(Charset.defaultCharset());
if (fileSize + bytes.length > sizeOfChunk)
break;
out.write(bytes);
fileSize += bytes.length;
line = br.readLine();
}
}
files.add(newFile);
}
}
return files;
}
The only problem here is file charset which is default system charset in this example. If you want to be able to change it let me know. I'll add third parameter to "splitFile" function for it.

Just in case anyone is interested in a Kotlin version.
It creates an iterator of ByteArray chunks:
class ByteArrayReader(val input: InputStream, val chunkSize: Int, val bufferSize: Int = 1024*8): Iterator<ByteArray> {
var eof: Boolean = false
init {
if ((chunkSize % bufferSize) != 0) {
throw RuntimeException("ChunkSize(${chunkSize}) should be a multiple of bufferSize (${bufferSize})")
}
}
override fun hasNext(): Boolean = !eof
override fun next(): ByteArray {
var buffer = ByteArray(bufferSize)
var chunkWriter = ByteArrayOutputStream(chunkSize) // no need to close - implementation is empty
var bytesRead = 0
var offset = 0
while (input.read(buffer).also { bytesRead = it } > 0) {
if (chunkWriter.use { out ->
out.write(buffer, 0, bytesRead)
out.flush()
offset += bytesRead
offset == chunkSize
}) {
return chunkWriter.toByteArray()
}
}
eof = true
return chunkWriter.toByteArray()
}
}

Split a file to multiple chunks (in memory operation), here I'm splitting any file to a size of 500kb(500000 bytes) and adding to a list :
public static List<ByteArrayOutputStream> splitFile(File f) {
List<ByteArrayOutputStream> datalist = new ArrayList<>();
try {
int sizeOfFiles = 500000;
byte[] buffer = new byte[sizeOfFiles];
try (FileInputStream fis = new FileInputStream(f); BufferedInputStream bis = new BufferedInputStream(fis)) {
int bytesAmount = 0;
while ((bytesAmount = bis.read(buffer)) > 0) {
try (OutputStream out = new ByteArrayOutputStream()) {
out.write(buffer, 0, bytesAmount);
out.flush();
datalist.add((ByteArrayOutputStream) out);
}
}
}
} catch (Exception e) {
//get the error
}
return datalist;
}

Split files in chunks depending upon your chunk size
val f = FileInputStream(file)
val data = ByteArray(f.available()) // Size of original file
var subData: ByteArray
f.read(data)
var start = 0
var end = CHUNK_SIZE
val max = data.size
if (max > 0) {
while (end < max) {
subData = data.copyOfRange(start, end)
start = end
end += CHUNK_SIZE
if (end >= max) {
end = max
}
//Function to upload your chunk
uploadFileInChunk(subData, isLast = false)
}
// For the Last Chunk
end--
subData = data.copyOfRange(start, end)
uploadFileInChunk(subData, isLast = true)
}
If you are taking the file from the user through intent you may get file URI as content, so in that case.
Uri uri = data.getData();
InputStream inputStream = getContext().getContentResolver().openInputStream(uri);
fileInBytes = IOUtils.toByteArray(inputStream);
Add the dependency in you build gradle to use IOUtils
compile 'commons-io:commons-io:2.11.0'
Now do a little modification in the above code to send your file to server.
var subData: ByteArray
var start = 0
var end = CHUNK_SIZE
val max = fileInBytes.size
if (max > 0) {
while (end < max) {
subData = fileInBytes.copyOfRange(start, end)
start = end
end += CHUNK_SIZE
if (end >= max) {
end = max
}
uploadFileInChunk(subData, isLast = false)
}
// For the Last Chunk
end--
subData = fileInBytes.copyOfRange(start, end)
uploadFileInChunk(subData, isLast = true)
}

java zip to binary format and then decompress

I have a task that
read a zip file from local into binary message
transfer binary message through EMS as String (done by java API)
receive transferred binary message as String (done by java API)
decompress the binary message and then print it out
The problem I am facing is DataFormatException while decompress the message.
I have no idea which part went wrong.
I use this to read file into binary message:
static String readFile_Stream(String fileName) throws IOException {
File file = new File(fileName);
byte[] fileData = new byte[(int) file.length()];
FileInputStream in = new FileInputStream(file);
in.read(fileData);
String content = "";
System.out.print("Sent message: ");
for(byte b : fileData)
{
System.out.print(getBits(b));
content += getBits(b);
}
in.close();
return content;
}
static String getBits(byte b)
{
String result = "";
for(int i = 0; i < 8; i++)
result = ((b & (1 << i)) == 0 ? "0" : "1") + result;
return result;
}
I use this to decompress message:
private static byte[] toByteArray(String input)
{
byte[] byteArray = new byte[input.length()/8];
for (int i=0;i<input.length()/8;i++)
{
String read_data = input.substring(i*8, i*8+8);
short a = Short.parseShort(read_data, 2);
byteArray[i] = (byte) a;
}
return byteArray;
}
public static byte[] unzipByteArray(byte[] file) throws IOException {
byte[] byReturn = null;
Inflater oInflate = new Inflater(false);
oInflate.setInput(file);
ByteArrayOutputStream oZipStream = new ByteArrayOutputStream();
try {
while (! oInflate.finished() ){
byte[] byRead = new byte[4 * 1024];
int iBytesRead = oInflate.inflate(byRead);
if (iBytesRead == byRead.length){
oZipStream.write(byRead);
}
else {
oZipStream.write(byRead, 0, iBytesRead);
}
}
byReturn = oZipStream.toByteArray();
}
catch (DataFormatException ex){
throw new IOException("Attempting to unzip file that is not zipped.");
}
finally {
oZipStream.close();
}
return byReturn;
}
The message I got is
java.io.IOException: Attempting to unzip file that is not zipped.
at com.sourcefreak.example.test.TibcoEMSQueueReceiver.unzipByteArray(TibcoEMSQueueReceiver.java:144)
at com.sourcefreak.example.test.TibcoEMSQueueReceiver.main(TibcoEMSQueueReceiver.java:54)
After check, the binary message does not corrupted after transmission.
Please help to figure out the problem.

Have you tried using InflaterInputStream? Based on my experience, using Inflater directly is rather tricky. You can use this to get started:
public static byte[] unzipByteArray(byte[] file) throws IOException {
InflaterInputStream iis = new InflaterInputStream(new ByteArrayInputStream(file));
ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte[] buffer = new byte[512];
int length = 0;
while ((length = iis.read(buffer, 0, buffer.length) != 0) {
baos.write(buffer, 0, length);
}
iis.close();
baos.close();
return baos.toByteArray();
}

I finally figure out the problem.
The problem is the original file is a .zip file, so I should use zipInputStream to unzip the file before further processing.
public static byte[] unzipByteArray(byte[] file) throws IOException {
// create a buffer to improve copy performance later.
byte[] buffer = new byte[2048];
byte[] content ;
// open the zip file stream
InputStream theFile = new ByteArrayInputStream(file);
ZipInputStream stream = new ZipInputStream(theFile);
ByteArrayOutputStream output = new ByteArrayOutputStream();
try
{
ZipEntry entry;
while((entry = stream.getNextEntry())!=null)
{
//String s = String.format("Entry: %s len %d added %TD", entry.getName(), entry.getSize(), new Date(entry.getTime()));
//System.out.println(s);
// Once we get the entry from the stream, the stream is
// positioned read to read the raw data, and we keep
// reading until read returns 0 or less.
//String outpath = outdir + "/" + entry.getName();
try
{
//output = new FileOutputStream(outpath);
int len = 0;
while ((len = stream.read(buffer)) > 0)
{
output.write(buffer, 0, len);
}
}
finally
{
// we must always close the output file
if(output!=null) output.close();
}
}
}
finally
{
// we must always close the zip file.
stream.close();
}
content = output.toByteArray();
return content;
}
This code work for zip file containing single file inside.