We Keep Coding

Writing StringBuilder to AWS S3 File

I have a StringBuilder (~1GB size) which converts to an almost 1GB String. Then it is sent to AWSS3Client to be put as a file on s3. Now to write the String to s3, I need to further convert the string to a ByteArrayInputStream, which takes another 1GB.
InputStream dataInputStream = new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8));
amazonS3Client.putObject(bucket, key, dataInputStream, new ObjectMetadata());
Now I end up with a JVM heap of 3GB. Is there a way to directly send the 1GB StringBuilder directly to S3 with little over 1GB heap?
The only way I could think of is to convert StringBuilder to a File instance (using FileWriter) --> S3 API which takes File as input. But this approach requires a 2-IOPS to the SSD -- penalty that I want to avoid.
Any better way to handle the problem is welcome.
[edit 1]
(as per comments) The builder size changed from 5GB to 1GB. I wanted to convey that the builderis a huge one (and hence used 5GB previously, which was my mistake)

The string-oriented classes are frustratingly locked down with final in many cases, which means that oftentimes buffers that already exist in memory have to be duplicated. This is probably to do with security but is nevertheless wasteful and frustrating. You should be able to do something with the following if you have a StringBuilder. Obviously you'll need a much bigger buffer than in the illustrative main method.
import java.io.ByteArrayInputStream;
import java.io.FilterInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.FileOutputStream;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.Charset;
import java.util.Objects;
/**
*
* #author CEHJ
* #version 1.0
*/
public class CharSequenceInputStream extends InputStream {
private CharSequence charSequence;
private int position;
private CharsetEncoder enc;
private Charset outputEncoding;
private CharBuffer cb;
private ByteBuffer bb;
public CharSequenceInputStream(CharSequence charSequence) {
this(charSequence, Charset.forName(System.getProperty("native.encoding", Charset.defaultCharset().name())));
}
public CharSequenceInputStream(CharSequence charSequence, Charset outputEncoding) {
this.charSequence = charSequence;
this.outputEncoding = outputEncoding;
position = 0;
enc = outputEncoding.newEncoder();
cb = CharBuffer.allocate(1);
bb = ByteBuffer.allocate(8);
}
// For testing only
public static void main(String[] args) throws IOException {
if (args.length < 2) {
System.err.printf("Usage: java CharSequenceInputStream <String to be stored> <Output file> [Char encoding for output]");
System.exit(1);
}
String content = args[0];
String outputPath = args[1];
StringBuilder sb = new StringBuilder(content);
CharSequenceInputStream in = null;
if (args.length > 2) {
in = new CharSequenceInputStream(sb, Charset.forName(args[2]));
} else {
in = new CharSequenceInputStream(sb);
}
try (OutputStream out = new FileOutputStream(outputPath)) {
int bytesRead = -1;
final int BUF_SIZE = 8;
byte[] buf = new byte[BUF_SIZE];
while ((bytesRead = in.read(buf, 0, BUF_SIZE)) > -1) {
out.write(buf, 0, bytesRead);
}
}
}
#Override
public int read() throws IOException {
int result = -1;
if (position < charSequence.length()) {
if (bb.remaining() == bb.capacity() || bb.position() == bb.limit()) {
// At start - nothing yet has been decoded
// OR - we've read all bytes in the buffer and need to refill it
cb.clear();
bb.clear();
char currentChar = charSequence.charAt(position++);
cb.append(currentChar);
cb.flip();
enc.reset();
enc.encode(cb, bb, false);
enc.encode(cb, bb, true);
enc.flush(bb);
bb.flip();
}
}
if (bb.position() < bb.limit()) {
result = bb.get() & 0xFF;
}
return result;
}
#Override
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}
#Override
public int read(byte b[], int off, int len) throws IOException {
Objects.checkFromIndexSize(off, len, b.length);
if (len == 0) {
return 0;
}
int c = read();
if (c == -1) {
return -1;
}
b[off] = (byte) c;
int i = 1;
try {
for (; i < len; i++) {
c = read();
if (c == -1) {
break;
}
b[off + i] = (byte) c;
}
} catch (IOException ee) {
}
return i;
}
#Override
public void close() {
// NO OP
}
}

How to correctly read byte java file

I want to read fast line by line big csv files (approx ~ 1gb) in UTF-8. I have created a class for it, but it doesn't work properly. UTF-8 decodes Cyrillic symbol from 2 bytes. I use byte buffer to read it, for example, it has 10 bytes length. So if symbol composed from 10 and 11 bytes in the file it wouldn't be decoded normally :(
public class MyReader extends InputStream {
private FileChannel channel;
private ByteBuffer buffer = ByteBuffer.allocate(10);
private int buffSize = 0;
private int position = 0;
private boolean EOF = false;
private CharBuffer charBuffer;
private MyReader() {}
static MyReader getFromFile(final String path) throws IOException {
MyReader myReader = new MyReader();
myReader.channel = FileChannel.open(Path.of(path),
StandardOpenOption.READ);
myReader.initNewBuffer();
return myReader;
}
private void initNewBuffer() {
try {
buffSize = channel.read(buffer);
buffer.position(0);
charBuffer = Charset.forName("UTF-8").decode(buffer);
buffer.position(0);
} catch (IOException e) {
throw new RuntimeException("Error reading file: {}", e);
}
}
#Override
public int read() throws IOException {
if (EOF) {
return -1;
}
if (position < charBuffer.length()) {
return charBuffer.array()[position++];
} else {
initNewBuffer();
if (buffSize < 1) {
EOF = true;
} else {
position = 0;
}
return read();
}
}
public char[] readLine() throws IOException {
int readResult = 0;
int startPos = position;
while (readResult != -1) {
readResult = read();
}
return Arrays.copyOfRange(charBuffer.array(), startPos, position);
}
}

Bad solution, but it works)
private void initNewBuffer() {
try {
buffSize = channel.read(buffer);
buffer.position(0);
charBuffer = StandardCharsets.UTF_8.decode(buffer);
if (buffSize > 0) {
byte edgeByte = buffer.array()[buffSize - 1];
if (edgeByte == (byte) 0xd0 ||
edgeByte == (byte) 0xd1 ||
edgeByte == (byte) 0xc2 ||
edgeByte == (byte) 0xd2 ||
edgeByte == (byte) 0xd3
) {
channel.position(channel.position() - 1);
charBuffer.limit(charBuffer.limit()-1);
}
}
buffer.position(0);
} catch (IOException e) {
throw new RuntimeException("Error reading file: {}", e);
}
}

First: the gain is questionable.
The Files class has many nice and quite production fast methods.
Bytes with high bit 1 (< 0) are part of a UTF-8 multibyte sequence.
With high bits 10 they are continuation bytes.
Sequences might be upto 6 bytes nowadays (I believe).
So the next buffer starts with some continuation bytes, they belong to the previous buffer.
The programming logic I gladly leave to you.

String serialization unit test failed

After running a Junit test for String serialization, it is failed and gave me the following results:
Expected: "netmodel"
Actual: "l"
The serialize method as follows
public static void serializeString(String objectToSerialize, OutputStream outputStream) {
byte[] bytesArr = objectToSerialize.getBytes();
serializeInt(bytesArr.length, outputStream);
try {
outputStream.write(bytesArr);
} catch (IOException e) {
e.printStackTrace();
}
}
And my deserialize method as follows
public static String deserializeString(InputStream inputStream) {
String deserializeObject = "";
char asciiToChar;
int stringByteArrayLength = deserializeInt(inputStream);
byte[] databytesArr = new byte[stringByteArrayLength];
try {
inputStream.read(databytesArr, 0, stringByteArrayLength);
}
catch (IOException e) {
e.printStackTrace();
}
for (int i = 0; i < databytesArr.length; i++) {
asciiToChar = (char) databytesArr[i];
deserializeObject = "" + Character.toString(asciiToChar);
}
return deserializeObject;
}
Finally, I wrote a unit test as follows
public class StringSerializerTest {
private InputStream iStream;
private ByteArrayOutputStream oStream;
#Before
public void init() {
oStream = new ByteArrayOutputStream();
}
String serialzeAndDeserializeObject(String stringValue) {
OutputStreamUtil.serializeString(stringValue, oStream);
iStream = new ByteArrayInputStream(oStream.toByteArray());
return InputStreamUtil.deserializeString(iStream);
}
#Test
public void equals_equal() {
String stringValue = "netmodel";
String deserializedStringValue = serialzeAndDeserializeObject(stringValue);
assertThat(deserializedStringValue).isEqualTo(stringValue);
}
}
what was wrong? and how to fix it?

You are reassigning the entire value of deserializeObject during each iteration of
for (int i = 0; i < databytesArr.length; i++) {
asciiToChar = (char) databytesArr[i];
deserializeObject = "" + Character.toString(asciiToChar);
}
This results in only the last character (l in this case) being stored in deserializeObject. This loop should append the next character to the deserializeObject as in the following:
for (int i = 0; i < databytesArr.length; i++) {
asciiToChar = (char) databytesArr[i];
deserializeObject += Character.toString(asciiToChar);
}
The corrected deserialization logic would be:
public static String deserializeString(InputStream inputStream) {
String deserializeObject = "";
char asciiToChar;
int stringByteArrayLength = deserializeInt(inputStream);
byte[] databytesArr = new byte[stringByteArrayLength];
try {
inputStream.read(databytesArr, 0, stringByteArrayLength);
}
catch (IOException e) {
e.printStackTrace();
}
for (int i = 0; i < databytesArr.length; i++) {
asciiToChar = (char) databytesArr[i];
deserializeObject += Character.toString(asciiToChar);
}
return deserializeObject;
}

The error was already reported by Justin Albano.
However take also care of strings with non-ASCII: like special characters.
Something like the following. Also one should close at the end to ensure flushing in case of a buffered stream. And theoretically a read could yield only a non-blocking part of the array. DataOutputStream has nice methods, though you seem to roll your own serialisation.
public static void serializeString(String objectToSerialize, OutputStream outputStream)
throws IOException {
byte[] bytesArr = objectToSerialize.getBytes(StandardCharsets.UTF_8);
serializeInt(bytesArr.length, outputStream);
outputStream.write(bytesArr);
}
public static String deserializeString(InputStream inputStream)
throws IOException {
int stringByteArrayLength = deserializeInt(inputStream);
byte[] databytesArr = new byte[stringByteArrayLength];
readFully(inputStream, databytesArr);
return new String(databytesArr, StandardCharsets.UTF_8);
}
private static void readFully(InputStream inputStream, byte[] bytes) throws IOException {
int i = 0;
while (i < bytes.length) {
int nread = inputStream.read(bytes, i, bytes.length - i);
if (nread <= 0) {
throw new IOException("Premature EOF");
}
i += nread;
}
}
Mind that StandardCharsets is not in Android SDK, only standard Java.

why is my program not reading/writing the bits?

Im currently creating a Huffman compression program.
But Im having some trouble with writing/reading the bits.
I want to be able to write specific bits to a file.
e.g first "0100" then "0101" should be written as a byte to a new file using fileOutputStream as "01000101" :69
Class BitFileWriter - writes bits to file by saving each byte in buffer and then writing when buffer is full (contains 8 bits).
In the main function of this class I have some tests to se if all bytes will be written to file.
but opening the text file it doesn't read "AB".
/**
* writes bits to file outPutStream.
*/
public class BitFileWriter {
private BufferedOutputStream out;
private int buffer; // 8-bit buffer of bits to write out
private int n; // number of bits remaining in buffer
private String filename;
public BitFileWriter(String filename){
this.filename = filename;
}
private void addBitToBuffer(boolean bit) throws IOException {
// add bit to buffer
this.buffer <<= 1;
if (bit) this.buffer |= 1;
n++;
//if buffer is full write a whole byte.
if(n == 8){
writeByte(this.buffer);
this.n = 0;
this.buffer = 0;
}
}
private void writeByte(int b) throws IOException {
this.out = new BufferedOutputStream(new
FileOutputStream(filename));
out.write(b);
}
public void flush() throws IOException {
this.out.flush();
}
public static void main(String[] args) throws IOException {
BitFileWriter bitFileWriter = new
BitFileWriter("./src/result.txt");
// byte: 01000001, A
bitFileWriter.addBitToBuffer(true);
bitFileWriter.addBitToBuffer(true);
bitFileWriter.addBitToBuffer(false);
bitFileWriter.addBitToBuffer(false);
bitFileWriter.addBitToBuffer(false);
bitFileWriter.addBitToBuffer(false);
bitFileWriter.addBitToBuffer(true);
bitFileWriter.addBitToBuffer(false);
//byte 01000011, B
bitFileWriter.addBitToBuffer(false);
bitFileWriter.addBitToBuffer(true);
bitFileWriter.addBitToBuffer(true);
bitFileWriter.addBitToBuffer(false);
bitFileWriter.addBitToBuffer(false);
bitFileWriter.addBitToBuffer(false);
bitFileWriter.addBitToBuffer(false);
bitFileWriter.addBitToBuffer(false);
bitFileWriter.flush();
}
}
Class BitFileReader - reads bits from file.
but reading all 16 bits that I wanted to write to result.txt doesn´t give me the bits I (think) have written.
/**
* Reads one bit at a time from a file.
*
*
*/
public class BitFileReader {
private BufferedInputStream in;
private int currentByte; // -1 if no more data
private int bitPos; // position in currentByte
/**
* Creates a BitFileReader by opening a connection to an actual file,
* the file named by the File object file in the file system.
*/
public BitFileReader(File file) throws IOException {
in = new BufferedInputStream(new FileInputStream(file));
currentByte = in.read();
bitPos = 7;
}
/** Returns true if this reader has another bit in its input. */
public boolean hasNextBit() {
return currentByte != -1 && in != null;
}
/** Reads a single bit. */
public int nextBit() throws IOException {
int res = (currentByte>>bitPos) & 1;
--bitPos;
if (bitPos < 0) {
currentByte = in.read(); // -1 if end of file has been reached (read returns -1 if end of file).
bitPos = 7;
}
return res ;
}
/** Closes this reader. */
public void close() throws IOException {
if (in != null) {
in.close();
}
}
//Test
public static void main(String[] args) throws IOException {
File temp;
BitFileReader reader;
reader = new BitFileReader(new File("./src/result.txt"));
System.out.print("first byte: ");
for(int i = 0; i <8; i++){
System.out.print(reader.nextBit());
}
System.out.print(". second byte: ");
for(int i = 0; i <8; i++){
System.out.print(reader.nextBit());
}
reader.close();
}
}
Output is: first byte: 01100000. second byte: 11111111

The first thing I would do, is to move the statement:
this.out = new BufferedOutputStream(new
FileOutputStream(filename));
from writeByte to the constructor

Removing ASCII characters in a string with encoding

I have a byte array which is filled by a serial port event and code is shown below:
private InputStream input = null;
......
......
public void SerialEvent(SerialEvent se){
if(se.getEventType == SerialPortEvent.DATA_AVAILABLE){
int length = input.available();
if(length > 0){
byte[] array = new byte[length];
int numBytes = input.read(array);
String text = new String(array);
}
}
}
The variable text contains the below characters,
"\033[K", "\033[m", "\033[H2J", "\033[6;1H" ,"\033[?12l", "\033[?25h", "\033[5i", "\033[4i", "\033i" and similar types..
As of now, I use String.replace to remove all these characters from the string.
I have tried new String(array , 'CharSet'); //Tried with all CharSet options but I couldn't able to remove those.
Is there any way where I can remove those characters without using replace method?

I gave a unsatisfying answer, thanks to #OlegEstekhin for pointing that out.
As noone else answered yet, and a solution is not a two-liner, here it goes.
Make a wrapping InputStream that throws away escape sequences. I have used a PushbackInputStream, where a partial sequence skipped, may still be pushed back for reading first. Here a FilterInputStream would suffice.
public class EscapeRemovingInputStream extends PushbackInputStream {
public static void main(String[] args) {
String s = "\u001B[kHello \u001B[H12JWorld!";
byte[] buf = s.getBytes(StandardCharsets.ISO_8859_1);
ByteArrayInputStream bais = new ByteArrayInputStream(buf);
EscapeRemovingInputStream bin = new EscapeRemovingInputStream(bais);
try (InputStreamReader in = new InputStreamReader(bin,
StandardCharsets.ISO_8859_1)) {
int c;
while ((c = in.read()) != -1) {
System.out.print((char) c);
}
System.out.println();
} catch (IOException ex) {
Logger.getLogger(EscapeRemovingInputStream.class.getName()).log(
Level.SEVERE, null, ex);
}
}
private static final Pattern ESCAPE_PATTERN = Pattern.compile(
"\u001B\\[(k|m|H\\d+J|\\d+:\\d+H|\\?\\d+\\w|\\d*i)");
private static final int MAX_ESCAPE_LENGTH = 20;
private final byte[] escapeSequence = new byte[MAX_ESCAPE_LENGTH];
private int escapeLength = 0;
private boolean eof = false;
public EscapeRemovingInputStream(InputStream in) {
this(in, MAX_ESCAPE_LENGTH);
}
#Override
public int read(byte[] b, int off, int len) throws IOException {
for (int i = 0; i < len; ++i) {
int c = read();
if (c == -1) {
return i == 0 ? -1 : i;
}
b[off + i] = (byte) c;
}
return len;
}
#Override
public int read() throws IOException {
int c = eof ? -1 : super.read();
if (c == -1) { // Throw away a trailing half escape sequence.
eof = true;
return c;
}
if (escapeLength == 0 && c != 0x1B) {
return c;
} else {
escapeSequence[escapeLength] = (byte) c;
++escapeLength;
String esc = new String(escapeSequence, 0, escapeLength,
StandardCharsets.ISO_8859_1);
if (ESCAPE_PATTERN.matcher(esc).matches()) {
escapeLength = 0;
} else if (escapeLength == MAX_ESCAPE_LENGTH) {
escapeLength = 0;
unread(escapeSequence);
return super.read(); // No longer registering the escape
}
return read();
}
}
}
User calls EscapeRemovingInputStream.read
this read may call some read's itself to fill an byte buffer escapeSequence
(a push-back may be done calling unread)
the original read returns.
The recognition of an escape sequence seems grammatical: command letter, numerical argument(s). Hence I use a regular expression.