We Keep Coding

Java client to C# server TCP

The problem is when i send up to 40 KB everything is okay when i send more sometime half of the data received some time nothing ,is there a limit of the networkstream.Read ,even though i cunked the data ,i can't determine if the problem form the java or the c# from the network stream or the Output stream
C# SERVER
private void ReadData(){
if (networkStream.DataAvailable)
{
int size = GetBufferSize();
Thread.Sleep(340);
byte[] myReadBuffer = new byte[size];
int numberOfBytesRead = 0;
while (true)
{
numberOfBytesRead = networkStream.Read(myReadBuffer, 0, myReadBuffer.Length);
if (numberOfBytesRead >= myReadBuffer.Length)
{
break;
}
}
string str = Encoding.UTF8.GetString(myReadBuffer, 0, size);
dynamic Message = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(str);
// Android Message , JSON String
if (OnAndroidMessage != null)
{
OnAndroidMessage(Message);
}
}
}
private int GetBufferSize()
{
byte[] myReadBuffer = new byte[4];
int numberOfBytesRead = 0;
do
{
numberOfBytesRead = networkStream.Read(myReadBuffer, 0, myReadBuffer.Length);
} while (networkStream.DataAvailable && numberOfBytesRead < myReadBuffer.Length);
if (numberOfBytesRead > 0)
{
// reverse the byte array.
if (BitConverter.IsLittleEndian)
{
Array.Reverse(myReadBuffer);
}
return BitConverter.ToInt32(myReadBuffer, 0);
}
else
{
return 0;
}
}
Java Client // i tested this also without cutting the data to smaller paces ,half of the data received not all of them
mBufferOut = socket.getOutputStream();
private void sendMessage(final String message) {
if (mBufferOut != null && message != null) {
try {
byte[] data = message.getBytes("UTF-8");
Log.d("_TAG", "Sending: " + message);
Log.d("_TAG", "Message length: " + Integer.toString(data.length));
mBufferOut.write(toByteArray(data.length));
mBufferOut.flush();
List<byte[]> divideArray = divideArray(data, 10000);
for (byte[] dataChunk : divideArray) {
Log.e("_TAG","CHUNK SIZE > " + Integer.toString(dataChunk.length));
mBufferOut.write(dataChunk, 0, dataChunk.length);
mBufferOut.flush();
}
} catch (Exception e1) {
e1.printStackTrace();
}
}
}
private List<byte[]> divideArray(byte[] source, int chunksize) {
List<byte[]> result = new ArrayList<byte[]>();
int start = 0;
while (start < source.length) {
int end = Math.min(source.length, start + chunksize);
result.add(Arrays.copyOfRange(source, start, end));
start += chunksize;
}
return result;
}
Any ideas ?

Solution from this post NetworkStream is reading data that should not be there
static void ReadExact(Stream stream, byte[] buffer, int offset, int count)
{
int read;
while(count > 0 && (read = stream.Read(buffer, offset, count)) > 0) {
offset += read;
count -= read;
}
if(count != 0) throw new EndOfStreamException();
}
the problem is the Read it takes size and want to get that size you need to give it chunks and check each chunk
And also read does not restart from where it stopped until it reads the amount is set to read meaning if i set to read 10 then if it not find the 10 then it will read what it find as example it reads 6 ,it will return 6 and when to loop another time ti read the rest it dose not start from 6 it start from 0 and read until 4 so you overwrite your data ,and if it read 10 from the first try then it set the read to finish so it dose not start from 0 ,it needs to read the amount the has been set to it to re set the read to new buffer location.

C++ Socket Send to Java Server gets hanged after some hours

My C++ program which is the client connects with the Java server . From the time of connection establishment the C++ client sends a block of data of size ~1MB to 3MB in a fixed frequency( say 10 sec).
My Java server opens a socket
Socket client = new ServerSocket(14001, 10).accept();//blocking
ReceiveThread st = new ReceiveThread(client);
and it receives the data from client as below.
private String getDataFromSocket(BufferedReader reader) throws IOException
{
int byteLimit = 1024*1024*2; //2 MB
String output = "";
char[] charArray = null;
int availableSize = is.available();
if(availableSize < 1) // If available size is 0 just return empty
{
return output;
}
while(availableSize > byteLimit) // Reads 2MB max if available size is more than 2 MB
{
charArray = new char[byteLimit];
reader.read(charArray,0,charArray.length);
output += new String(charArray);
availableSize = is.available();
}
charArray = new char[availableSize];
reader.read(charArray,0,charArray.length);
output = output +new String(charArray);
return output;
}
The above GetDataFromSocket keeps on checking for available data till the socket is closed gracefully.
the C++ connects with the Java server
void CreateSocket()
{
int err, nRet = 0;
sockfd = 0;
WORD wVersionRequested;
WSADATA wsaData;
//WSACleanup();// Is this needed?
wVersionRequested = MAKEWORD(1, 1);
while (1)
{
err = WSAStartup(wVersionRequested, &wsaData);
if (err != 0)
{
Sleep(50);
continue;
}
else
{
break;
}
}
while (1)
{
sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd == -1 || sockfd == INVALID_SOCKET)
{
Sleep(50);
continue;
}
else
{
nRet = 1;
break;
}
}
}
void ConnectWithServer()
{
int nRet = 0;
char myname[256] = { 0 };
int wsaErr = 0, portNum = 0, retryCount=0;
struct hostent *h = NULL;
struct sockaddr_in server_addr;
gethostname(myname, 256);
portNum = 1401;
while (1)
{
if ((h = gethostbyname(myname)) != NULL)
{
memset(&server_addr, 0, sizeof(struct sockaddr_in));
memcpy((char *)&server_addr.sin_addr, h->h_addr, h->h_length);
server_addr.sin_family = AF_INET;
server_addr.sin_port = htons(portNum);
server_addr.sin_addr = *((struct in_addr*) h->h_addr);
}
if (0 == connect(sockfd, (struct sockaddr *)&server_addr, sizeof(server_addr)))
{
nRet = 1;
break;
}
else
{
}
Sleep(50);
}
}
The connection establishment to the server is done by the above two functions and it returns success. After these steps i am sending the data buffer to the Java server once in every 10 seconds.
while(index<retryCount)
{
string toSend = wstring_to_utf8(sRequestData);
nRet = send(sockfd, toSend.c_str(), toSend.length(), 0);
if (nRet == SOCKET_ERROR)
{
wsaErr = WSAGetLastError();
Sleep(DEFAULT);
index++;
}
else if(nRet == toSend.length())
{
break;
}
else
{
index = 0;
}
}
The problem here is, after some hours of send and receive from C++ to Java server , the send gets hanged for infinite time. The execution never comes out from the send() function. But after the hang if i abruptly close the Java server , then the send returns socket error and again works well for some hours and the hang still occurs.
As i mentioned i keep on sending data to server of size varied from 1 MB to 3 MB ten seconds once. What could be the issue here? How can i sort this out?

Java SocketChannel Read Entire String

In my current project, I am trying to transmit a string from one computer to another, and after finding and learning from numerous examples I have managed to get a basic form of communication working.
The issue I am having is if one computer tries sending a message that is too long, it seems to get broken up into multiple parts (roughly 3700 characters), and my parsing method fails.
I am using a Selector to iterate through all of the channels. Here is the relevant code:
if(key.isReadable()) {
// Get the channel and read in the data
SocketChannel keyChannel = (SocketChannel)key.channel();
ByteBuffer buffer = buffers.get(keyChannel);
int length = 0;
try {
length = keyChannel.read(buffer);
} catch ( IOException ioe) {
closeChannel(keyChannel);
}
if(length > 0) {
buffer.flip();
// Gather the entire message before processing
while( buffer.remaining() > 0) {
byte[] data = new byte[buffer.remaining()];
buffer.get(data);
fireReceiveEvent(keyChannel, data);//Send the data for processing
}
buffer.compact();
} else if (length < 0) {
closeChannel(keyChannel);
}
}
How can I guarantee that the entire message (regardless of length) is read at once before passing it along?

After talking to numerous people that know more about this than I do. The issue turns out to be that with TCP it is impossible to know when an entire "message" has arrived because there is no such thing as a message since TCP works on a two-way byte stream. The solution is to create your own protocol and implements your own definition of "message".
For my project, every message either starts with a [ or { and ends with a ] or } depending on the starting character. I search through the received data, and if there is a complete message, I grab it and pass it along to the handler. Otherwise skip the channel, and wait for more to arrive.
Here is the final version of my code that handles the message receiving.
if(key.isReadable()) {
// Get the channel and read in the data
SocketChannel keyChannel = (SocketChannel)key.channel();
ByteBuffer buffer = buffers.get(keyChannel);
int length = 0;
try {
length = keyChannel.read(buffer);
} catch ( IOException ioe) {
key.cancel();
closeChannel(keyChannel);
}
if (length > 0) {
buffer.flip();
// Gather the entire message before processing
if (buffer.remaining() > 0) {
byte[] data = new byte[buffer.remaining()];
buffer.get(data);
buffer.rewind();
int index = 0;
int i = 0;
// Check for the beginning of a packet
//[ = 91
//] = 93
//{ = 123
//} = 125
if (data[0] == 91 || data[0] == 123) {
// The string we are looking for
byte targetByte = (byte) (data[0] + 2);
for (byte b : data) {
i += 1;
if (b == targetByte) {
index = i;
break;
}
}
if (index > 0) {
data = new byte[index];
buffer.get(data, 0, index);
fireReceiveEvent(keyChannel, data);
}
} else {
for (byte b : data) {
i += 1;
if (b == 91 || b == 123) {
index = i;
break;
}
}
if (index > 0) {
data = new byte[index];
buffer.get(data, 0, index); // Drain the data that we don't want
}
}
}
buffer.compact();
} else if (length < 0) {
key.cancel();
closeChannel(keyChannel);
}
}

Proper handling of chuncked Http Response within Socket

I have some major problem detecting the true end of a http response within Socket (I have to use sockets as it was requested). We're communicating with a webservice that sends chuncked responses. I have no problems reading a response if it returns in one piece. However when it's being split all hell breaks loose :).
For example:
UserA -> RequestA -> Response[1] -> Processed
UserA -> RequestA -> Response[1] -> Processed
UserB -> RequestB -> a)Response[0,1] -> Processed[a.0]
UserB -> RequestB -> b)Response[0,1] -> Processed[a.1] <- From the previous line. And thus the true response to request B have to be processed again.
What is the preferred way to handle this kind of situation? Btw, the WS also returns the Content-Length header attribute, but honestly I have a head-ache from handling that. For that it seems like I have to read the headers fields to a ByteArrayOutputStream and check if it contains the Content-Length information. Then retrieve the actual length and wait until the is.available() reaches this value. Since the available method returns an estimation I do not trust it. So what would be the proper way?

The preferred way is to use existing code that already handles it: for example, HttpURLConnection or the Apache HTTP Client. There's no good reason for you to be reinventing the wheel.

The correct answer should be:
private static byte[] convert(final InputStream is) throws IOException {
final byte[] END_SIG = new byte[]{"\r".getBytes()[0], "\n".getBytes()[0]};
final List<Byte> streamBytes = new ArrayList<Byte>();
int readByte;
byte[] bytes;
// Read HTTP header:
while ((readByte = is.read()) != -1) {
streamBytes.add((byte) readByte);
if (streamBytes.size() > 4) {
int sbsize = streamBytes.size();
int rp = sbsize - 4;
int np = sbsize - 2;
int rn = sbsize - 3;
int nn = sbsize - 1;
if (END_SIG[0] == streamBytes.get(rp) && END_SIG[0] == streamBytes.get(np) && END_SIG[1] == streamBytes.get(rn) && END_SIG[1] == streamBytes.get(nn)) {
break;
}
}
}
// Convert to byte[]
bytes = new byte[streamBytes.size()];
for (int i = 0, iMAX = bytes.length; i < iMAX; ++i) {
bytes[i] = streamBytes.get(i);
}
// drop header
streamBytes.clear();
// Convert byte[] to String & retrieve the content-length:
final String HEADER = new String(bytes);
int startIndex = HEADER.indexOf("Content-Length:") + "Content-Length:".length() + 1;
int length = 0;
int I = startIndex;
while (Character.isDigit(HEADER.charAt(I++))) {
++length;
}
final String CL = HEADER.substring(startIndex, startIndex + length);
// Determine the number of bytes to read from now on:
int ContentLength = Integer.parseInt(CL);
while (streamBytes.size() < ContentLength) {
byte[] buffer = new byte[256];
int rc = is.read(buffer);
for (int irc = 0; irc < rc; ++irc) {
streamBytes.add(buffer[irc]);
}
}
// Convert to byte[]
bytes = new byte[streamBytes.size()];
for (int i = 0, iMAX = bytes.length; i < iMAX; ++i) {
bytes[i] = streamBytes.get(i);
}
return bytes;
}
And in one place this is the answer to a question.

Replace nio sockets to exec'ed software for binary protocol

I have to make an abstaction in my software - replace direct unblockable NIO sockets ( client/server ) to software abstraction.
For example, instead of connecting via tcp client would exec openssl s_client -connect xxx.xxx.xxx.xxx . I have written a little demo, and it even works. Sometimes :(
The first trouble is that Process's streams can't be used with Selector, so I can't replace socketchannel with any other type of channel, so I have to read/write without any chance to avoid blocking.
The second one is that a protocol is a duplex binary file-transfer protocol ( binkp ), so process's buffered streams are unusabe. I've tried to avoid that converting in/out data to base64 and it works, but also sometimes.
I can't understant why it works or not sometimes. I put a piece of test code below. The first word is frame's length, but first bit is ignored. Please, tell me your guesses. Thanks.
public class BufferedSocketBase64 {
static class InToOut implements Runnable {
InputStream is;
OutputStream os;
boolean direction; //
public InToOut(InputStream is, OutputStream os, boolean direction) {
super();
this.is = is;
this.os = os;
this.direction = direction;
}
#Override
public void run() {
System.out.println(Thread.currentThread().getId() + " start "
+ ((direction) ? "encode from to" : "decode from to"));
boolean eof = false;
while (true) {
if (direction) {
// encode to base64 data
try {
int[] head = new int[2];
for (int i = 0; i < 2; i++) {
head[i] = is.read();
}
int len = (head[0] & 0xff << 8 | head[1] & 0xff) & 0x7FFF;
byte[] buf = new byte[len + 2];
buf[0] = (byte) (head[0] & 0xff);
buf[1] = (byte) (head[1] & 0xff);
for (int i = 2; i < len; i++) {
buf[i] = (byte) (is.read() & 0xff);
}
System.out.println(Thread.currentThread()
.getId() + " << " + new String(buf));
if (len > 0) {
String send = Base64Util.encode(buf, len);
send += "\n";
os.write(send.getBytes());
os.flush();
}
} catch (IOException e) {
eof = true;
}
} else { // decode from base64
try {
StringBuilder sb = new StringBuilder(1024);
byte c = 0x0a;
do {
c = (byte) is.read();
if (c >= 0 && c != 0x0a) {
sb.append(new String(new byte[] { c }));
}
} while (c != 0x0a && c >= 0);
if (sb.length() != 0) {
try {
byte[] buf = Base64Util.decode(sb.toString());
System.out.println(Thread.currentThread()
.getId() + " >> " + buf.length);
os.write(buf);
os.flush();
} catch (StringIndexOutOfBoundsException e) {
System.out
.println(Thread.currentThread().getId()
+ " error on " + sb.toString());
}
}
} catch (IOException e) {
eof = true;
}
}
if (eof) {
System.out.println(Thread.currentThread().getId() + " EOF");
break;
}
}
try {
is.close();
os.close();
} catch (IOException e) {
}
}
}
public static void main(String[] args) throws Exception {
Process proc2 = Runtime.getRuntime().exec("nc -l -p 2020");
Process proc1 = Runtime.getRuntime().exec("nc 127.0.0.1 2020");
Socket sock1 = new Socket();
sock1.connect(new InetSocketAddress("127.0.0.1", 24554), 30);
Socket sock2 = new Socket();
sock2.connect(new InetSocketAddress("127.0.0.1", 24557), 30);
new Thread(new InToOut(sock1.getInputStream(), proc1.getOutputStream(),
true)).start();
new Thread(new InToOut(proc1.getInputStream(), sock1.getOutputStream(),
false)).start();
new Thread(new InToOut(sock2.getInputStream(), proc2.getOutputStream(),
true)).start();
new Thread(new InToOut(proc2.getInputStream(), sock2.getOutputStream(),
false)).start();
}
UPDATED:
I've found right way. I uses syncchronized queries for each stream and synchronized threads to fill or erase that queries. All threads mutually blocks themselves. And it works! :)
Sorry for bother.

I've found right way. I uses syncchronized queries for each stream and synchronized threads to fill or erase that queries. All threads mutually blocks themselves. And it works! :) Sorry for bother.