We Keep Coding

Is it possible to use SSLEngine with *Blocking* I/O

I am trying to integrate some two code bases. One code base uses blocking I/O. The other code base uses non-blocking I/O.
The hook where I can integrate the two of these is a plain old-fashioned acceptor thread.
This acceptor thread reads the sub-protocol information from the socket and then based on the sub-protocol name forwards to the corresponding handler.
The other side of the code has it's own selector thread but only exposes a higher level set of constructs.
So basically I need to - in the acceptor thread's spawned worker thread - fire up the SSLEngine validate some sub-protocol information and then hand the whole thing off to the other code base's selector thread.
To make things more complicated, there is a fall-back path on the second code base whereby if it gets a Socket that was not opened with a SocketChannel it will drop down to blocking mode... and this is the bit that is causing me issues...
Namely it is not safe for me to assume that Socket.getChannel()!=null
So my SSLEngine code needs to take account of that possibility and set up the SSLEngine without using the non-blocking I/O APIs...
So far, I keep hitting blocking read calls that stall the engine...
Question does anyone know of any examples where SSLEngine has been used with a traditional InputStream/OutputStream rather than with a SocketChannel

Yes for handshake.
It is sad that we can't use SSLSocket.getSession for SSLEngine.
But we cat use SSLEngine in blocking mode.
See jdk1.8.0_112/sample/nio/server/ChannelIOSecure.java
Blocking:
method1(){
....
socketChannel.configureBlocking(true);
SSLEngine engine = sslContext.createSSLEngine();
engine.setUseClientMode(false);//server
engine.setNeedClientAuth(true);
this.outNetBB = ByteBuffer.allocate(engine.getSession().getPacketBufferSize());
outNetBB.position(0);
outNetBB.limit(0);
this.inNetBB = ByteBuffer.allocate(engine.getSession().getPacketBufferSize());
this.requestBB = ByteBuffer.allocate(engine.getSession().getApplicationBufferSize());
this.hsBB = ByteBuffer.allocate(engine.getSession().getApplicationBufferSize());
initialHSComplete = false;
while(initialHSComplete != true)
doHandshake(socketChannel, engine, null);
}
private boolean tryFlush(ByteBuffer bb, SocketChannel socketChannel) throws IOException {
socketChannel.write(bb);
return !bb.hasRemaining();
}
private SSLEngineResult.HandshakeStatus doTasks(SSLEngine sslEngine) {
Runnable runnable;
/*
* We could run this in a separate thread, but
* do in the current for now.
*/
while ((runnable = sslEngine.getDelegatedTask()) != null) {
runnable.run();
}
return sslEngine.getHandshakeStatus();
}
private ByteBuffer outNetBB;
int netBBSize;
private ByteBuffer inNetBB;
int appBBSize;
private ByteBuffer requestBB;
private ByteBuffer hsBB;
private boolean initialHSComplete; // Handshake complete status
HandshakeStatus initialHSStatus = HandshakeStatus.NEED_UNWRAP; //server
private boolean doHandshake(SocketChannel sc, SSLEngine sslEngine, SelectionKey sk) throws IOException {
SSLEngineResult result;
if (initialHSComplete) {
return initialHSComplete;
}
/*
* Flush out the outgoing buffer, if there's anything left in
* it.
*/
if (outNetBB.hasRemaining()) {
System.out.println("doha wtf");
if (!tryFlush(outNetBB, sc)) {
return false;
}
// See if we need to switch from write to read mode.
switch (initialHSStatus) {
// Is this the last buffer?
case FINISHED:
initialHSComplete = true;
// Fall-through to reregister need for a Read.
case NEED_UNWRAP:
if (sk != null) {
sk.interestOps(SelectionKey.OP_READ);
}
break;
}
return initialHSComplete;
}
switch (initialHSStatus) {
case NEED_UNWRAP:
System.out.println("before read");
if (sc.read(inNetBB) == -1) {
sslEngine.closeInbound();
return initialHSComplete;
}
System.out.println("after read");
needIO:
while (initialHSStatus == HandshakeStatus.NEED_UNWRAP) {
System.out.println("initialHSStatus"+initialHSStatus);
resizeRequestBB(); // expected room for unwrap
inNetBB.flip();
result = sslEngine.unwrap(inNetBB, requestBB);
inNetBB.compact();
System.out.println("result"+result);
initialHSStatus = result.getHandshakeStatus();
switch (result.getStatus()) {
case OK:
switch (initialHSStatus) {
case NOT_HANDSHAKING:
throw new IOException(
"Not handshaking during initial handshake");
case NEED_TASK:
initialHSStatus = doTasks(sslEngine);
break;
case FINISHED:
initialHSComplete = true;
break needIO;
}
break;
case BUFFER_UNDERFLOW:
// Resize buffer if needed.
netBBSize = sslEngine.getSession().getPacketBufferSize();
if (netBBSize > inNetBB.capacity()) {
resizeResponseBB();
}
/*
* Need to go reread the Channel for more data.
*/
if (sk != null) {
sk.interestOps(SelectionKey.OP_READ);
}
break needIO;
case BUFFER_OVERFLOW:
// Reset the application buffer size.
appBBSize =
sslEngine.getSession().getApplicationBufferSize();
break;
default: //CLOSED:
throw new IOException("Received" + result.getStatus() +
"during initial handshaking");
}
System.out.println("bottom of needIO");
} // "needIO" block.
System.out.println("after needIO "+initialHSStatus);
/*
* Just transitioned from read to write.
*/
if (initialHSStatus != HandshakeStatus.NEED_WRAP) {
break;
}
// Fall through and fill the write buffers.
case NEED_WRAP:
/*
* The flush above guarantees the out buffer to be empty
*/
outNetBB.clear();
result = sslEngine.wrap(hsBB, outNetBB);
outNetBB.flip();
initialHSStatus = result.getHandshakeStatus();
System.out.println("result wrap="+result);
switch (result.getStatus()) {
case OK:
if (initialHSStatus == HandshakeStatus.NEED_TASK) {
initialHSStatus = doTasks(sslEngine);
}
System.out.println("here");
if (sk != null) {
sk.interestOps(SelectionKey.OP_WRITE);
}
System.out.println("here2");
break;
default: // BUFFER_OVERFLOW/BUFFER_UNDERFLOW/CLOSED:
throw new IOException("Received" + result.getStatus() +
"during initial handshaking");
}
break;
default: // NOT_HANDSHAKING/NEED_TASK/FINISHED
throw new RuntimeException("Invalid Handshaking State" +
initialHSStatus);
} // switch
return initialHSComplete;
}
/*private void tryFlush(SocketChannel sc) throws IOException {
System.out.println("flush"+outNetBB);
sc.write(outNetBB);
if (!outNetBB.hasRemaining())
outNetBB.clear();
}*/
//}
//}
private void resizeResponseBB() {
ByteBuffer bb = ByteBuffer.allocate(netBBSize);
inNetBB.flip();
bb.put(inNetBB);
inNetBB = bb;
}
protected void resizeRequestBB() {
int remaining = appBBSize;
if (requestBB.remaining() < remaining) {
// Expand buffer for large request
ByteBuffer bb = ByteBuffer.allocate(requestBB.capacity() * 2);
requestBB.flip();
bb.put(requestBB);
requestBB = bb;
}
}
To change to non-blocking add:
socketChannel.configureBlocking(true);
Selector selector = Selector.open();
SelectionKey sk = socketChannel.register(selector, SelectionKey.OP_READ);
while(initialHSComplete != true){
selector.select();
doHandshake(socketChannel, engine, sk);

IOIO UART readback problems

I'm extending the BaseIOIOLooper to open up a UART device and send messages. I'm testing with a readback, where I send a packet over a line and receive that packet on another line and print it out. Because I don't want the InputStream.read() method to block, I am handling packet formation and input in a different thread. I have narrowed my problem down to the InputStream.read() method, which returns -1 (no bytes read, but no exception).
Here is what it looks like in the Looper thread:
#Override
protected void setup() throws ConnectionLostException, InterruptedException {
log_.write_log_line(log_header_ + "Beginning IOIO setup.");
// Initialize IOIO UART pins
// Input at pin 1, output at pin 2
try {
inQueue_ = MinMaxPriorityQueue.orderedBy(new ComparePackets())
.maximumSize(QUEUESIZE).create();
outQueue_ = MinMaxPriorityQueue.orderedBy(new ComparePackets())
.maximumSize(QUEUESIZE).create();
ioio_.waitForConnect();
uart_ = ioio_.openUart(1, 2, 38400, Uart.Parity.NONE, Uart.StopBits.ONE);
// Start InputHandler. Takes packets from ELKA on inQueue_
in_= new InputHandler(inQueue_, uart_.getInputStream());
in_.start();
// Start OutputHandler. Takes packets from subprocesses on outQueue_
out_= new OutputHandler(outQueue_);
out_.start();
// Get output stream
os_=uart_.getOutputStream();
// Set default target state
setTargetState(State.TRANSFERRING);
currInPacket_[0]=1; //Initial value to start transferring
log_.write_log_line(log_header_ + "IOIO setup complete.\n\t" +
"Input pin set to 1\n\tOutput pin set to 2\n\tBaud rate set to 38400\n\t" +
"Parity set to even\n\tStop bits set to 1");
} catch (IncompatibilityException e) {
log_.write_log_line(log_header_+e.toString());
} catch (ConnectionLostException e) {
log_.write_log_line(log_header_+e.toString());
} catch (Exception e) {
log_.write_log_line(log_header_+"mystery exception: "+e.toString());
}
}
And in the InputHandler thread:
#Override
public void run() {
boolean notRead;
byte i;
log_.write_log_line(log_header_+"Beginning InputHandler thread");
while (!stop) {
i = 0;
notRead = true;
nextInPacket = new byte[BUFFERSIZE];
readBytes = -1;
//StringBuilder s=new StringBuilder();
//TODO re-implement this with signals
while (i < READATTEMPTS && notRead) {
try {
// Make sure to adjust packet size. Done manually here for speed.
readBytes = is_.read(nextInPacket, 0, BUFFERSIZE);
/* Debugging
for (int j=0;j<nextInPacket.length;j++)
s.append(Byte.toString(nextInPacket[j]));
log_.write_log_line(log_header_+s.toString());
*/
if (readBytes != -1) {
notRead = false;
nextInPacket= new byte[]{1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0};
synchronized (q_) {
q_.add(nextInPacket);
}
//log_.write_log_line(log_header_ + "Incoming packet contains valid data.");
} else i++;
} catch (IOException e) {
log_.write_log_line(log_header_ + "mystery exception:\n\t" + e.toString());
}
}
if (i>=READATTEMPTS)
log_.write_log_line(log_header_+"Too many read attempts from input stream.");
/*
try {
sleep(100);
} catch (InterruptedException e) {
log_.write_log_line(log_header_+"fuck");
}
*/
}
}
On an oscilloscope, pins 1 and 2 both read an oscillating voltage, albeit at a very high amplitude, which is of some concern. Point is nothing is available to be read from the InputStream in the InputHandler class. Any ideas?

-1 returned from read() should only happen whenever the UART is closed. The closure can happen as result of explicitly calling close() on the Uart object or calling softReset() on the IOIO object.
The Android log might give you some clues about what's going on.
The reading you're seeing on the oscilloscope is suspicious: how high is "very high amplitude"? You should only ever see 0V or 3.3V on those pins, or floating in case the pins where not opened (or closed) for some reason.

Java SerialPortEvent issue: DATA_AVAILABLE calls faster then incoming data

I'm posting for the first time so I hope that I write everything according to the format.
Currently I'm working on a project using Serial Communication with RXTX between a java application and a measuring device. This works great, but now I want to catch the data that the device sends with events.
The code below works but has the following issue:
DATA_AVAILABLE will be called 4 times before all the data will be send.
I catch this in a string called vBuffer and I'm able to catch the data to get the full string.
Now I want to return this data (the full string), but cannot find a SerialPortEvent that will wait till all data is send to return this string.
In the example below I use OUTPUT_BUFFER_EMPTY but this is called at the beginning of sending a command. This means that when sending the command for the 2nd time, the OUTPUT_BUFFER_EMPTY event will return vBuffer with data from the first command, and immediately after starts the 2nd command. At 3rd time OUTPUT_BUFFER_EMPTY sends the data from the 2nd command and starts the 3rd etc.
Is there a way in DATA_AVAILABLE to wait till all data is send, or is there another event that will be called after all data is send?
Extra info: A command is send with a Stringbuilder of chars to make sure the right format for the device is send. The lay-out of a command is as follows: <STX><COMMAND><RTX><CR><LF>.
Might I be able to catch the end by looking at when the command ends? If so, how?
Update: This is the code how I send a function:
StringBuilder message = new StringBuilder();
message.append(new Character((char) 2)); // STX (Start of Text)
message.append("M"); // Command character
message.append(new Character((char) 3)); // ETX (End of Text
message.append(new Character((char) 13)); // CR (Carriage Return)
message.append(new Character((char) 10)); // LF (Line Feed)
outputStream.write(message.toString().getBytes());
After this the DATA_AVAILABLE will kick in. But doesn't wait till all received data is done.
Edit: To bump this up, still not further with the problem.
serialEvent Method :
public void serialEvent(SerialPortEvent event)
{
switch (event.getEventType())
{
case SerialPortEvent.BI:
case SerialPortEvent.OE:
case SerialPortEvent.FE:
case SerialPortEvent.PE:
case SerialPortEvent.CD:
case SerialPortEvent.CTS:
case SerialPortEvent.DSR:
case SerialPortEvent.RI:
case SerialPortEvent.OUTPUT_BUFFER_EMPTY:
if (vBuffer != "")
{
System.out.println(vBuffer);
}
break;
case SerialPortEvent.DATA_AVAILABLE:
byte[] readBuffer = new byte[40];
try
{
while (inputStream.available() > 0)
{
int numBytes = inputStream.read(readBuffer);
}
vBuffer += new String(readBuffer);
System.out.print(new String(readBuffer));
}
catch (IOException e)
{
System.out.println(e);
}
break;

I've found a way to check the command set STX and ETX too see if the message is complete (ETX is in the end of the message). If this checks true, then I've got a complete message.
Issue solved!
public void serialEvent(SerialPortEvent event)
{
if (event.getEventType() == SerialPortEvent.DATA_AVAILABLE)
{
System.out.println("Data available event received");
try
{
int available = inputStream.available();
byte[] readBuffer = new byte[available];
if (available > 0)
{
inputStream.read(readBuffer);
}
messageBuffer = messageBuffer + new String(readBuffer);
try
{
int start = messageBuffer.indexOf(new Character((char) 2));
int end = messageBuffer.indexOf(new Character((char) 10));
if (start >= 0 && end >= 0)
{
System.out.println("We found 1 complete message!!");
System.out.println(messageBuffer.substring(start, end));
_fireBufferEvent();
messageBuffer = "";
}
}
catch (IndexOutOfBoundsException ex)
{
System.out.println("IndexOutOfBoundsException, message not complete yet. Waiting for more data.");
}
}
catch (IOException ex)
{
System.out.println("IOException while reading data:");
System.out.println(ex);
}
}

Java network code not sending

I'm writing my first non-trivial Java app that uses:
networking
a GUI
threads
It's a IM program. When I send a message, the server doesn't output what it should. I'm sorry this description is so bad, but I don't know how to narrow the problem down further.
public class MachatServer {
//snip
public static void sendMessage(int targetId, int fromId, String message) {
ConnectedClient targetClient = getClient(targetId);
// Also runs
System.out.println("Sending message: " + message + "\n\nfrom " + fromId + " to " + targetId);
targetClient.addOutCommand("/message:" + fromId + ":" + message + "\n");
}
}
class ConnectedClient implements Runnable {
public void run() {
String contact;
contact = s.getInetAddress().toString();
System.out.println("Connected to " + contact);
try {
out.write("/connected" + "\n");
out.flush();
String command;
while(true) {
if(shouldExit) {
s.close();
break;
}
if(in.hasNextLine()) {
command = in.nextLine();
commandProcessor.addInCommand(command);
}
Thread.sleep(100);
}
} catch(Exception e) {
e.printStackTrace();
}
}
// snip
public void addOutCommand(String command) {
commandProcessor.addOutCommand(command);
//
// My guess is that the problem is with this method as the next line
// Does not print out.
//
//
System.out.println("" + thisId + " recieved to send: " + command);
}
}
class CommandProcessor implements Runnable {
// snip
public void run() {
String currentCommandIn;
String currentCommandOut;
while(true) {
try {
currentCommandIn = inQueue.poll();
if(currentCommandIn != null) {
System.out.println("Processing: " + currentCommandIn);
String[] commandArr = CommandParser.parseRecievedCommand(currentCommandIn);
if(commandArr[0].equalsIgnoreCase("message")) {
int target = Integer.parseInt(commandArr[1]);
String message = commandArr[2];
// This definetly runs
System.out.println("Message sending to: " + target);
MachatServer.sendMessage(target, this.conId, message);
} else if(commandArr[0].equalsIgnoreCase("quit")) {
// Tell the server to disconnect us.
MachatServer.disconnect(conId);
break;
}
currentCommandOut = outQueue.poll();
if(currentCommandOut != null) {
try {
out.write(currentCommandOut + "\n");
System.out.println(currentCommandOut + "sent");
out.flush();
} catch (IOException e) {
e.printStackTrace();
}
}
} catch(Exception e) {
e.printStackTrace();
}
public synchronized void addOutCommand(String command) {
if(command != null) {
try {
outQueue.push(command);
} catch(Exception e) {
System.out.println(command);
e.printStackTrace();
}
// Does not print
System.out.println("Ready to send: " + command);
} else {
System.out.println("Null command recieved");
}
//snip
}
The full source code is at my github, in case I have narrowed the problem down incorrectly.
The expected output should be when I telnet in and send "/message:0:test", it should send "/message:myid:test" to the client with ID 0. The actual output is nothing.

This is probably not a complete answer, but there are a few serious issues with your code that could be the cause of your problem, so you should fix those first.
First, the loop in CommandProcessor.run is busy-waiting, i.e., it runs constantly. You should use blocking operations. Also, inQueue and outQueue are accessed from two different threads so you need synchronization on every access. I recommend using something implementing the java.util.concurrent.BlockingQueue interface to solve both issues. And finally, when checking your full code, it appears that you also need to synchronize access to the ConnedtedClient.shouldExit field (I believe you can use `java.util.concurrent.atomic.AtomicBoolean as a replacement but I'm not sure).
And the reason why this could be the cause of your problem: Since CommandProcessor.run is not synchronizing on anything (or accessing anything volatile), the Java virtual machine can assume that nothing from outside can modify anything it examines, so in theory, when the run method first notices that inQueue and outQueue are both empty, it can optimize the whole method into nothing, as it can assume that it is the only thing that can modify them. But I don't know whether this can actually happen in practice, as the JVM needs to know quite a bit about the LinkedList implementation and notice that the thread is just doing these two checks in a loop. But it's always best to be safe because that way your code is guaranteed to work.

The field outQueue is uninitialized in CommandProcessor, and you commented out the printStackTrace() that would have helped you figure it out.

Maybe the problem is that the data you send is to short....
A friend of mine had a similar problem a couple of years ago, and it turned out the data was being buffered until it had enough data to send...
It had something to do with optimizing the amount of network traffic... I believe he mentioned something called "Nagle's algorithm" when he finally solved it....
Hope this can be of some help...

How do I handle multiple streams in Java?

I'm trying to run a process and do stuff with its input, output and error streams. The obvious way to do this is to use something like select(), but the only thing I can find in Java that does that is Selector.select(), which takes a Channel. It doesn't appear to be possible to get a Channel from an InputStream or OutputStream (FileStream has a getChannel() method but that doesn't help here)
So, instead I wrote some code to poll all the streams:
while( !out_eof || !err_eof )
{
while( out_str.available() )
{
if( (bytes = out_str.read(buf)) != -1 )
{
// Do something with output stream
}
else
out_eof = true;
}
while( err_str.available() )
{
if( (bytes = err_str.read(buf)) != -1 )
{
// Do something with error stream
}
else
err_eof = true;
}
sleep(100);
}
which works, except that it never terminates. When one of the streams reaches end of file, available() returns zero so read() isn't called and we never get the -1 return that would indicate EOF.
One solution would be a non-blocking way to detect EOF. I can't see one in the docs anywhere. Alternatively is there a better way of doing what I want to do?
I see this question here:
link text
and although it doesn't exactly do what I want, I can probably use that idea, of spawning separate threads for each stream, for the particular problem I have now. But surely that isn't the only way to do it? Surely there must be a way to read from multiple streams without using a thread for each?

As you said, the solution outlined in this Answer is the traditional way of reading both stdout and stderr from a Process. A thread-per-stream is the way to go, even though it is slightly annoying.

You will indeed have to go the route of spawning a Thread for each stream you want to monitor. If your use case allows for combining both stdout and stderr of the process in question you need only one thread, otherwise two are needed.
It took me quite some time to get it right in one of our projects where I have to launch an external process, take its output and do something with it while at the same time looking for errors and process termination and also being able to terminate it when the java app's user cancels the operation.
I created a rather simple class to encapsulate the watching part whose run() method looks something like this:
public void run() {
BufferedReader tStreamReader = null;
try {
while (externalCommand == null && !shouldHalt) {
logger.warning("ExtProcMonitor("
+ (watchStdErr ? "err" : "out")
+ ") Sleeping until external command is found");
Thread.sleep(500);
}
if (externalCommand == null) {
return;
}
tStreamReader =
new BufferedReader(new InputStreamReader(watchStdErr ? externalCommand.getErrorStream()
: externalCommand.getInputStream()));
String tLine;
while ((tLine = tStreamReader.readLine()) != null) {
logger.severe(tLine);
if (filter != null) {
if (filter.matches(tLine)) {
informFilterListeners(tLine);
return;
}
}
}
} catch (IOException e) {
logger.logExceptionMessage(e, "IOException stderr");
} catch (InterruptedException e) {
logger.logExceptionMessage(e, "InterruptedException waiting for external process");
} finally {
if (tStreamReader != null) {
try {
tStreamReader.close();
} catch (IOException e) {
// ignore
}
}
}
}
On the calling side it looks like this:
Thread tExtMonitorThread = new Thread(new Runnable() {
public void run() {
try {
while (externalCommand == null) {
getLogger().warning("Monitor: Sleeping until external command is found");
Thread.sleep(500);
if (isStopRequested()) {
getLogger()
.warning("Terminating external process on user request");
if (externalCommand != null) {
externalCommand.destroy();
}
return;
}
}
int tReturnCode = externalCommand.waitFor();
getLogger().warning("External command exited with code " + tReturnCode);
} catch (InterruptedException e) {
getLogger().logExceptionMessage(e, "Interrupted while waiting for external command to exit");
}
}
}, "ExtCommandWaiter");
ExternalProcessOutputHandlerThread tExtErrThread =
new ExternalProcessOutputHandlerThread("ExtCommandStdErr", getLogger(), true);
ExternalProcessOutputHandlerThread tExtOutThread =
new ExternalProcessOutputHandlerThread("ExtCommandStdOut", getLogger(), true);
tExtMonitorThread.start();
tExtOutThread.start();
tExtErrThread.start();
tExtErrThread.setFilter(new FilterFunctor() {
public boolean matches(Object o) {
String tLine = (String)o;
return tLine.indexOf("Error") > -1;
}
});
FilterListener tListener = new FilterListener() {
private boolean abortFlag = false;
public boolean shouldAbort() {
return abortFlag;
}
public void matched(String aLine) {
abortFlag = abortFlag || (aLine.indexOf("Error") > -1);
}
};
tExtErrThread.addFilterListener(tListener);
externalCommand = new ProcessBuilder(aCommand).start();
tExtErrThread.setProcess(externalCommand);
try {
tExtMonitorThread.join();
tExtErrThread.join();
tExtOutThread.join();
} catch (InterruptedException e) {
// when this happens try to bring the external process down
getLogger().severe("Aborted because auf InterruptedException.");
getLogger().severe("Killing external command...");
externalCommand.destroy();
getLogger().severe("External command killed.");
externalCommand = null;
return -42;
}
int tRetVal = tListener.shouldAbort() ? -44 : externalCommand.exitValue();
externalCommand = null;
try {
getLogger().warning("command exit code: " + tRetVal);
} catch (IllegalThreadStateException ex) {
getLogger().warning("command exit code: unknown");
}
return tRetVal;
Unfortunately I don't have to for a self-contained runnable example, but maybe this helps.
If I had to do it again I would have another look at using the Thread.interrupt() method instead of a self-made stop flag (mind to declare it volatile!), but I leave that for another time. :)