We Keep Coding

Using Netty with ClamAV Instream

I have been stuggling with a configuration using Netty to stream bytes to a ClamAV service. I am running in an Apache Camel route.
Using Netty, I am unable to intercept the "INSTREAM size limit exceeded" message.
INSTREAM
It is mandatory to prefix this command with n or z.
Scan a stream of data. The stream is sent to clamd in chunks, after INSTREAM, on the same socket on which the command was sent. This avoids the overhead of establishing new TCP connections and problems with NAT. The format of the chunk is: '' where is the size of the following data in bytes expressed as a 4 byte unsigned integer in network byte order and is the actual chunk. Streaming is terminated by sending a zero-length chunk. Note: do not exceed StreamMaxLength as defined in clamd.conf, otherwise clamd will reply with INSTREAM size limit exceeded and close the connection.
Using a straight synchronous socket connection I have no issues. Can anyone point me in the right direction for how I should be using Netty to do this? Or should I just stick with a synchronous socket connection.
Implementation using synchronous sockets. Credit to https://github.com/solita/clamav-java "Antti Virtanen".
private class UseSocket implements Processor{
#Override
public void process(Exchange exchange) throws Exception{
try (BufferedInputStream message = new BufferedInputStream(exchange.getIn().getBody(InputStream.class));
Socket socket = new Socket("localhost", 3310);
BufferedOutputStream socketOutput = new BufferedOutputStream(socket.getOutputStream())){
byte[] command = "zINSTREAM\0".getBytes();
socketOutput.write(command);
socketOutput.flush();
byte[] chunk = new byte[2048];
int chunkSize;
try(BufferedInputStream socketInput = new BufferedInputStream(socket.getInputStream())){
for(chunkSize = message.read(chunk);chunkSize > -1;chunkSize = message.read(chunk)){
socketOutput.write(ByteBuffer.allocate(4).putInt(chunkSize).array());
socketOutput.write(chunk, 0, chunkSize);
socketOutput.flush();
if(processReply(socketInput, exchange)){
return;
}
}
socketOutput.write(ByteBuffer.allocate(4).putInt(0).array());
socketOutput.flush();
processReply(socketInput, exchange);
}
}
}
private boolean processReply(BufferedInputStream in, Exchange exchange) throws Exception{
if(in.available() > 0) {
logger.info("processing reply");
byte[] replyBytes = new byte[256];
int replySize = in.read(replyBytes);
if (replySize > 0) {
String reply = new String(replyBytes, 0, replySize, StandardCharsets.UTF_8);
String avStatus = "infected";
if ("stream: OK\0".equals(reply)) {
avStatus = "clean";
} else if ("INSTREAM size limit exceeded. ERROR\0".equals(reply)) {
avStatus = "overflow";
}
exchange.getIn().setHeader("av-status", avStatus);
return true;
}
}
return false;
}
}
Implementation using Netty with inbound and outbound channel handlers.
private class UseNetty implements Processor{
#Override
public void process(Exchange exchange) throws Exception{
logger.info(CLASS_NAME + ": Creating Netty client");
EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
try{
Bootstrap bootstrap = new Bootstrap();
bootstrap.group(eventLoopGroup);
bootstrap.channel(NioSocketChannel.class);
bootstrap.remoteAddress(new InetSocketAddress("localhost", 3310));
bootstrap.handler(new ClamAvChannelIntializer(exchange));
ChannelFuture channelFuture = bootstrap.connect().sync();
channelFuture.channel().closeFuture().sync();
}catch(Exception ex) {
logger.error(CLASS_NAME + ": ERROR", ex);
}
finally
{
eventLoopGroup.shutdownGracefully();
logger.info(CLASS_NAME + ": Netty client closed");
}
}
}
public class ClamAvChannelIntializer extends ChannelInitializer<SocketChannel> {
private Exchange exchange;
public ClamAvChannelIntializer(Exchange exchange){
this.exchange = exchange;
}
#Override
protected void initChannel(SocketChannel socketChannel) throws Exception {
socketChannel.pipeline().addLast(new ClamAvClientWriter());
socketChannel.pipeline().addLast(new ClamAvClientHandler(exchange));
}
}
public class ClamAvClientHandler extends SimpleChannelInboundHandler<ByteBuf> {
String CLASS_NAME;
Logger logger;
private Exchange exchange;
public static final int MAX_BUFFER = 2048;
public ClamAvClientHandler(Exchange exchange){
super();
CLASS_NAME = this.getClass().getName();
logger = LoggerFactory.getLogger(CLASS_NAME);
this.exchange = exchange;
}
#Override
public void channelActive(ChannelHandlerContext channelHandlerContext) throws Exception{
logger.info(CLASS_NAME + ": Entering channelActive");
channelHandlerContext.write(exchange);
logger.info(CLASS_NAME + ": Exiting channelActive");
}
#Override
public void exceptionCaught(ChannelHandlerContext channelHandlerContext, Throwable cause){
cause.printStackTrace();
channelHandlerContext.close();
}
#Override
protected void channelRead0(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf) {
logger.info(CLASS_NAME + ": Entering channelRead0");
String reply = byteBuf.toString(CharsetUtil.UTF_8);
logger.info(CLASS_NAME + ": Reply = " + reply);
String avStatus = "infected";
if ("stream: OK\0".equals(reply)) {
avStatus = "clean";
} else if ("INSTREAM size limit exceeded. ERROR\0".equals(reply)) {
avStatus = "overflow";
} else{
logger.warn("Infected or unknown reply = " + reply);
}
exchange.getIn().setHeader("av-status", avStatus);
logger.info(CLASS_NAME + ": Exiting channelRead0");
channelHandlerContext.close();
}
}
public class ClamAvClientWriter extends ChannelOutboundHandlerAdapter {
String CLASS_NAME;
Logger logger;
public static final int MAX_BUFFER = 64000;//2^16
public ClamAvClientWriter(){
CLASS_NAME = this.getClass().getName();
logger = LoggerFactory.getLogger(CLASS_NAME);
}
#Override
public void write(ChannelHandlerContext channelHandlerContext, Object o, ChannelPromise channelPromise) throws Exception{
logger.info(CLASS_NAME + ": Entering write");
Exchange exchange = (Exchange)o;
try(BufferedInputStream message = new BufferedInputStream(exchange.getIn().getBody(InputStream.class))){
channelHandlerContext.writeAndFlush(Unpooled.copiedBuffer("zINSTREAM\0".getBytes()));
byte[] chunk = new byte[MAX_BUFFER];
for(int i=message.read(chunk);i>-1;i=message.read(chunk)){
byte[] chunkSize = ByteBuffer.allocate(4).putInt(i).array();
channelHandlerContext.write(Unpooled.copiedBuffer(chunkSize));
channelHandlerContext.writeAndFlush(Unpooled.copiedBuffer(chunk, 0, i));
}
channelHandlerContext.writeAndFlush(Unpooled.copiedBuffer(ByteBuffer.allocate(4).putInt(0).array()));
}
logger.info(CLASS_NAME + ": Exiting write");
}
}

I finally gave up on trying to use Netty for this. I created a new Camel Processor and packaged the socket stream in it. Code below in case anyone runs into a similar issue.
public class ClamAvInstream implements Processor {
Logger logger;
private final int MAX_BUFFER = 2048;
public ClamAvInstream() {
logger = LoggerFactory.getLogger(this.getClass().getName());
}
#Override
public void process(Exchange exchange) throws Exception {
try (BufferedInputStream message = new BufferedInputStream(exchange.getIn().getBody(InputStream.class));
Socket socket = new Socket("localhost", 3310);
BufferedOutputStream socketOutput = new BufferedOutputStream(socket.getOutputStream())) {
byte[] command = "zINSTREAM\0".getBytes();
socketOutput.write(command);
socketOutput.flush();
byte[] chunk = new byte[MAX_BUFFER];
int chunkSize;
try (BufferedInputStream socketInput = new BufferedInputStream(socket.getInputStream())) {
for (chunkSize = message.read(chunk); chunkSize > -1; chunkSize = message.read(chunk)) {
socketOutput.write(ByteBuffer.allocate(4).putInt(chunkSize).array());
socketOutput.write(chunk, 0, chunkSize);
socketOutput.flush();
receivedReply(socketInput, exchange);
}
socketOutput.write(ByteBuffer.allocate(4).putInt(0).array());
socketOutput.flush();
receivedReply(socketInput, exchange);
} catch(ClamAvException ex){ //close socketInput
logger.warn(ex.getMessage());
}
}//close message, socket, socketOutput
}
private class ClamAvException extends Exception{
private ClamAvException(String error){
super(error);
}
}
private void receivedReply(BufferedInputStream in, Exchange exchange) throws Exception{
if(in.available() > 0){
byte[] replyBytes = new byte[256];
int replySize = in.read(replyBytes);
if (replySize > 0) {
String reply = new String(replyBytes, 0, replySize, StandardCharsets.UTF_8);
logger.info("reply="+reply);
if(reply.contains("OK")){
exchange.getIn().setHeader("av-status", "clean");
}else if(reply.contains("ERROR")){
if(reply.equals("INSTREAM size limit exceeded. ERROR\0")){
exchange.getIn().setHeader("av-status", "overflow");
}else {
exchange.getIn().setHeader("av-status", "error");
}
throw new ClamAvException(reply);
}else if(reply.contains("FOUND")){
exchange.getIn().setHeader("av-status", "infected");
}else{
exchange.getIn().setHeader("av-status", "unknown");
}
}
}
}
}

Netty server's channelRead receives empty ButBuf with 0 size

------------ client-side pipeline (Client can send any type of messages i.e. HTTP requests, or binary packets)--------
Bootstrap bootstrap = new Bootstrap()
.group(group)
// .option(ChannelOption.TCP_NODELAY, true)
// .option(ChannelOption.SO_KEEPALIVE, true)
.channel(NioSocketChannel.class)
.handler(new ChannelInitializer() {
#Override
protected void initChannel(Channel channel) throws Exception {
channel.pipeline()
.addLast("agent-traffic-shaping", ats)
.addLast("length-decoder", new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4))
.addLast("agent-client", new AgentClientHandler())
.addLast("4b-length", new LengthFieldPrepender(4))
;
}
});
------------------------------ Server-side pipeline-----------------
ServerBootstrap b = new ServerBootstrap()
.group(group)
// .option(ChannelOption.TCP_NODELAY, true)
// .option(ChannelOption.SO_KEEPALIVE, true)
.channel(NioServerSocketChannel.class)
.localAddress(new InetSocketAddress(port))
.childHandler(new ChannelInitializer() {
#Override
protected void initChannel(Channel channel) throws Exception {
channel.pipeline()
.addLast("agent-traffic-shaping", ats)
.addLast("length-decoder", new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4))
.addLast(new AgentServerHandler())
.addLast("4b-length", new LengthFieldPrepender(4));
}
}
);
ChannelFuture f = b.bind().sync();
log.info("Started agent-side server at Port {}", port);
-------- Server's channelRead method-----------------
#Override
public void channelRead(ChannelHandlerContext ctx, Object msg) {
ByteBuf data = (ByteBuf) msg;
log.info("SIZE {}", data.capacity());
String s = data.readCharSequence(data.capacity(), Charset.forName("utf-8")).toString();
System.out.print(s);
if (buffer != null) buffer.incomingPacket((ByteBuf) msg);
else {
log.error("Receiving buffer NULL for Remote Agent {}:{} ", remoteAgentIP, remoteAgentPort);
((ByteBuf) msg).release();
}
/* totalBytes += ((ByteBuf) msg).capacity();*/
}
------------ Client writing on Channel (ByteBuf data contains valid HTTP request with size of 87 Bytes)--------
private void writeToAgentChannel(Channel currentChannel, ByteBuf data) {
String s = data.readCharSequence(data.capacity(), Charset.forName("utf-8")).toString();
log.info("SIZE {}", data.capacity());
System.out.print(s);
ChannelFuture cf = currentChannel.write(data);
currentChannel.flush();
/* wCount++;
if (wCount >= request.getRequest().getBufferSize() * request.getRequest().getNumParallelSockets()) {
for (Channel channel : channels)
channel.flush();
wCount = 0;
}*/
cf.addListener(new ChannelFutureListener() {
#Override
public void operationComplete(ChannelFuture channelFuture) throws Exception {
if (cf.isSuccess()) {
totalBytes += data.capacity();
}
else log.error("Failed to write packet to channel {}", cf.cause());
}
});
}
However Server receives an empty ByteBuf with size of zero. What could be possible cause here?

Your issue seems to come from the client, where you accidentally consume all bytes inside a bytebuf when you try to debug it.
String s = data.readCharSequence(data.capacity(), Charset.forName("utf-8")).toString();
log.info("SIZE {}", data.capacity());
System.out.print(s);
Calling readCharSequence consumes the data, leaving you with 0 bytes left.
I suggest using a DebugHandler to debug your pipeline, as that is tested not to affect the data

Netty async write response and large data unknown with size

I develop a netty http server, but when I write the response in the method ChannelInboundHandlerAdapter.channelRead0, my response result comes from another server and the size of the result is unknown, so its http response headers maybe has content-length or chunked. so I use a buffer, if it's enough (read up full data) regardless of content-length or chunked, I use content-length, otherwise I use chunked.
How I hold the write channel of first connection then pass it to the seconde Handler inorder to write the response. ( I just directly pass ctx to write but nothing returns)
How I conditionally decide write chunked data to channel or normal data with content-length (it seems not to work to add ChunkWriteHandler if chunk is needed when channelRead0.
take a simple code for example:
```java
EventLoopGroup bossGroup = new NioEventLoopGroup();
final EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
ServerBootstrap serverBootstrap = new ServerBootstrap();
serverBootstrap.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class)
.handler(new LoggingHandler(LogLevel.INFO))
.childHandler(new ChannelInitializer<Channel>(){
#Override
protected void initChannel(Channel ch) throws Exception
{
System.out.println("Start, I accept client");
ChannelPipeline pipeline = ch.pipeline();
// Uncomment the following line if you want HTTPS
// SSLEngine engine =
// SecureChatSslContextFactory.getServerContext().createSSLEngine();
// engine.setUseClientMode(false);
// pipeline.addLast("ssl", new SslHandler(engine));
pipeline.addLast("decoder", new HttpRequestDecoder());
// Uncomment the following line if you don't want to handle HttpChunks.
// pipeline.addLast("aggregator", new HttpChunkAggregator(1048576));
pipeline.addLast("encoder", new HttpResponseEncoder());
// Remove the following line if you don't want automatic content
// compression.
//pipeline.addLast("aggregator", new HttpChunkAggregator(1048576));
pipeline.addLast("chunkedWriter", new ChunkedWriteHandler());
pipeline.addLast("deflater", new HttpContentCompressor());
pipeline.addLast("handler", new SimpleChannelInboundHandler<HttpObject>(){
#Override
protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception
{
System.out.println("msg=" + msg);
final ChannelHandlerContext ctxClient2Me = ctx;
// TODO: Implement this method
Bootstrap bs = new Bootstrap();
try{
//bs.resolver(new DnsAddressResolverGroup(NioDatagramChannel.class, DefaultDnsServerAddressStreamProvider.INSTANCE));
//.option(ChannelOption.TCP_NODELAY, java.lang.Boolean.TRUE)
bs.resolver(DefaultAddressResolverGroup.INSTANCE);
}catch(Exception e){
e.printStackTrace();
}
bs.channel(NioSocketChannel.class);
EventLoopGroup cg = workerGroup;//new NioEventLoopGroup();
bs.group(cg).handler(new ChannelInitializer<Channel>(){
#Override
protected void initChannel(Channel ch) throws Exception
{
System.out.println("start, server accept me");
// TODO: Implement this method
ch.pipeline().addLast("http-request-encode", new HttpRequestEncoder());
ch.pipeline().addLast(new HttpResponseDecoder());
ch.pipeline().addLast("http-res", new SimpleChannelInboundHandler<HttpObject>(){
#Override
protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception
{
// TODO: Implement this method
System.out.println("target = " + msg);
//
if(msg instanceof HttpResponse){
HttpResponse res = (HttpResponse) msg;
HttpUtil.isTransferEncodingChunked(res);
DefaultHttpResponse resClient2Me = new DefaultHttpResponse(HttpVersion.HTTP_1_1, res.getStatus());
//resClient2Me.headers().set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED);
//resClient2Me.headers().set(HttpHeaderNames.CONTENT_LENGTH, "");
ctxClient2Me.write(resClient2Me);
}
if(msg instanceof LastHttpContent){
// now response the request of the client, it wastes x seconds from receiving request to response
ctxClient2Me.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT).addListener(ChannelFutureListener.CLOSE);
ctx.close();
}else if( msg instanceof HttpContent){
//ctxClient2Me.write(new DefaultHttpContent(msg)); write chunk by chunk ?
}
}
});
System.out.println("end, server accept me");
}
});
final URI uri = new URI("http://example.com/");
String host = uri.getHost();
ChannelFuture connectFuture= bs.connect(host, 80);
System.out.println("to connect me to server");
connectFuture.addListener(new ChannelFutureListener(){
#Override
public void operationComplete(ChannelFuture cf) throws Exception
{
}
});
ChannelFuture connetedFuture = connectFuture.sync(); // TODO optimize, wait io
System.out.println("connected me to server");
DefaultFullHttpRequest req = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri.getRawPath());
//req.headers().set(HttpHeaderNames.HOST, "");
connetedFuture.channel().writeAndFlush(req);
System.out.println("end of Client2Me channelRead0");
System.out.println("For the seponse of Me2Server, see SimpleChannelInboundHandler.channelRead0");
}
});
System.out.println("end, I accept client");
}
});
System.out.println("========");
ChannelFuture channelFuture = serverBootstrap.bind(2080).sync();
channelFuture.channel().closeFuture().sync();
} finally {
bossGroup.shutdownGracefully();
workerGroup.shutdownGracefully();
}
```

After a bit of struggle trying to send response from non-Netty eventloop thread, I finally figured out the problem. If your client is closing the outputstream using
socketChannel.shutdownOutput()
then you need to set ALLOW_HALF_CLOSURE property true in Netty so it won't close the channel.
Here's a sample server. The client is left as an exercise to the reader :-)
final ServerBootstrap b = new ServerBootstrap();
b.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.option(ChannelOption.SO_KEEPALIVE, true)
.option(ChannelOption.ALLOW_HALF_CLOSURE, true) // This option doesn't work
.handler(new LoggingHandler(LogLevel.INFO))
.childHandler(new ChannelInitializer<io.netty.channel.socket.SocketChannel>() {
#Override
protected void initChannel(io.netty.channel.socket.SocketChannel ch) throws Exception {
ch.pipeline().addLast(new ChannelInboundHandlerAdapter() {
#Override
public void channelRegistered(ChannelHandlerContext ctx) throws Exception {
ctx.channel().config().setOption(ChannelOption.ALLOW_HALF_CLOSURE, true); // This is important
}
#Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
ByteBuffer byteBuffer = ((ByteBuf) msg).nioBuffer();
String id = ctx.channel().id().asLongText();
// When Done reading all the bytes, send response 1 second later
timer.schedule(new TimerTask() {
#Override
public void run() {
ctx.write(Unpooled.copiedBuffer(CONTENT.asReadOnlyBuffer()));
ctx.flush();
ctx.close();
log.info("[{}] Server time to first response byte: {}", id, System.currentTimeMillis() - startTimes.get(id));
startTimes.remove(id);
}
}, 1000);
}
}
}
});
Channel ch = b.bind("localhost", PORT).sync().channel();
ch.closeFuture().sync();
Ofcourse, as mentioned by others in the thread, you cannot send Strings, you need to send a ByteBuf using Unpooled.copiedBuffer

See the comments about Channel, so you can reserve the Channel received in ChannelInboundHandlerAdapter.channelRead(ChannelHandlerContext ctx, Object msg) (msg is not released after returning automatically) or SimpleChannelInboundHandler.channelRead0(ChannelHandlerContext ctx, I msg) (it releases the received messages automatically after returning) for later use. Maybe you can refer to the example at the end, pass the channel to another ChannelHandler.
All I/O operations are asynchronous.
All I/O operations in Netty are asynchronous. It means any I/O calls will return immediately with no guarantee that the requested I/O operation has been completed at the end of the call. Instead, you will be returned with a ChannelFuture instance which will notify you when the requested I/O operation has succeeded, failed, or canceled.
public interface Channel extends AttributeMap, Comparable<Channel> {
/**
* Request to write a message via this {#link Channel} through the {#link ChannelPipeline}.
* This method will not request to actual flush, so be sure to call {#link #flush()}
* once you want to request to flush all pending data to the actual transport.
*/
ChannelFuture write(Object msg);
/**
* Request to write a message via this {#link Channel} through the {#link ChannelPipeline}.
* This method will not request to actual flush, so be sure to call {#link #flush()}
* once you want to request to flush all pending data to the actual transport.
*/
ChannelFuture write(Object msg, ChannelPromise promise);
/**
* Request to flush all pending messages.
*/
Channel flush();
/**
* Shortcut for call {#link #write(Object, ChannelPromise)} and {#link #flush()}.
*/
ChannelFuture writeAndFlush(Object msg, ChannelPromise promise);
/**
* Shortcut for call {#link #write(Object)} and {#link #flush()}.
*/
ChannelFuture writeAndFlush(Object msg);
}
There is no need to worry about this if you has added HttpResponseEncoder (it is a subclass of HttpObjectEncoder, which has a private filed private int state = ST_INIT; to remember whether to encode HTTP body data as chunked) into ChannelPipeline, the only thing to do is add a header 'transfer-encoding: chunked', e.g. HttpUtil.setTransferEncodingChunked(srcRes, true);.
```java
public class NettyToServerChat extends SimpleChannelInboundHandler<HttpObject> {
private static final Logger LOGGER = LoggerFactory.getLogger(NettyToServerChat.class);
public static final String CHANNEL_NAME = "NettyToServer";
protected final ChannelHandlerContext ctxClientToNetty;
/** Determines if the response supports keepalive */
private boolean responseKeepalive = true;
/** Determines if the response is chunked */
private boolean responseChunked = false;
public NettyToServerChat(ChannelHandlerContext ctxClientToNetty) {
this.ctxClientToNetty = ctxClientToNetty;
}
#Override
protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception {
if (msg instanceof HttpResponse) {
HttpResponse response = (HttpResponse) msg;
HttpResponseStatus resStatus = response.status();
//LOGGER.info("Status Line: {} {} {}", response.getProtocolVersion(), resStatus.code(), resStatus.reasonPhrase());
if (!response.headers().isEmpty()) {
for (CharSequence name : response.headers().names()) {
for (CharSequence value : response.headers().getAll(name)) {
//LOGGER.info("HEADER: {} = {}", name, value);
}
}
//LOGGER.info("");
}
//response.headers().set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED);
HttpResponse srcRes = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
if (HttpUtil.isTransferEncodingChunked(response)) {
responseChunked = true;
HttpUtil.setTransferEncodingChunked(srcRes, true);
ctxNettyToServer.channel().write(srcRes);
//ctx.channel().pipeline().addAfter(CHANNEL_NAME, "ChunkedWrite", new ChunkedWriteHandler());
} else {
ctxNettyToServer.channel().write(srcRes);
//ctx.channel().pipeline().remove("ChunkedWrite");
}
}
if (msg instanceof LastHttpContent) { // prioritize the subclass interface
ctx.close();
LOGGER.debug("ctxNettyToServer.channel().isWritable() = {}", ctxNettyToServer.channel().isWritable());
Thread.sleep(3000);
LOGGER.debug("ctxNettyToServer.channel().isWritable() = {}", ctxNettyToServer.channel().isWritable());
if(!responseChunked){
HttpContent content = (HttpContent) msg;
// https://github.com/netty/netty/blob/4.1/transport/src/main/java/io/netty/channel/SimpleChannelInboundHandler.java
// #see {#link SimpleChannelInboundHandler<I>#channelRead(ChannelHandlerContext, I)}
ctxNettyToServer.writeAndFlush(content.retain()).addListener(ChannelFutureListener.CLOSE);
}else{
ctxNettyToServer.close();
}
LOGGER.debug("ctxNettyToServer.channel().isWritable() = {}", ctxNettyToServer.channel().isWritable());
} else if (msg instanceof HttpContent) {
HttpContent content = (HttpContent) msg;
// We need to do a ReferenceCountUtil.retain() on the buffer to increase the reference count by 1
ctxNettyToServer.write(content.retain());
}
}
}
```

Cannot reply in netty using udp socket

I would like to use netty to create a udp server that reply back.
The old code was in c and used this kind of code :
sockHndl = socket(AF_INET,SOCK_DGRAM,0);
sockConf.sin_family = AF_INET;
sockConf.sin_addr.s_addr = INADDR_ANY;
sockConf.sin_port = htons(33333);
bind(sockHndl, (struct sockaddr*)&sockConf, sizeof(sockConf));
// in a thread
recvfrom(sockHndl, buffer, 1515, 0, (struct sockaddr*)&sockConf, &s);
// process any recevied data
....
// reply
sendto(sockHndl, buffer, strlen(buffer), 0, (struct sockaddr*)&sockConf, sizeof(sockConf));
This code works, but I have to re-implement it in java using netty.
Using netty in tcp is not a problem, but replying with netty to the client is a problem for me, if I want to write on the channel, it failed with a java.nio.channels.NotYetConnectedException.
Maybe this is linked to this issue on netty, but I don't know how I can reproduce the c behaviour in my code.
final Bootstrap b = new Bootstrap();
b.group(comIOEventLoop).channel(NioDatagramChannel.class).handler(new ChannelInitializer<NioDatagramChannel>()
{
protected void initChannel(NioDatagramChannel ch) throws Exception
{
final ChannelPipeline p = ch.pipeline();
p.addLast("SITCDecoder", new SITCDecoder());
p.addLast("SITCEncoder", new SITCEncoder());
p.addLast("SITCHandler", new SimpleChannelInboundHandler<ITCUdpRequest>()
{
protected void channelRead0(ChannelHandlerContext ctx, ITCUdpRequest msg) throws Exception
{
// here ctx.channel().remoteAddress() == null !
switch(msg.getType())
{
case GET_PARAMS_0:
send(ctx.channel(), new ITCUdpResponse(-80, -20));
break;
}
}
public void send(final Channel channel, final ITCUdpResponse data)
{
if(data == null)
{
LOGGER.error("data == null !!!");
}
channel.writeAndFlush(data).addListener(new GenericFutureListener<Future<Void>>()
{
public void operationComplete(final Future<Void> future) throws Exception
{
if(future.isDone() && future.isSuccess())
{
LOGGER.debug("OK");
}
else
{
LOGGER.error("error " + future.isDone() + " - " + future.isSuccess());
if(!future.isSuccess())
{
future.cause().printStackTrace();
}
}
}
});
}
});
}
});
channel = b.bind(port).sync().channel();
channel.closeFuture().await();
How can I reply to the client ?
Thanks.
Extra edit :
Decoder code :
public class SITCDecoder extends SimpleChannelInboundHandler<DatagramPacket>
{
private static final Logger LOGGER = LoggerFactory.getLogger(SITCDecoder.class);
private static final String MSG_DD_HEADER = "SITCDDVAL:";
private static final int MSG_DD_HEADER_SIZE = MSG_DD_HEADER.length();
protected void channelRead0(final ChannelHandlerContext ctx, final DatagramPacket msg) throws Exception
{
final ByteBuf data = (ByteBuf) msg.content();
if(data != null)
{
// we must be able to read at last <code>MSG_HEADER_SIZE</code> Bytes
if(data.readableBytes() < MSG_DD_HEADER_SIZE)
{
LOGGER.error("Not enought data");
return;
}
if(!data.readBytes(MSG_DD_HEADER_SIZE).toString(StandardCharsets.ISO_8859_1).equals(MSG_DD_HEADER))
{
LOGGER.error("Header not found");
}
final String payload = data.readBytes(data.readableBytes()).toString(StandardCharsets.ISO_8859_1);
final RecyclableArrayList out = RecyclableArrayList.newInstance();
out.add(new ITCUdpRequest(payload));
final int size = out.size();
for(int i = 0; i < size; i++)
{
ctx.fireChannelRead(out.get(i));
}
out.recycle();
}
else
{
ctx.fireChannelRead(msg);
}
}
}
Encoder code :
public class SITCEncoder extends MessageToByteEncoder<ITCUdpResponse>
{
private static final String MSG_RP_HEADER = "SITCRPVAL:";
private static final char MSG_RP_FOOTER = '*';
private static final char VAL_PREFIX = '[';
private static final char VAL_SUFFIX = ']';
protected void encode(final ChannelHandlerContext ctx, final ITCUdpResponse msg, final ByteBuf out) throws Exception
{
final StringBuilder str = new StringBuilder();
str.append(MSG_RP_HEADER);
for(final Object val : msg.getParams())
{
str.append(VAL_PREFIX).append(val).append(VAL_SUFFIX);
}
str.append(MSG_RP_FOOTER);
out.ensureWritable(str.length());
out.writeBytes(str.toString().getBytes());
}
}

Ok, so after reading so suggestions that stackoverflow gave me for my question, I have switched from Nio* to Oio* (like said here) and everything works as expected without changing anything.
Thansk.
PS: why Oio works where Nio don't ?

Java Websocket Client without a Browser

I am working on a project that requires real-time interaction between users. I want to have a HTML5 web client (simple enough) and also a local client (preferably Java) with both being able to connect to the server. I have done some research and have not found a conclusive answer to whether or not the local client can connect to the server without a browser.
Question: Is there any way to connect from a local Java client to a websocket server without a browse? I have seen some browser wrappers in other languages that might make this possible. If not, I am open to suggestions.
Thanks.

You might also consider using JSR 356 - Java API for WebSocket. It is part of Java EE 7, but client can be run from plain Java SE without any issues. There are multiple implementations available right now and following will work in all of them:
programmatic API:
final WebSocketContainer webSocketContainer = ContainerProvider.getWebSocketContainer();
Session session = webSocketContainer.connectToServer(new Endpoint() {
#Override
public void onOpen(Session session, EndpointConfig config) {
// session.addMessageHandler( ... );
}
}, URI.create("ws://some.uri"));
annotated API:
public static void main(String[] args) throws IOException, DeploymentException {
final WebSocketContainer webSocketContainer = ContainerProvider.getWebSocketContainer();
webSocketContainer.connectToServer(MyEndpoint.class, URI.create("ws://some.uri"));
}
#ClientEndpoint
public static class MyEndpoint {
// text
#OnMessage
void onMessage(Session session, String message) {
// ...
}
// binary
#OnMessage
void onMessage(Session session, ByteBuffer message) {
// ...
}
// #OnClose, #OnOpen, #OnError
}
please see linked page for further details (full specification).
There are various implementations out here, basically every Java container has one. I am working on Glassfish/WebLogic implementation and its called Tyrus, feel free to try it out (we provide easy to use all in one bundle, see http://search.maven.org/...).

You most certainly CAN utilize WebSockets from desktop applications in Java, outside the browser sandbox. The thinking behind this is that you can create thick clients that create TCP connections, so of course they should be able to create WebSocket connections on top of those TCP connections.
One of the newest and best APIs for doing so is written by Kaazing, taking the point of view that a WebSocket is just like a socket and can be created using simple "ws://" URIs.
The API is discussed in detail on the Kaazing Gateway 5.0 Java WebSocket Documentation site. You can download the plain Gateway from Kaazing here
Create a websocket:
import com.kaazing.net.ws.WebSocket;
import com.kaazing.net.ws.WebSocketFactory;
wsFactory = WebSocketFactory.createWebSocketFactory();
ws = wsFactory.createWebSocket(URI.create("ws://example.com:8001/path"));
ws.connect(); // This will block or throw an exception if failed.
To send messages, add a WebSocketMessageWriter object:
WebSocketMessageWriter writer = ws.getMessageWriter();
String text = "Hello WebSocket!";
writer.writeText(text); // Send text message
To receive or consume messages, add WebSocket and WebSocketMessageReader objects:
wsFactory = WebSocketFactory.createWebSocketFactory();
ws = wsFactory.createWebSocket(URI.create("ws://example.com:8001/path"));
ws.connect(); // This will block or throw an exception if failed.
WebSocketMessageReader reader = ws.getMessageReader();
WebSocketMessageType type = null; // Block till a message arrives
// Loop till the connection goes away
while ((type = reader.next()) != WebSocketMessageType.EOS) {
switch (type) { // Handle both text and binary messages
case TEXT:
CharSequence text = reader.getText();
log("RECEIVED TEXT MESSAGE: " + text.toString());
break;
case BINARY:
ByteBuffer buffer = reader.getBinary();
log("RECEIVED BINARY MESSAGE: " + getHexDump(buffer));
break;
}
}
(Full Disclosure: I used to work at Kaazing Corporation as a server engineer.)

Vert.x has a java websocket client:
VertxFactory.newVertx()
.createHttpClient()
.setHost("localhost")
.setPort(8080)
.connectWebsocket("/ws", new Handler<WebSocket>() {
#Override
public void handle(final WebSocket webSocket) {
// Listen
webSocket.dataHandler(new Handler<Buffer>() {
#Override
public void handle(Buffer buff) {
log.info("Received {}", buff.toString());
}
});
// Publish
webSocket.writeTextFrame("Heya");
}
});

Netty is a good choice for such task, it's a high performance network application framework and it supports SSL elegantly, here is netty websocket client example from netty github:
public final class WebSocketClient {
static final String URL = System.getProperty("url", "ws://127.0.0.1:8080/websocket");
public static void main(String[] args) throws Exception {
URI uri = new URI(URL);
String scheme = uri.getScheme() == null? "ws" : uri.getScheme();
final String host = uri.getHost() == null? "127.0.0.1" : uri.getHost();
final int port;
if (uri.getPort() == -1) {
if ("ws".equalsIgnoreCase(scheme)) {
port = 80;
} else if ("wss".equalsIgnoreCase(scheme)) {
port = 443;
} else {
port = -1;
}
} else {
port = uri.getPort();
}
if (!"ws".equalsIgnoreCase(scheme) && !"wss".equalsIgnoreCase(scheme)) {
System.err.println("Only WS(S) is supported.");
return;
}
final boolean ssl = "wss".equalsIgnoreCase(scheme);
final SslContext sslCtx;
if (ssl) {
sslCtx = SslContextBuilder.forClient()
.trustManager(InsecureTrustManagerFactory.INSTANCE).build();
} else {
sslCtx = null;
}
EventLoopGroup group = new NioEventLoopGroup();
try {
// Connect with V13 (RFC 6455 aka HyBi-17). You can change it to V08 or V00.
// If you change it to V00, ping is not supported and remember to change
// HttpResponseDecoder to WebSocketHttpResponseDecoder in the pipeline.
final WebSocketClientHandler handler =
new WebSocketClientHandler(
WebSocketClientHandshakerFactory.newHandshaker(
uri, WebSocketVersion.V13, null, true, new DefaultHttpHeaders()));
Bootstrap b = new Bootstrap();
b.group(group)
.channel(NioSocketChannel.class)
.handler(new ChannelInitializer<SocketChannel>() {
#Override
protected void initChannel(SocketChannel ch) {
ChannelPipeline p = ch.pipeline();
if (sslCtx != null) {
p.addLast(sslCtx.newHandler(ch.alloc(), host, port));
}
p.addLast(
new HttpClientCodec(),
new HttpObjectAggregator(8192),
WebSocketClientCompressionHandler.INSTANCE,
handler);
}
});
Channel ch = b.connect(uri.getHost(), port).sync().channel();
handler.handshakeFuture().sync();
BufferedReader console = new BufferedReader(new InputStreamReader(System.in));
while (true) {
String msg = console.readLine();
if (msg == null) {
break;
} else if ("bye".equals(msg.toLowerCase())) {
ch.writeAndFlush(new CloseWebSocketFrame());
ch.closeFuture().sync();
break;
} else if ("ping".equals(msg.toLowerCase())) {
WebSocketFrame frame = new PingWebSocketFrame(Unpooled.wrappedBuffer(new byte[] { 8, 1, 8, 1 }));
ch.writeAndFlush(frame);
} else {
WebSocketFrame frame = new TextWebSocketFrame(msg);
ch.writeAndFlush(frame);
}
}
} finally {
group.shutdownGracefully();
}
}
}
public class WebSocketClientHandler extends SimpleChannelInboundHandler<Object> {
private final WebSocketClientHandshaker handshaker;
private ChannelPromise handshakeFuture;
public WebSocketClientHandler(WebSocketClientHandshaker handshaker) {
this.handshaker = handshaker;
}
public ChannelFuture handshakeFuture() {
return handshakeFuture;
}
#Override
public void handlerAdded(ChannelHandlerContext ctx) {
handshakeFuture = ctx.newPromise();
}
#Override
public void channelActive(ChannelHandlerContext ctx) {
handshaker.handshake(ctx.channel());
}
#Override
public void channelInactive(ChannelHandlerContext ctx) {
System.out.println("WebSocket Client disconnected!");
}
#Override
public void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception {
Channel ch = ctx.channel();
if (!handshaker.isHandshakeComplete()) {
handshaker.finishHandshake(ch, (FullHttpResponse) msg);
System.out.println("WebSocket Client connected!");
handshakeFuture.setSuccess();
return;
}
if (msg instanceof FullHttpResponse) {
FullHttpResponse response = (FullHttpResponse) msg;
throw new IllegalStateException(
"Unexpected FullHttpResponse (getStatus=" + response.status() +
", content=" + response.content().toString(CharsetUtil.UTF_8) + ')');
}
WebSocketFrame frame = (WebSocketFrame) msg;
if (frame instanceof TextWebSocketFrame) {
TextWebSocketFrame textFrame = (TextWebSocketFrame) frame;
System.out.println("WebSocket Client received message: " + textFrame.text());
} else if (frame instanceof PongWebSocketFrame) {
System.out.println("WebSocket Client received pong");
} else if (frame instanceof CloseWebSocketFrame) {
System.out.println("WebSocket Client received closing");
ch.close();
}
}
#Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
cause.printStackTrace();
if (!handshakeFuture.isDone()) {
handshakeFuture.setFailure(cause);
}
ctx.close();
}
}