We Keep Coding

RabbitMQ doesn't choose a right consumer

I took the example from here http://www.rabbitmq.com/tutorials/tutorial-six-java.html, added one more RPC call from RPCClient and added some logging into stdout. As a result, when the second call is executed, rabbitmq uses the consumer with wrong correlation id which is not expected behavior. Is it a bug or am I getting anything wrong?
RPCServer:
package com.foo.rabbitmq;
import com.rabbitmq.client.ConnectionFactory;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Consumer;
import com.rabbitmq.client.DefaultConsumer;
import com.rabbitmq.client.AMQP;
import com.rabbitmq.client.Envelope;
import java.io.IOException;
import java.util.concurrent.TimeoutException;
public class RPCServer {
private static final String RPC_QUEUE_NAME = "sap-consume";
private static int fib(int n) {
if (n ==0) return 0;
if (n == 1) return 1;
return fib(n-1) + fib(n-2);
}
public static void main(String[] argv) {
ConnectionFactory factory = new ConnectionFactory();
factory.setHost("localhost");
factory.setPort(5672);
Connection connection = null;
try {
connection = factory.newConnection();
final Channel channel = connection.createChannel();
channel.queueDeclare(RPC_QUEUE_NAME, false, false, false, null);
channel.basicQos(1);
System.out.println(" [x] Awaiting RPC requests");
Consumer consumer = new DefaultConsumer(channel) {
#Override
public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body) throws IOException {
AMQP.BasicProperties replyProps = new AMQP.BasicProperties
.Builder()
.correlationId(properties.getCorrelationId())
.build();
String response = "";
try {
String message = new String(body,"UTF-8");
int n = Integer.parseInt(message);
System.out.println(" [.] fib(" + message + ")");
response += fib(n);
}
catch (RuntimeException e){
System.out.println(" [.] " + e.toString());
}
finally {
channel.basicPublish( "", properties.getReplyTo(), replyProps, response.getBytes("UTF-8"));
channel.basicAck(envelope.getDeliveryTag(), false);
// RabbitMq consumer worker thread notifies the RPC server owner thread
synchronized(this) {
this.notify();
}
}
}
};
channel.basicConsume(RPC_QUEUE_NAME, false, consumer);
// Wait and be prepared to consume the message from RPC client.
while (true) {
synchronized(consumer) {
try {
consumer.wait();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
} catch (IOException | TimeoutException e) {
e.printStackTrace();
}
finally {
if (connection != null)
try {
connection.close();
} catch (IOException _ignore) {}
}
}
}
RPCCLient:
package com.bar.rabbitmq;
import com.rabbitmq.client.ConnectionFactory;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.DefaultConsumer;
import com.rabbitmq.client.AMQP;
import com.rabbitmq.client.Envelope;
import java.io.IOException;
import java.util.UUID;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeoutException;
public class RPCClient {
private Connection connection;
private Channel channel;
private String requestQueueName = "sap-consume";
private String replyQueueName;
public RPCClient() throws IOException, TimeoutException {
ConnectionFactory factory = new ConnectionFactory();
factory.setHost("localhost");
factory.setPort(5672);
connection = factory.newConnection();
channel = connection.createChannel();
replyQueueName = channel.queueDeclare().getQueue();
}
public String call(String message) throws IOException, InterruptedException {
final String corrId = UUID.randomUUID().toString();
AMQP.BasicProperties props = new AMQP.BasicProperties
.Builder()
.correlationId(corrId)
.replyTo(replyQueueName)
.build();
channel.basicPublish("", requestQueueName, props, message.getBytes("UTF-8"));
final BlockingQueue<String> response = new ArrayBlockingQueue<String>(1);
channel.basicConsume(replyQueueName, true, new DefaultConsumer(channel) {
#Override
public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body) throws IOException {
if (properties.getCorrelationId().equals(corrId)) {
System.out.println("Correlation Id" + properties.getCorrelationId() + " corresponds to expected one.");
response.offer(new String(body, "UTF-8"));
} else {
System.out.println("Correlation Id" + properties.getCorrelationId() + " doesn't correspond to expected one " + corrId);
}
}
});
return response.take();
}
public void close() throws IOException {
connection.close();
}
public static void main(String[] argv) {
RPCClient rpc = null;
String response = null;
try {
rpc = new RPCClient();
System.out.println(" [x] Requesting fib(30)");
response = rpc.call("30");
System.out.println(" [.] Got '" + response + "'");
System.out.println(" [x] Requesting fib(40)");
response = rpc.call("40");
System.out.println(" [.] Got '" + response + "'");
} catch (IOException | TimeoutException | InterruptedException e) {
e.printStackTrace();
} finally {
if (rpc != null) {
try {
rpc.close();
} catch (IOException _ignore) {
}
}
}
}
}

Yes you found a bug in the tutorial code. I have opened a pull request to fix it here and you can find the explanation of what's happening as well:
https://github.com/rabbitmq/rabbitmq-tutorials/pull/174
NOTE: the RabbitMQ team monitors the rabbitmq-users mailing list and only sometimes answers questions on StackOverflow.

This example is simplistic: it uses one queue for the reply. By sending a second request, you register a new consumer to the reply, but the consumer of the first request is still listening and actually steals the response of the second request. That's why the client seems to use the same correlation ID.
We updated the client code to use an exclusive, auto-delete queue for each request. This queue will be auto-deleted by the server because its only consumer is unsubscribed after the response has been received. This is a bit more involved but closer to a real-world scenario.
Note the best way to deal with the reply queue with RabbitMQ is to use direct reply-to. This uses pseudo-queues which are lighter than real queues. We don't mention direct reply-to in the tutorial to keep it as simple as possible, but this is the preferred feature to use in production.

Setting up a Client socket and ServerSocket listener (Java)

I'm trying to set up a peer to peer connection in java.
I'm trying to set up my program to listen for an incoming connection while outwardly being able to connect to a different client.
How can I instantiate my socket connection: socketConnection as whatever is connected to the program. Ideally like so:
if(socketConnection.isConnectedToExternalPeer()){
//do stuff
} else if (socketConnection.hasAnIncomingConnection()){
//do stuff
}
After consulting #L.Spillner 's solution I've put together the following code below, this only issue is that I can't quite grasp how to go about accepting a connection, this is evident from the fact that when I try to set up streams the program ends up in a loop while waiting for the peer's reply:
public class Client implements AutoCloseable {
// Any other ThreadPool can be used as well
private ExecutorService cachedExecutor = null;
private ExecutorService singleThreadExecutor = null;
// port this client shall listen on
private int port = 0;
// Name of the client
private String name = null;
// indicates that a connection is ongoing
private boolean isConnected = false;
// the socket the Client is currently connected with
private Socket activeConenctionSocket = null;
// The ServerSocket which will be listening for any incoming connection
private ServerSocket listener = null;
// The socket which has been accepted by the ServerSocket
private Future<Socket> acceptedSocket;
private ObjectInputStream inputStream = null;
private ObjectOutputStream outputStream = null;
private BloomChain bloomChain = null;
/**
* #param port Port number by which this client shall be accessed.
* #param name The name of this Client.
*/
public Client( int port, String name )
{
this.port = port;
this.name = name;
this.bloomChain = new BloomChain();
this.cachedExecutor = Executors.newCachedThreadPool();
this.singleThreadExecutor = Executors.newSingleThreadExecutor();
this.listener = createListeningSocket();
startListening();
}
private ServerSocket createListeningSocket()
{
ServerSocket temp = null;
try
{
temp = new ServerSocket( this.port );
}
catch ( IOException e )
{
e.printStackTrace();
}
return temp;
}
private void startListening()
{
if ( !this.isConnected )
{
this.listener = createListeningSocket();
this.acceptedSocket = this.cachedExecutor.submit( new ServAccept( this.listener ) );
}
}
/**
* Attempts to connect to any other socket specified by the hostname and the targetport.
*
* #param host The hostname of the target to connect.
* #param targetport The port of the target.
*/
public void connect( String host, int targetport )
{
try
{ System.out.println(host);
System.out.println(targetport);
this.activeConenctionSocket = new Socket( InetAddress.getByName( host ), targetport );
setUpStreams(this.activeConenctionSocket);
this.isConnected = true;
System.out.println(InetAddress.getAllByName(host));
}
catch ( IOException e )
{
e.printStackTrace();
}
try
{
this.listener.close();
}
catch ( IOException e )
{
// this will almost certainly throw an exception but it is intended.
}
}
public void setUpStreams(Socket socket) throws IOException {
this.outputStream = new ObjectOutputStream(socket.getOutputStream());
this.outputStream.flush();
this.inputStream = new ObjectInputStream(socket.getInputStream());
}
#Override
public void close() throws Exception
{
// close logic (can be rather nasty)
}
public void sendMessage(String message){
if(bloomChain.size()<1){
bloomChain.addBlock(new Block(message, "0"));
} else {
bloomChain.addBlock(new Block(message, bloomChain.get(bloomChain.size()-1).getPreviousHash()));
}
try {
this.outputStream.writeObject(bloomChain);
this.outputStream.flush();
} catch (IOException e) {
e.printStackTrace();
}
}
public String mineMessage(){
final String[] receivedMessage = {null};
final Block tempBlock = this.bloomChain.get(this.bloomChain.size()-1);
this.singleThreadExecutor.submit(()->{
tempBlock.mineBlock(bloomChain.getDifficulty());
receivedMessage[0] = tempBlock.getData();
});
return receivedMessage[0];
}
public String dataListener(){
if(isConnected) {
try {
BloomChain tempChain = (BloomChain) this.inputStream.readObject();
if (tempChain.isChainValid()) {
this.bloomChain = tempChain;
return mineMessage();
}
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
}
return null;
}
public ServerSocket getListener() {
return this.listener;
}
public boolean isConnected(){
return isConnected;
}
public ObjectOutputStream getOutputStream(){
return this.outputStream;
}
public ObjectInputStream getInputStream(){
return this.inputStream;
}
}
EDIT 2:
I tried to await for acceptedSocket.get() to return a socket in a separate thread as follows:
new Thread(()->{
setupStreams(this.acceptedSocket.get());
//try-catch blocks omitted
}).start();
This successfully wait for acceptedSocket to return a connected socket however when I try to connect to another locally running client i get the following error: java.net.SocketException: socket closed

Okay after some tinkering I finally figured out a neat little solution:
We want to be able to listen and connect at the same time so we need a ServerSocket and issue an ServerSocket#accept call to accept incoming cnnections.
However this method is blocking the thread so in order to being able to proceed with our programm we have to outsource this call into another thread and luckly the default Java API does provide a simple way to do so.
The following codesample is not finished but provides the core functionality:
Client.java:
public class Client
implements AutoCloseable
{
// Any other ThreadPool can be used as well
private ExecutorService es = Executors.newCachedThreadPool();
// port this client shall listen on
private int port;
// Name of the client
private String name;
// indicates that a connection is ongoing
private boolean isConnected = false;
// the socket the Client is currently connected with
private Socket activeConenctionSocket;
// The ServerSocket which will be listening for any incoming connection
private ServerSocket listener;
// The socket which has been accepted by the ServerSocket
private Future<Socket> acceptedSocket;
/**
* #param port Port number by which this client shall be accessed.
* #param name The name of this Client.
*/
public Client( int port, String name )
{
this.port = port;
this.name = name;
this.listener = createListeningSocket();
startListening();
}
private ServerSocket createListeningSocket()
{
ServerSocket temp = null;
try
{
temp = new ServerSocket( port );
}
catch ( IOException e )
{
e.printStackTrace();
}
return temp;
}
private void startListening()
{
if ( !isConnected )
{
listener = createListeningSocket();
acceptedSocket = es.submit( new ServAccept( listener ) );
}
}
/**
* Attempts to connect to any other socket specified by the hostname and the targetport.
*
* #param host The hostname of the target to connect.
* #param targetport The port of the target.
*/
public void connect( String host, int targetport )
{
isConnected = true;
try
{
activeConenctionSocket = new Socket( InetAddress.getByName( host ), targetport );
}
catch ( IOException e )
{
e.printStackTrace();
}
try
{
listener.close();
}
catch ( IOException e )
{
// this will almost certainly throw an exception but it is intended.
}
}
#Override
public void close() throws Exception
{
// close logic (can be rather nasty)
}
}
Let's walk through there step by step on how we instantiate a new Client object:
When we instantiate our object we create a new ServerSocket
We start listenting by creating a new Thread of a Callable<V> Object which I've named ServAccept for example purposes.
Now we have a Future<T> object which will contain a socket if any connection gets accepted.
A positive side effect of the startListening() method is, that you can make it public and call it once more if the connection has dropped.
The conenct(...) method almost works the same way as your setupConnection() method but with a small twist. The ServerSocket, which is still listening in another thread, will be close. The reason for this is, that there is no other way to exit the accept() method the other thread is stuck in.
The last thing (which you have to figure out) is when to check if the Future object is already done.
ServAccept.java
public class ServAccept
implements Callable<Socket>
{
ServerSocket serv;
public ServAccept( ServerSocket sock )
{
this.serv = sock;
}
#Override
public Socket call() throws Exception
{
return serv.accept();
}
}
EDIT:
As a matter of fact I have to admit that my approach might not be a very well rounded approach for the task so I decided to change tweak some things. This time instead of using a Future Object I decided to go with Events / a custom EventListener which is just sitting there and listening for a connection to receive. I tested the connection functionality and it works just fine but I haven't implemented a solution to determine if a Client really conncted to a peer. I just made sure that a client can only hold one connection at a time.
The changes:
ServerAccept.java
import java.io.IOException;
import java.net.ServerSocket;
public class ServAccept implements Runnable
{
private ServerSocket serv;
private ConnectionReceivedListener listener;
public ServAccept( ServerSocket sock,ConnectionReceivedListener con )
{
this.serv = sock;
this.listener = con;
}
#Override
public void run()
{
try
{
listener.onConnectionReceived( new ConnectionReceivedEvent( serv.accept() ) );
} catch (IOException e)
{
// planned exception here.
}
}
}
Does no longer implement Callable<V> but Runnable the only reason for that change is that we do not longer await any return since we will work with a listener and some juicy events. Anyway in order to do so we need to create and pass a listener to this object. But first we should take a look at the listener / event structure:
ConnectionReceivedListener.java
import java.util.EventListener;
#FunctionalInterface
public interface ConnectionReceivedListener extends EventListener
{
public void onConnectionReceived(ConnectionReceivedEvent event);
}
Just a simple interface from what we build some anonymous classes or lambda expressions. Nothing to fancy. It doen't even need to extend the EventListener interface but I love to do that to remind me what the purpose of the class is.
ConnectionReceivedEvent.java
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;
public class ConnectionReceivedEvent
{
private Socket accepted;
public ConnectionReceivedEvent( Socket sock )
{
this.accepted = sock;
}
public Socket getSocket()
{
return accepted;
}
public OutputStream getOutput() throws IOException
{
return accepted.getOutputStream();
}
public InputStream getInput() throws IOException
{
return accepted.getInputStream();
}
public int getPort()
{
return accepted.getPort();
}
}
Nothing to fancy as well, just passing a Socket as a constructor parameter and defining some getters from which most will not be used in this example.
But how to we use it now?
private void startListening()
{
if (!isConnected)
{
closeIfNotNull();
listener = createListeningSocket();
es.execute( new ServAccept( listener, event -> setAccepted( event.getSocket() ) ) );
}
}
private void setAccepted( Socket socket )
{
if (!isConnected)
{
this.activeConenctionSocket = socket;
setUpStreams( socket );
} else
{
sendError( socket );
}
}
We still make use of our ExecutorService and creating a new Thread with the ServAccept class. However since we do not expect any return I changed from ExecutorService#submit to ExecutorService#execute (just a matter of opinion and taste).
But ServAccept needs two arguments now. The ServerSocket and the Listener to use. Fortunately we can use annonymous classes and since our Listener does only feature one method we can even use a lambda expression. event -> setAccepted(event.getSocket()).
As an answer to your 2nd edit: I did a logical mistake. Not the ServerSocket#close method does throw the exception whe interrupting a ServerSocket#accept call but rather the accept() call itself throws the exception. In other words the exception you got was intended and i suppressed another one by mistake.

ActiveMQ Transport: tcp: Thread RUNNABLE state - too many threads hanging

Below ActiveMQ implementation is present in code. Sometimes, system stops working and become very slow. When I checked thread dump using JavaMelody - I have seen too many threads are on Runnable state for long time and is not being terminated.
ActiveMQ version - activemq-all-5.3.0.jar
Please refer below code :
Broker :
public class ActiveMQ extends HttpServlet {
private static final long serialVersionUID = -1234568008764323456;
private static final Logger logger = Logger.getLogger(ActiveMQ.class.getName());
public Listener listener;
private String msgBrokerUrl = "tcp://localhost:61602";
public BrokerService broker = null;
public TransportConnector connector = null;
#Override
public void init() throws ServletException {
try {
broker = new BrokerService();
broker.setPersistent(false);
broker.setUseJmx(false);
connector = broker.addConnector(msgBrokerUrl);
broker.setUseShutdownHook(true);
System.out.println("BROKER LOADED");
broker.start();
broker.deleteAllMessages();
listener = new Listener();
} catch (Exception e) {
e.printStackTrace();
}
}
}
Listener:
public class Listener implements MessageListener {
private String msgQueueName = "jms/queue/MessageQueue";
public Session session;
public Destination adminQueue;
public static String id;
public ActiveMQConnection connection;
public MessageConsumer consumer = null;
public Listener() {
try {
ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory(
new URI("failover://(" + "tcp://localhost:61602" + "?wireFormat.cacheEnabled=false"
+ "&wireFormat.maxInactivityDuration=0&wireFormat.tightEncodingEnabled=true)?maxReconnectDelay=1000"));
connection = (ActiveMQConnection) connectionFactory.createConnection();
connection.start();
session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
adminQueue = session.createQueue(msgQueueName);
id = new Timestamp(new Date().getTime()).toString();
consumer = this.session.createConsumer(this.adminQueue, "ID='" + id + "'");
consumer.setMessageListener(this);
} catch (JMSException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
}
#SuppressWarnings("unchecked")
#Override
public void onMessage(Message message) {
TextMessage msg = (TextMessage) message;
try {
String xmlMsg = msg.getText();
// business logic
} catch (JMSException ex) {
ex.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
}
Producer :
public class Producer {
private static String url = "tcp://localhost:61602";
private static String msgQueueName = "jms/queue/MessageQueue";
public ConnectionFactory connectionFactory = null;
public Connection connection = null;
public Session session = null;
public Destination destination = null;
public Producer() {
connectionFactory = new ActiveMQConnectionFactory(url);
}
public void sendResponse(String xml, DataBean objDataBean) {
sendToQueue(xml, msgQueueName, objDataBean);
}
private void sendToQueue(String xml, String msgQueueName, DataBean obj) {
try {
Connection connection = connectionFactory.createConnection();
connection.start();
Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
Destination destination = session.createQueue(msgQueueName);
MessageProducer producer = session.createProducer(destination);
TextMessage message = session.createTextMessage(xml);
message.setJMSExpiration(1000);
message.setStringProperty(obj.getMsgKey(), obj.getMsgValue());
producer.send(message);
xml = null;
session.close();
connection.close();
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
for (int msg = 0; msg < 20; msg++) {
DataBean obj = getData();
new Producer().sendResponse(xml, obj);
;
}
}
}
Hanging Threads Exception details :
Type 1 :
ActiveMQ Transport: tcp:///127.0.0.1:41818
java.net.SocketInputStream.socketRead0(Native Method)
java.net.SocketInputStream.read(SocketInputStream.java:152)
java.net.SocketInputStream.read(SocketInputStream.java:122)
org.apache.activemq.transport.tcp.TcpBufferedInputStream.fill(TcpBufferedInputStream.java:50)
org.apache.activemq.transport.tcp.TcpBufferedInputStream.read(TcpBufferedInputStream.java:58)
java.io.DataInputStream.readInt(DataInputStream.java:387)
org.apache.activemq.openwire.OpenWireFormat.unmarshal(OpenWireFormat.java:272)
org.apache.activemq.transport.tcp.TcpTransport.readCommand(TcpTransport.java:210)
org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:202)
org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:185)
java.lang.Thread.run(Thread.java:745)
Type 2 :
ActiveMQ Transport: tcp://localhost/127.0.0.1:61602
java.net.SocketInputStream.socketRead0(Native Method)
java.net.SocketInputStream.read(SocketInputStream.java:152)
java.net.SocketInputStream.read(SocketInputStream.java:122)
org.apache.activemq.transport.tcp.TcpBufferedInputStream.fill(TcpBufferedInputStream.java:50)
org.apache.activemq.transport.tcp.TcpBufferedInputStream.read(TcpBufferedInputStream.java:58)
java.io.DataInputStream.readInt(DataInputStream.java:387)
org.apache.activemq.openwire.OpenWireFormat.unmarshal(OpenWireFormat.java:272)
org.apache.activemq.transport.tcp.TcpTransport.readCommand(TcpTransport.java:210)
org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:202)
org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:185)
java.lang.Thread.run(Thread.java:745)
Please could you give some hints on this issue for further investigation.
Edit:
I read few posts on internet and concluded that I must update activemq jar file and implement timeout but when I started reading about timeout setting then I got confused whether I should set timeout in producer and consumer or failover or on message or broker service. Timeout at each component has different purpose then where I should implement timeout considering above code and exception.

Creating a connection is very expensive and when you close it, the port is retained for up to 3 minutes to ensure it is shutdown cleanly.
You want to create connections only when you really have to avoid performance problems. I suggest you create the connection once, and keep that connection open unless you get an error. This can improve performance by 2 to 3 orders of magnitude.
This is a good performance tuning pattern which applies in many cases;
only create and destroy expensive resources when you really have to.
operations you perform many times should be kept to a minimum. i.e do repeatedly as little as possible.

Netty Channel fail when write and flush too many and too fast

When I write a producer to publish message to my server. I've seen this:
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:192)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:384)
at io.netty.buffer.UnpooledUnsafeDirectByteBuf.setBytes(UnpooledUnsafeDirectByteBuf.java:447)
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:881)
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:242)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:119)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:511)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:468)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:382)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:354)
at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
at java.lang.Thread.run(Thread.java:745)
I've searched all around and was told that because of channel is closed.
But, in my code. I'm just close my channel when my channel pool destroy the channel.
Here my code:
public static class ChannelFactory implements PoolableObjectFactory<Channel> {
private final Bootstrap bootstrap;
private String host;
private int port;
public ChannelFactory(Bootstrap bootstrap, String host, int port) {
this.bootstrap = bootstrap;
this.host = host;
this.port = port;
}
#Override
public Channel makeObject() throws Exception {
System.out.println("Create new channel!!!");
bootstrap.validate();
return bootstrap.connect(host, port).channel();
}
#Override
public void destroyObject(Channel channel) throws Exception {
ChannelFuture close = channel.close();
if (close.isSuccess()) {
System.out.println(channel + " close successfully");
}
}
#Override
public boolean validateObject(Channel channel) {
System.out.println("Validate object");
return (channel.isOpen());
}
#Override
public void activateObject(Channel channel) throws Exception {
System.out.println(channel + " is activated");
}
#Override
public void passivateObject(Channel channel) throws Exception {
System.out.println(channel + " is passivated");
}
/**
* #return the host
*/
public String getHost() {
return host;
}
/**
* #param host the host to set
* #return
*/
public ChannelFactory setHost(String host) {
this.host = host;
return this;
}
/**
* #return the port
*/
public int getPort() {
return port;
}
/**
* #param port the port to set
* #return
*/
public ChannelFactory setPort(int port) {
this.port = port;
return this;
}
}
And here is my Runner:
public static class Runner implements Runnable {
private Channel channel;
private ButtyMessage message;
private MyChannelPool channelPool;
public Runner(MyChannelPool channelPool, Channel channel, ButtyMessage message) {
this.channel = channel;
this.message = message;
this.channelPool = channelPool;
}
#Override
public void run() {
channel.writeAndFlush(message.content()).syncUninterruptibly().addListener(new ChannelFutureListener() {
#Override
public void operationComplete(ChannelFuture future) throws Exception {
channelPool.returnObject(future.channel());
}
});
}
}
And my main:
public static void main(String[] args) throws InterruptedException {
final String host = "127.0.0.1";
final int port = 8080;
int jobSize = 100;
int jobNumber = 10000;
final Bootstrap b = func(host, port);
final MyChannelPool channelPool = new MyChannelPool(new ChannelFactory(b, host, port));
ExecutorService threadPool = Executors.newFixedThreadPool(1);
for (int i = 0; i < jobNumber; i++) {
try {
threadPool.execute(new Runner(channelPool, channelPool.borrowObject(), new ButtyMessage()));
} catch (Exception ex) {
System.out.println("ex = " + ex.getMessage());
}
}
}
With ButtyMessage extends ByteBufHolder.
In my Runner class, if I sleep(10) after writeAndFlush it run quite OK. But I don't want to reply on sleep. So I use ChannelFutureListener, but the result is bad. If I send about 1000 to 10.000 messages, it will crash and throw exception above. Is there any way to avoid this?
Thanks all.
Sorry for my bad explain and my English :)

You have several issues that could explain this. Most of them are related to wrong usage of asynchronous operations and future usage.
I don't know if it could be in link with your issue but, if you really want to print when the channel is really closed, you have to wait on the future, since the future on close() (or any other operations) immediately returns, without waiting for the real close. Therefore your test if (close.isSuccess()) shall be always false.
public void destroyObject(final Channel channel) throws Exception {
channel.close().addListener(new ChannelFutureListener() {
#Override
public void operationComplete(ChannelFuture close) {
if (close.isSuccess()) {
System.out.println(channel + " close successfully");
}
}
});
}
However, as I suppose it is only for debug purpose, it is not mandatory.
Another one: you send back to your pool a channel that is not already connected (which could explain your sleep(10) maybe?). You have to wait on the connect().
public Channel makeObject() throws Exception {
System.out.println("Create new channel!!!");
//bootstrap.validate(); // this is implicitely called in connect()
ChannelFuture future = bootstrap.connect(host, port).awaitUninterruptibly();
if (future.isSuccess()) {
return future.channel();
} else {
// do what you need to do when the connection is not done
}
}
third one: validation of a connected channel might be better using isActive():
#Override
public boolean validateObject(Channel channel) {
System.out.println("Validate object");
return channel.isActive(); // instead of isOpen()
}
fourth one: in your runner, you wrongly await on the future while you should not. You can remove your syncUninterruptibly() and let the rest as is.
#Override
public void run() {
Channel.writeAndFlush(message.content()).addListener(new ChannelFutureListener() {
#Override
public void operationComplete(ChannelFuture future) throws Exception {
channelPool.returnObject(future.channel());
}
});
}
And finally, I suppose you know your test is completely sequential (1 thread in your pool), such that each client will reuse over and over the very same channel?
Could you try to change the 4 points to see if it corrects your issue?
EDIT: after requester comment
For syncUntinterruptibly(), I did not read carefully. If you want to block on write, then you don't need the extra addListener since the future is done once the sync is over. So you can directly call your channelPool.returnObject as next command just after your sync.
So you should write it this way, simpler.
#Override
public void run() {
Channel.writeAndFlush(message.content()).syncUntinterruptibly();
channelPool.returnObject(future.channel());
}
For fireChannelActive, it will be called as soon as the connect finished (so from makeObject, sometime in the future). Moreover, once disconnected (as you did have notice in your exception), the channel is no more usable and must be recreated from zero. So I would suggest to use isActive however, such that, if not active, it will be removed using destroyObject...
Take a look at the channel state model here.

Finally, I've found a solution for myself. But, I'm still think about another solution. (this solution is exactly copy from 4.0.28 netty release note)
final String host = "127.0.0.1";
final int port = 8080;
int jobNumber = 100000;
final EventLoopGroup group = new NioEventLoopGroup(100);
ChannelPoolMap<InetSocketAddress, MyChannelPool> poolMap = new AbstractChannelPoolMap<InetSocketAddress, MyChannelPool>() {
#Override
protected MyChannelPool newPool(InetSocketAddress key) {
Bootstrap bootstrap = func(group, key.getHostName(), key.getPort());
return new MyChannelPool(bootstrap, new _AbstractChannelPoolHandler());
}
};
ChannelPoolMap<InetSocketAddress, FixedChannelPool> poolMap1 = new AbstractChannelPoolMap<InetSocketAddress, FixedChannelPool>() {
#Override
protected FixedChannelPool newPool(InetSocketAddress key) {
Bootstrap bootstrap = func(group, key.getHostName(), key.getPort());
return new FixedChannelPool(bootstrap, new _AbstractChannelPoolHandler(), 10);
}
};
final ChannelPool myChannelPool = poolMap.get(new InetSocketAddress(host, port));
final CountDownLatch latch = new CountDownLatch(jobNumber);
for (int i = 0; i < jobNumber; i++) {
final int counter = i;
final Future<Channel> future = myChannelPool.acquire();
future.addListener(new FutureListener<Channel>() {
#Override
public void operationComplete(Future<Channel> f) {
if (f.isSuccess()) {
Channel ch = f.getNow();
// Do somethings
ch.writeAndFlush(new ButtyMessage().content()).addListener(new ChannelFutureListener() {
#Override
public void operationComplete(ChannelFuture future) throws Exception {
if (future.isSuccess()) {
System.out.println("counter = " + counter);
System.out.println("future = " + future.channel());
latch.countDown();
}
}
});
// Release back to pool
myChannelPool.release(ch);
} else {
System.out.println(f.cause().getMessage());
f.cause().printStackTrace();
}
}
});
}
try {
latch.await();
System.exit(0);
} catch (InterruptedException ex) {
System.out.println("ex = " + ex.getMessage());
}
As you can see, I use SimpleChannelPool and FixedChannelPool (an implementation of SimpleChannelPool provided by netty).
What it can do:
SimpleChannelPool: open channels as much as it need ---> if you has 100.000 msg -> cuz error, of course. Many socket open, then IOExeption: Too many file open occur. (is that really pool? Create as much as possible and throw exception? I don't call this is pooling)
FixedChannelPool: not work in my case (Still study why? =)) Sorry for my stupidness)
Indeed, I want to use ObjectPool instead. And I may post it as soon as when I finish. Tks #Frederic Brégier for helping me so much!

Unit testing a Java multi-threaded network app

I am writing a Java multi-threaded network application and having real difficulty coming up with a way to unit test the object which sends and receives communication from network clients.
The object sends out a message to a number of clients and then waits for responses from the clients.
As each client responds, a dashboard-style GUI is updated.
In more detail...
A Message object represents a text message to be sent and contains an array of Clients which should receive the message.
The Message object is responsible for dispatching itself to all the appropriate clients.
When the dispatch() method is invoked on a Message object, the object spawns a new thread (MessageDispatcher) for each client in the Client array.
Each MessageDispatcher:
opens a new TCP socket (Socket) to the client
delivers the message to its client... PrintWriter out.println(msg text)
creates a 'Status' object which is passed to a Queue in the Message object and then on to the GUI.
Each Status object represents ONE of the following events:
Message passed to Socket (via Printwriter out.println() )
Display receipt received from client (via BufferedReader/InputStreamReader in.readline()... blocks until network input is received )
User acknowledge receipt received from client (via same method as above)
So.. I want to unit test the Message object. (using JUnit)
The unit test is called MessageTest.java (included below).
My first step has been to set up a Message object with a single recipient.
I then used JMockit to create a mock Socket object which can supply a mock OutputStream object (I am using ByteArrayOutputStream which extends OutputStream) to PrintWriter.
Then, when the MessageDispatcher calls (PrintWriter object).out, the message text will be ideally passed to my mock Socket object (via the mock OutputStream) which can check that the message text is OK.
And the sample principle for the InputStreamReader.... The mock Socket object also supplies a mock InputStreamReader object which supplies a mock BufferedReader which is called by the MessageDispatcher (as mentioned previously, MessageDispatcher blocks on in.readLine() ). At this point the mock BufferedReader should supply a fake confirmation to the MessageDispatcher...
// mock Socket
Mockit.redefineMethods(Socket.class, new Object()
{
ByteArrayOutputStream output = new ByteArrayOutputStream();
ByteArrayInputStream input = new ByteArrayInputStream();
public OutputStream getOutputStream()
{
return output;
}
public InputStream getInputStream()
{
return input;
}
});
If this wasn't multi-threaded, this should all work OK. However I have no idea how to do this with multiple threads. Can anyone give me any advice or tips?
Also if you have any input on the design (eg. Message object responsible for its own delivery rather than a separate delivery object.. "dependency injection"-style / separate thread for each client delivery) then I would be interested to hear that too.
UPDATE: here is the code:
Message.java
public class Message {
Client[] to;
String contents;
String status;
StatusListener listener;
BlockingQueue<Status> statusQ;
public Message(Client[] to, String contents, StatusListener listener)
{
this.to = to;
this.contents = contents;
this.listener = listener;
}
public void dispatch()
{
try {
// open a new thread for each client
// keep a linked list of socket references so that all threads can be closed
List<Socket> sockets = Collections.synchronizedList(new ArrayList<Socket>());
// initialise the statusQ for threads to report message status
statusQ = new ArrayBlockingQueue<Status>(to.length*3); // max 3 status objects per thread
// dispatch to each client individually and wait for confirmation
for (int i=0; i < to.length; i++) {
System.out.println("Started new thread");
(new Thread(new MessageDispatcher(to[i], contents, sockets, statusQ))).start();
}
// now, monitor queue and empty the queue as it fills up.. (consumer)
while (true) {
listener.updateStatus(statusQ.take());
}
}
catch (Exception e) { e.printStackTrace(); }
}
// one MessageDispatcher per client
private class MessageDispatcher implements Runnable
{
private Client client;
private String contents;
private List<Socket> sockets;
private BlockingQueue<Status> statusQ;
public MessageDispatcher(Client client, String contents, List<Socket> sockets, BlockingQueue<Status> statusQ) {
this.contents = contents;
this.client = client;
this.sockets = sockets;
this.statusQ = statusQ;
}
public void run() {
try {
// open socket to client
Socket sk = new Socket(client.getAddress(), CLIENTPORT);
// add reference to socket to list
synchronized(sockets) {
sockets.add(sk);
}
PrintWriter out = new PrintWriter(sk.getOutputStream(), true);
BufferedReader in = new BufferedReader(new InputStreamReader(sk.getInputStream()));
// send message
out.println(contents);
// confirm dispatch
statusQ.add(new Status(client, "DISPATCHED"));
// wait for display receipt
in.readLine();
statusQ.add(new Status(client, "DISPLAYED"));
// wait for read receipt
in.readLine();
statusQ.add(new Status(client, "READ"));
}
catch (Exception e) { e.printStackTrace(); }
}
}
}
.... and the corresponding unit test:
MessageTest.java
public class MessageTest extends TestCase {
Message msg;
static final String testContents = "hello there";
public void setUp() {
// mock Socket
Mockit.redefineMethods(Socket.class, new Object()
{
ByteArrayOutputStream output = new ByteArrayOutputStream();
ByteArrayInputStream input = new ByteArrayInputStream();
public OutputStream getOutputStream()
{
return output;
}
public InputStream getInputStream()
{
return input;
}
});
// NB
// some code removed here for simplicity
// which uses JMockit to overrides the Client object and give it a fake hostname and address
Client[] testClient = { new Client() };
msg = new Message(testClient, testContents, this);
}
public void tearDown() {
}
public void testDispatch() {
// dispatch to client
msg.dispatch();
}
}

Notice that the sending of multiple messages (multicast) can be achieved in a single blocking method through the NIO API (java.nio) as well, without the creation of new threads. NIO is quite complex, though.
I would start by writing the tests first, with a test-defined StatusListener implementation which stores all update events in a list. When the dispatch() method returns, the test can execute asserts on the state of the event list.
Using threads or NIO is an implementation detail for the Message class. So, unless you don't mind coupling the tests to this implementation detail, I would recommend introducing a helper class that would be responsible for sending multiple asynchronous messages and notifying the Message object upon any asynchronous replies. Then, you can mock the helper class in the unit tests, without coupling them to either threads or NIO.
I successfully implemented a test for the case of sending a message to one client. I also made some changes to the original production code, as follows:
public class Message
{
private static final int CLIENT_PORT = 8000;
// Externally provided:
private final Client[] to;
private final String contents;
private final StatusListener listener;
// Internal state:
private final List<Socket> clientConnections;
private final BlockingQueue<Status> statusQueue;
public Message(Client[] to, String contents, StatusListener listener)
{
this.to = to;
this.contents = contents;
this.listener = listener;
// Keep a list of socket references so that all threads can be closed:
clientConnections = Collections.synchronizedList(new ArrayList<Socket>());
// Initialise the statusQ for threads to report message status:
statusQueue = new ArrayBlockingQueue<Status>(to.length * 3);
}
public void dispatch()
{
// Dispatch to each client individually and wait for confirmation:
sendContentsToEachClientAsynchronously();
Status statusChangeReceived;
do {
try {
// Now, monitor queue and empty the queue as it fills up (consumer):
statusChangeReceived = statusQueue.take();
}
catch (InterruptedException ignore) {
break;
}
}
while (listener.updateStatus(statusChangeReceived));
closeRemainingClientConnections();
}
private void closeRemainingClientConnections()
{
for (Socket connection : clientConnections) {
try {
connection.close();
}
catch (IOException ignore) {
// OK
}
}
clientConnections.clear();
}
private void sendContentsToEachClientAsynchronously()
{
for (Client client : to) {
System.out.println("Started new thread");
new Thread(new MessageDispatcher(client)).start();
}
}
// One MessageDispatcher per client.
private final class MessageDispatcher implements Runnable
{
private final Client client;
MessageDispatcher(Client client) { this.client = client; }
public void run()
{
try {
communicateWithClient();
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
private void communicateWithClient() throws IOException
{
// Open connection to client:
Socket connection = new Socket(client.getAddress(), CLIENT_PORT);
try {
// Add client connection to synchronized list:
clientConnections.add(connection);
sendMessage(connection.getOutputStream());
readRequiredReceipts(connection.getInputStream());
}
finally {
connection.close();
}
}
// Send message and confirm dispatch.
private void sendMessage(OutputStream output)
{
PrintWriter out = new PrintWriter(output, true);
out.println(contents);
statusQueue.add(new Status(client, "DISPATCHED"));
}
private void readRequiredReceipts(InputStream input) throws IOException
{
BufferedReader in = new BufferedReader(new InputStreamReader(input));
// Wait for display receipt:
in.readLine();
statusQueue.add(new Status(client, "DISPLAYED"));
// Wait for read receipt:
in.readLine();
statusQueue.add(new Status(client, "READ"));
}
}
}
public final class MessageTest extends JMockitTest
{
static final String testContents = "hello there";
static final String[] expectedEvents = {"DISPATCHED", "DISPLAYED", "READ"};
#Test
public void testSendMessageToSingleClient()
{
final Client theClient = new Client("client1");
Client[] testClient = {theClient};
new MockUp<Socket>()
{
#Mock(invocations = 1)
void $init(String host, int port)
{
assertEquals(theClient.getAddress(), host);
assertTrue(port > 0);
}
#Mock(invocations = 1)
public OutputStream getOutputStream() { return new ByteArrayOutputStream(); }
#Mock(invocations = 1)
public InputStream getInputStream()
{
return new ByteArrayInputStream("reply1\nreply2\n".getBytes());
}
#Mock(minInvocations = 1) void close() {}
};
StatusListener listener = new MockUp<StatusListener>()
{
int eventIndex;
#Mock(invocations = 3)
boolean updateStatus(Status status)
{
assertSame(theClient, status.getClient());
assertEquals(expectedEvents[eventIndex++], status.getEvent());
return eventIndex < expectedEvents.length;
}
}.getMockInstance();
new Message(testClient, testContents, listener).dispatch();
}
}
The JMockit test above uses the new MockUp class, not yet available in the latest release. It can be replaced with Mockit.setUpMock(Socket.class, new Object() { ... }), though.

perhaps instead of redefining the methods getOutputStream and getInputStream, you can instead use an AbstractFactory in your Message class which creates output and input streams. In normal operation the factory will use a Socket to do that. However, for testing give it a factory which gives it streams of your choosing. That way you have more control over exactly what is happening.