We Keep Coding

Classic word count topology in Storm using 2 RabbitMQ queues

I have to write a simple "Word Count" Topology in Java and Storm. In particular, i have an external data source generating CSV (comma separated) string like
Daniel, 0.5654, 144543, user, 899898, Comment,,,
These strings are inserted into a RabbitMQ queue called "input". This datasource works well, and i can see the strings in the queue.
Now, i modified the classic topology adding the RabbitMQSpout. The goal is to do a word count for the first field of every CSV line, and publish results into a new queue called "output". The problem is that i cannot see any tuple inside the new queue, but the topology was submitted and RUNNING.
So, summing up:
external data source puts items into the input queue
RabbitMQSpout takes items from input queue and insert them into topology
classic word-count topology isperformed
last bolt puts results into output queue
Problem:
i can see items inside input queue, but nothing into output, even if i used same method to send item into the queue in the external data source (and it works) and in RabbitMQExporter (does not work...)
Some code below
RabbitMQSpout
public class RabbitMQSpout extends BaseRichSpout {
public static final String DATA = "data";
private SpoutOutputCollector _collector;
private RabbitMQManager rabbitMQManager;
#Override
public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
_collector = _collector;
rabbitMQManager = new RabbitMQManager("localhost", "rabbitmq", "rabbitmq", "test");
}
#Override
public void nextTuple() {
Utils.sleep(1000);
String data = rabbitMQManager.receive("input");
if (data != null) {
_collector.emit(new Values(data));
}
}
#Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields(DATA));
}
}
SplitBolt
public class SplitBolt extends BaseRichBolt {
private OutputCollector _collector;
public SplitSentenceBolt() { }
#Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this._collector = collector;
this.SPACE = Pattern.compile(",");
}
#Override
public void execute(Tuple input) {
String sentence = input.getStringByField(RabbitMQSpout.DATA);
String[] words = sentence.split(",");
if (words.length > 0)
_collector.emit(new Values(words[0]));
}
#Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word"));
}
#Override
public Map<String, Object> getComponentConfiguration() {
return null;
}
}
WordCountBolt
public class WordCountBolt extends BaseBasicBolt {
Map<String, Integer> counts = new HashMap<String, Integer>();
#Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
String word = tuple.getString(0);
Integer count = counts.get(word);
if (count == null)
count = 0;
count++;
counts.put(word, count);
System.out.println(count);
collector.emit(new Values(word, count));
}
#Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word", "count"));
}
}
RabbitMQExporterBolt
public RabbitMQExporterBolt(String rabbitMqHost, String rabbitMqUsername, String rabbitMqPassword,
String defaultQueue) {
super();
this.rabbitMqHost = rabbitMqHost;
this.rabbitMqUsername = rabbitMqUsername;
this.rabbitMqPassword = rabbitMqPassword;
this.defaultQueue = defaultQueue;
}
#Override
public void prepare(#SuppressWarnings("rawtypes") Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
this.collector=outputCollector;
this.rabbitmq = new RabbitMQManager(rabbitMqHost, rabbitMqUsername, rabbitMqPassword, defaultQueue);
}
#Override
public void execute(Tuple tuple) {
String word = tuple.getString(0);
Integer count = tuple.getInteger(1);
String output = word + " " + count;
rabbitmq.send(output);
}
#Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields("word"));
}
}
Topology
public class WordCountTopology {
private static final String RABBITMQ_HOST = "rabbitmq";
private static final String RABBITMQ_USER = "rabbitmq";
private static final String RABBITMQ_PASS = "rabbitmq";
private static final String RABBITMQ_QUEUE = "output";
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("spout", new RabbitMQSpout(), 1);
builder.setBolt("split", new SplitSentenceBolt(), 1)
.shuffleGrouping("spout");
builder.setBolt("count", new WordCountBolt(), 1)
.fieldsGrouping("split", new Fields("word"));
Config conf = new Config();
conf.setDebug(true);
if (args != null && args.length > 0) {
builder.setBolt("exporter",
new RabbitMQExporterBolt(
RABBITMQ_HOST, RABBITMQ_USER,
RABBITMQ_PASS, RABBITMQ_QUEUE ),
1)
.shuffleGrouping("count");
conf.setNumWorkers(3);
StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
} else {
conf.setMaxTaskParallelism(3);
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("word-count", conf, builder.createTopology());
Thread.sleep(10000);
cluster.shutdown();
}
}
}
RabbitMQManager
public class RabbitMQManager {
private String host;
private String username;
private String password;
private ConnectionFactory factory;
private Connection connection;
private String defaultQueue;
public RabbitMQManager(String host, String username, String password, String queue) {
super();
this.host = host;
this.username = username;
this.password = password;
this.factory = null;
this.connection = null;
this.defaultQueue = queue;
this.initialize();
this.initializeQueue(defaultQueue);
}
private void initializeQueue(String queue){
ConnectionFactory factory = new ConnectionFactory();
factory.setHost(host);
factory.setUsername(username);
factory.setPassword(password);
Connection connection;
try {
connection = factory.newConnection();
Channel channel = connection.createChannel();
boolean durable = false;
boolean exclusive = false;
boolean autoDelete = false;
channel.queueDeclare(queue, durable, exclusive, autoDelete, null);
channel.close();
connection.close();
} catch (IOException | TimeoutException e) {
e.printStackTrace();
}
}
private void initialize(){
factory = new ConnectionFactory();
factory.setHost(host);
factory.setUsername(username);
factory.setPassword(password);
try {
connection = factory.newConnection();
} catch (IOException | TimeoutException e) {
e.printStackTrace();
}
}
public void terminate(){
if (connection != null && connection.isOpen()){
try {
connection.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
private boolean reopenConnectionIfNeeded(){
try {
if (connection == null){
connection = factory.newConnection();
return true;
}
if (!connection.isOpen()){
connection = factory.newConnection();
}
} catch (IOException | TimeoutException e) {
e.printStackTrace();
return false;
}
return true;
}
public boolean send(String message){
return this.send(defaultQueue, message);
}
public boolean send(String queue, String message){
try {
reopenConnectionIfNeeded();
Channel channel = connection.createChannel();
channel.basicPublish("", queue, null, message.getBytes());
channel.close();
return true;
} catch (IOException | TimeoutException e) {
e.printStackTrace();
}
return false;
}
public String receive(String queue) {
try {
reopenConnectionIfNeeded();
Channel channel = connection.createChannel();
Consumer consumer = new DefaultConsumer(channel);
return channel.basicConsume(queue, true, consumer);
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
}

WorkStealingPool exits unexpectedly

I submitted some Runnables to an ExecutorService. Inside these Runnables, wait() and notify() are called. The code works with newFixedThreadPool as the ExecutorService. With newWorkStealingPool, the process exits unexpectedly without any error message.
import java.net.URL;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.IOException;
// For regular expressions
import java.util.regex.Matcher;
import java.util.regex.MatchResult;
import java.util.regex.Pattern;
import java.util.*;
import java.util.concurrent.*;
public class TestPipeline {
public static void main(String[] args) {
runAsThreads();
}
private static void runAsThreads() {
final BlockingQueue<String> urls = new OneItemQueue<String>();
final BlockingQueue<Webpage> pages = new OneItemQueue<Webpage>();
final BlockingQueue<Link> refPairs = new OneItemQueue<Link>();
final BlockingQueue<Link> uniqRefPairs = new OneItemQueue<Link>();
final ExecutorService executor = Executors.newWorkStealingPool(6);
// final ExecutorService executor = Executors.newFixedThreadPool(6);
executor.submit(new UrlProducer(urls));
executor.submit(new PageGetter(urls, pages));
executor.submit(new LinkScanner(pages,refPairs));
executor.submit(new Uniquifier<Link>(refPairs,uniqRefPairs));
executor.submit(new LinkPrinter(uniqRefPairs));
}
}
class UrlProducer implements Runnable {
private final BlockingQueue<String> output;
public UrlProducer(BlockingQueue<String> output) {
this.output = output;
}
public void run() {
System.out.println("in producer");
for (int i=0; i<urls.length; i++)
output.put(urls[i]);
}
private static final String[] urls =
{ "http://www.itu.dk", "http://www.di.ku.dk", "http://www.miele.de",
"http://www.microsoft.com", "http://www.cnn.com", "http://www.dr.dk",
"http://www.vg.no", "http://www.tv2.dk", "http://www.google.com",
"http://www.ing.dk", "http://www.dtu.dk", "http://www.bbc.co.uk"
};
}
class PageGetter implements Runnable {
private final BlockingQueue<String> input;
private final BlockingQueue<Webpage> output;
public PageGetter(BlockingQueue<String> input, BlockingQueue<Webpage> output) {
this.input = input;
this.output = output;
}
public void run() {
while (true) {
System.out.println("in pagegetter");
String url = input.take();
// System.out.println("PageGetter: " + url);
try {
String contents = getPage(url, 200);
output.put(new Webpage(url, contents));
} catch (IOException exn) { System.out.println(exn); }
}
}
public static String getPage(String url, int maxLines) throws IOException {
// This will close the streams after use (JLS 8 para 14.20.3):
try (BufferedReader in
= new BufferedReader(new InputStreamReader(new URL(url).openStream()))) {
StringBuilder sb = new StringBuilder();
for (int i=0; i<maxLines; i++) {
String inputLine = in.readLine();
if (inputLine == null)
break;
else
sb.append(inputLine).append("\n");
}
return sb.toString();
}
}
}
class Uniquifier<T> implements Runnable{
private final Set<T> set = new HashSet<T>();
private final BlockingQueue<T> input;
private final BlockingQueue<T> output;
public Uniquifier(BlockingQueue<T> input, BlockingQueue<T> output){
this.input = input;
this.output = output;
}
public void run(){
while(true){
System.out.println("in uniquifier");
T item = input.take();
if(!set.contains(item)){
set.add(item);
output.put(item);
}
}
}
}
class LinkScanner implements Runnable {
private final BlockingQueue<Webpage> input;
private final BlockingQueue<Link> output;
public LinkScanner(BlockingQueue<Webpage> input,
BlockingQueue<Link> output) {
this.input = input;
this.output = output;
}
private final static Pattern urlPattern
= Pattern.compile("a href=\"(\\p{Graph}*)\"");
public void run() {
while (true) {
System.out.println("in link scanner");
Webpage page = input.take();
// System.out.println("LinkScanner: " + page.url);
// Extract links from the page's <a href="..."> anchors
Matcher urlMatcher = urlPattern.matcher(page.contents);
while (urlMatcher.find()) {
String link = urlMatcher.group(1);
output.put(new Link(page.url, link));
}
}
}
}
class LinkPrinter implements Runnable {
private final BlockingQueue<Link> input;
public LinkPrinter(BlockingQueue<Link> input) {
this.input = input;
}
public void run() {
while (true) {
System.out.println("in link printer");
Link link = input.take();
// System.out.println("LinkPrinter: " + link.from);
System.out.printf("%s links to %s%n", link.from, link.to);
}
}
}
class Webpage {
public final String url, contents;
public Webpage(String url, String contents) {
this.url = url;
this.contents = contents;
}
}
class Link {
public final String from, to;
public Link(String from, String to) {
this.from = from;
this.to = to;
}
// Override hashCode and equals so can be used in HashSet<Link>
public int hashCode() {
return (from == null ? 0 : from.hashCode()) * 37
+ (to == null ? 0 : to.hashCode());
}
public boolean equals(Object obj) {
Link that = obj instanceof Link ? (Link)obj : null;
return that != null
&& (from == null ? that.from == null : from.equals(that.from))
&& (to == null ? that.to == null : to.equals(that.to));
}
}
// Different from java.util.concurrent.BlockingQueue: Allows null
// items, and methods do not throw InterruptedException.
interface BlockingQueue<T> {
void put(T item);
T take();
}
class OneItemQueue<T> implements BlockingQueue<T> {
private T item;
private boolean full = false;
public void put(T item) {
synchronized (this) {
while (full) {
try { this.wait(); }
catch (InterruptedException exn) { }
}
full = true;
this.item = item;
this.notifyAll();
}
}
public T take() {
synchronized (this) {
while (!full) {
try { this.wait(); }
catch (InterruptedException exn) { }
}
full = false;
this.notifyAll();
return item;
}
}
}

Because the Pool is allocating threads dynamically, there are no threads alive after runAsThreads exits because that's the end of the main thread. There needs to be at least on thread running to keep the application alive. Adding a call to awaitTermination is needed. It's not needed for the fixed pool because that will always have active threads until it is explicitly shut down as noted in the JavaDocs.

MVEL executeExpression function cannot be concurrent

Run the main function in File2 , the problem is : threads stuck at "rval=MVEL.executeExpression(compiledExpression, vars);" , 10 threads run in sequential order, not parallel , I wanna know why this happened.
PS: I'm using MVEL 2.2 , the latest version
File1:MVELHelper.java
public class MVELHelper {
private static ParserContext _ctx = new ParserContext(false);
//public static Object execute(String expression, Map<String, Object> vars, Databus databus) throws Exception {
public static Object execute(String expression, Map<String, Object> vars) throws Exception {
Object rval = null;
try {
if(vars == null) {
rval = MVEL.eval(expression, new HashMap<String,Object>());
}
else {
rval = MVEL.eval(expression, vars);
}
return rval;
}
catch(Exception e) {
throw new Exception("MVEL FAILED:"+expression,e);
}
}
public static Serializable compile(String text, ParserContext ctx)
throws Exception {
if(ctx == null) {
//ctx = _ctx;
ctx=new ParserContext(false);
}
Serializable exp = null;
try {
exp = MVEL.compileExpression(text, ctx);
//exp = MVEL.compileExpression(text);
}
catch (Exception e) {
throw new Exception("failed to compile expression.", e);
}
return exp;
}
public static Object compileAndExecute(String expression, Map<String, Object> vars) throws Exception {
Object rval = null;
try {
Serializable compiledExpression=compile(expression,null);
System.out.println("[COMPILE OVER, Thread Id="+Thread.currentThread().getId()+"] ");
if(vars == null) {
rval=MVEL.executeExpression(compiledExpression, new HashMap<String,Object>());
//rval = MVEL.eval(exp, new HashMap<String,Object>());
}
else {
//rval=MVEL.executeExpression(compiledExpression, vars,(VariableResolverFactory)null);
rval=MVEL.executeExpression(compiledExpression, vars);
//rval = MVEL.eval(expression, vars);
}
return rval;
}
catch(Exception e) {
throw new Exception("MVEL FAILED:"+expression,e);
}
}
}
File2:ExecThread3.java
public class ExecThread3 implements Runnable{
Map dataMap=null;
public Map getDataMap() {
return dataMap;
}
public void setDataMap(Map dataMap) {
this.dataMap = dataMap;
}
#Override
public void run() {
Map varsMap = new HashMap();
Map dataMap=new HashMap();
dataMap.put("count",100);
varsMap.put("dataMap", dataMap);
String expression="System.out.println(\"[BEFORE Thread Id=\"+Thread.currentThread().getId()+\"] \"+dataMap.get(\"count\"));"+
"Thread.sleep(3000);"+
"System.err.println(\"[AFTER Thread Id=\"+Thread.currentThread().getId()+\"] \"+dataMap.get(\"count\"));";
try {
//MVEL.compileExpression(expression);
MVELHelper.compileAndExecute(expression, varsMap);
}
catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static void main(String[] args) {
for(int k=0;k<10;k++){
ExecThread3 execThread=new ExecThread3();
new Thread(execThread).start();
}
}
}

why Threads get blocked when i use Apache-Commons-Pool?

Here is my demo:
PoolableObjectFactoryImpl.java
public class PoolableObjectFactoryImpl implements PoolableObjectFactory<Result> {
private static Logger logger = Logger.getLogger("BackgroundLog");
#Override
public void activateObject(Result obj) throws Exception {
logger.info("==activate result.==");
obj.setResult(-999);
}
#Override
public void destroyObject(Result obj) throws Exception {
logger.info("==destroy result.==");
obj = null;
}
#Override
public Result makeObject() throws Exception {
logger.info("==make result.==");
Result result = new Result();
return result;
}
#Override
public void passivateObject(Result obj) throws Exception {
logger.info("==passivate result.==");
obj.setResult(-999);
}
#Override
public boolean validateObject(Result obj) {
/*if(obj.getResult() == -999){
logger.info("==validate result true.==");
return true;
}else{
logger.info("==validate result false.==");
return false;
}*/
logger.info("==validate result true.==");
return true;
}
}
ThreadPool.java
public class ThreadPool extends GenericObjectPool {
private static Logger logger = Logger.getLogger("BackgroundLog");
private static ThreadPool pool = null;
private Map<String, String> map = getConfig();
private ThreadPool() {
this.setFactory(new PoolableObjectFactoryImpl());
this.setMaxActive(Integer.parseInt(map.get("maxActive")));
this.setWhenExhaustedAction(Byte.valueOf(map.get("whenExhaustedAction")));
this.setMaxWait(Long.parseLong(map.get("maxWait")));
this.setMaxIdle(Integer.parseInt(map.get("maxIdle")));
this.setTestOnBorrow(Boolean.valueOf(map.get("testOnBorrow")));
this.setTestOnReturn(Boolean.valueOf(map.get("testOnReturn")));
this.setTimeBetweenEvictionRunsMillis(Long.parseLong(map.get("timeBetweenEvictionRunsMillis")));
this.setNumTestsPerEvictionRun(Integer.parseInt(map.get("numTestsPerEvictionRun")));
this.setMinEvictableIdleTimeMillis(Long.parseLong(map.get("minEvictableIdleTimeMillis")));
this.setTestWhileIdle(Boolean.valueOf(map.get("testWhileIdle")));
}
public static ThreadPool getInstance() {
if (pool == null) {
synchronized (ThreadPool.class) {
if (pool == null) {
logger.info("thread pool is initialized.");
pool = new ThreadPool();
}
}
}
return pool;
}
/**
*
* <p>Title: getConfig</p>
* <p>Description: get pool configuration</p>
* #return
*/
public Map<String, String> getConfig() {
Map<String, String> map = new HashMap<String, String>();
Properties props = new Properties();
try {
InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream("pool.properties");
props.load(in);
Enumeration en = props.propertyNames();
while (en.hasMoreElements()) {
String key = (String) en.nextElement();
map.put(key, props.getProperty(key));
}
in.close();
} catch (Throwable t) {
logger.error(t.getMessage(), t);
}
return map;
}
}
Result.java
public class Result {
private int result;
public Result(){
}
public int getResult(){
return this.result;
}
public void setResult(int result){
this.result = result;
}
}
Test.java
public class Test implements Runnable {
private static Logger logger = Logger.getLogger("BackgroundLog");
private String name = null;
public Test(String name){
this.name = name;
}
public String getName(){
return this.name;
}
public void setName(String name){
this.name = name;
}
#Override
public void run() {
ThreadPool pool = ThreadPool.getInstance();
for(int i=0;i<1000;i++){
try {
Result result = (Result)pool.borrowObject();
logger.info("numActive: "+ pool.getNumActive()+"\t"+"numIdle: "+pool.getNumIdle());
logger.info("thread "+getName()+" "+i+" borrow object from pool "+result.getResult()+".");
result.setResult(0);
pool.returnObject(result);
logger.info("return object to pool.");
Thread.sleep(100);
} catch (Exception e) {
logger.info("thread "+getName()+" "+i);
e.printStackTrace();
}
}
}
public static void main(String[] args) {
for(int i=0;i<50;i++){
Thread t = new Thread(new Test("t"+i));
t.start();
}
}
}
Next is the configuration properties:
Next is the threads view from Jprofiler when it has 4 threads:
After Test.java is running a few minutes,some threads keep beling blocked,only one is still running but does not print any log.I don't really understand thread thing.
can anyone explain why? how to avoid threads being blocked?

Consider posting logs of an execution cycle.
Did you try commenting Thread.sleep line, because sleep will hold onto the lock it has acquired till the thread is in sleep mode.

Try replacing "Thread.sleep(100);" with:
try {
synchronized (this) {
this.wait(200);
}
} catch (InterruptedException e) {
}

StreamCorruptedException occurs during object decoding

I am writing a netty client and server application that will write JVM GC stats to a time-series database to analyse for around 300 servers. However my pipeline is throwing lots of these exceptions:
10/02/2012 10:14:23.415 New I/O server worker #2-2 ERROR GCEventsCollector - netty error
java.io.StreamCorruptedException: invalid type code: 6E
at java.io.ObjectInputStream.readObject0(Unknown Source)
at java.io.ObjectInputStream.defaultReadFields(Unknown Source)
at java.io.ObjectInputStream.readSerialData(Unknown Source)
at java.io.ObjectInputStream.readOrdinaryObject(Unknown Source)
at java.io.ObjectInputStream.readObject0(Unknown Source)
at java.io.ObjectInputStream.readObject(Unknown Source)
at org.jboss.netty.handler.codec.serialization.ObjectDecoder.decode(ObjectDecoder.java:94)
at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:282)
at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:216)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:274)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:261)
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:349)
at org.jboss.netty.channel.socket.nio.NioWorker.processSelectedKeys(NioWorker.java:280)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:200)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
It looks like it's creating an OutputStream but one already exists - so it throws that specific exception. It appears in my AIT environment where >300 servers are connecting but not in DEV where 1 agent is only connecting.
I suspect it is either a bug in the object decoder or I have incorrect code somewhere. Please could anyone explain why this happens?
Here is the collector:
public class GCEventsCollector extends AbstractDataCollector {
protected static final Logger logger = Logger.getLogger(GCEventsCollector.class);
private static final ExecutorService WORKERS = Executors.newCachedThreadPool();
private static final ChannelGroup GROUP = new DefaultChannelGroup("gc-events");
private final int port;
private final ServerBootstrap bootstrap;
public GCEventsCollector(final int port) {
logger.info("Creating GC Events collector on port " + port);
this.port = port;
this.bootstrap = newServerBootstrap(port);
}
/**
* Creates a bootstrap for creating bindings to sockets. Each channel has a pipeline, which contains the
* logic for handling a message such as encoding, decoding, buffering, etc.
*
* #param port port of socket
* #return configured bootstrap
*/
private ServerBootstrap newServerBootstrap(int port) {
ExecutorService bossExecutor = Executors.newCachedThreadPool();
ExecutorService workerExecutor = Executors.newCachedThreadPool();
NioServerSocketChannelFactory channelFactory =
new NioServerSocketChannelFactory(bossExecutor, workerExecutor);
ServerBootstrap bootstrap = new ServerBootstrap(channelFactory);
ChannelHandler collectorHandler = new CollectorHandler();
bootstrap.setPipelineFactory(new CollectorPipelineFactory(collectorHandler));
bootstrap.setOption("localAddress", new InetSocketAddress(port));
return bootstrap;
}
protected KDBCategory[] getKDBCategories() {
return new KDBCategory[] { KDBCategory.GC_EVENTS };
}
/**
* Bind to a socket to accept messages
*
* #throws Exception
*/
public void doStart() throws Exception {
Channel channel = bootstrap.bind();
GROUP.add(channel);
}
/**
* Disconnect the channel to stop accepting messages and wait until disconnected
*
* #throws Exception
*/
public void doStop() throws Exception {
logger.info("disconnecting");
GROUP.close().awaitUninterruptibly();
}
class CollectorHandler extends SimpleChannelHandler {
#Override
public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent e)
throws Exception {
super.channelOpen(ctx, e);
GROUP.add(e.getChannel());
}
#Override
public void channelConnected(ChannelHandlerContext ctx,
ChannelStateEvent e) throws Exception {
super.channelConnected(ctx, e);
logger.info("channel connected");
}
#Override
public void channelDisconnected(ChannelHandlerContext ctx,
ChannelStateEvent e) throws Exception {
super.channelDisconnected(ctx, e);
logger.info("channel disconnected");
}
#Override
public void messageReceived(ChannelHandlerContext ctx, final MessageEvent e) throws Exception {
if (logger.isDebugEnabled()) {
logger.debug("Received GcStats event: " + e.toString());
}
WORKERS.execute(new Runnable() {
public void run() {
saveData(KDBCategory.GC_EVENTS, (GcEventsPersister) e.getMessage());
}
});
}
#Override
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
logger.error("netty error", e.getCause());
}
}
private static class CollectorPipelineFactory implements ChannelPipelineFactory {
private final ChannelHandler handler;
private CollectorPipelineFactory(ChannelHandler handler) {
this.handler = handler;
}
#Override
public ChannelPipeline getPipeline() throws Exception {
return Channels.pipeline(new ObjectDecoder(), handler);
}
}
}
Here is the agent:
public class GCEventsAgent {
private final static Logger logger = Logger.getLogger(GCEventsAgent.class);
private static final ExecutorService bosses = Executors.newCachedThreadPool();
private static final ExecutorService workers = Executors.newCachedThreadPool();
private static final Timer timer = new HashedWheelTimer();
private static final String localHostName;
private static final ParseExceptionListener exceptionListener = new ExceptionListener();
static {
String name = "";
try {
name = InetAddress.getLocalHost().getHostName();
} catch (UnknownHostException e) {
logger.error("cannot retrieve local host name", e);
}
localHostName = name;
}
public static void main(final String[] args) {
checkArgument(args.length >= 3, "Usage: GCEventsAgent [log4j cfg] [mba cfg] [server cfg] [process 1] ... [process n]");
System.setProperty("log4j.configuration", "file:log4j.properties");
final String log4jConfig = args[0];
DOMConfigurator.configure(log4jConfig);
final String mbaConfig = args[1];
final String serverConfigPath = args[2];
final ServerConfig serverCfg = new ServerConfig(serverConfigPath);
setup(serverCfg, args);
}
private static void setup(ServerConfig cfg, String[] args) {
final String host = cfg.getParameter(String.class, "host");
final int port = cfg.getParameter(Integer.class, "port");
if (args.length == 3)
configurePolling(cfg, host, port);
else
configureProcesses(cfg, args, host, port);
}
private static void configureProcesses(final ServerConfig cfg,
final String[] args,
final String host,
final int port) {
final List<File> logFiles = logFiles(cfg, args);
logger.info("Initializing GC Agent for [" + host + ":" + port + "]");
final NioClientSocketChannelFactory channelFactory =
new NioClientSocketChannelFactory(bosses, workers);
final ClientBootstrap bootstrap = new ClientBootstrap(channelFactory);
bootstrap.setOption("remoteAddress", new InetSocketAddress(host, port));
final GCParserFactory parserFactory = new DefaultParserFactory();
final AgentProcessHandler agentHandler =
new AgentProcessHandler(bootstrap, logFiles, parserFactory);
bootstrap.setPipelineFactory(new ChannelPipelineFactory() {
#Override
public ChannelPipeline getPipeline() throws Exception {
return Channels.pipeline(new ObjectEncoder(), agentHandler);
}
});
bootstrap.connect().awaitUninterruptibly();
}
private static void configurePolling(ServerConfig cfg, String host, int port) {
final int frequency = cfg.getParameter(Integer.class, "frequency");
final NioClientSocketChannelFactory channelFactory =
new NioClientSocketChannelFactory(newCachedThreadPool(), newCachedThreadPool());
final ClientBootstrap bootstrap = new ClientBootstrap(channelFactory);
bootstrap.setOption("remoteAddress", new InetSocketAddress(host, port));
final GcParserSupplier parserSupplier = new GcParserSupplier();
final ConcurrentMap<File, Tailer> tailerMap = Maps.newConcurrentMap();
final ParseExceptionListener exceptionListener = new ExceptionListener();
final Set<File> discoveredSet = Sets.newHashSet();
final File directory = new File(cfg.getParameter(String.class, "logDirectory"));
final TailManager tailManager =
new TailManager(discoveredSet, tailerMap, parserSupplier, exceptionListener, localHostName);
final DetectionTask detectionTask = new DetectionTask(directory, discoveredSet, tailManager);
final FileWatcher fileWatcher =
new FileWatcher(Executors.newScheduledThreadPool(1), detectionTask, frequency);
final Timer timer = new HashedWheelTimer();
final EfxAgentHandler agentHandler =
new EfxAgentHandler(bootstrap, tailManager, fileWatcher, timer);
bootstrap.setPipelineFactory(new ChannelPipelineFactory() {
#Override public ChannelPipeline getPipeline() throws Exception {
return Channels.pipeline(new ObjectEncoder(), agentHandler);
}
});
bootstrap.connect().awaitUninterruptibly();
}
private static List<File> logFiles(ServerConfig cfg, String[] args) {
String logDir = cfg.getParameter(String.class, "logDirectory");
final int n = args.length;
List<File> logFiles = new ArrayList<File>(n-3);
for (int i = 3; i < n; i++) {
String filePath = logDir + args[i] + ".gc.log";
try {
File file = new File(filePath);
if (!file.exists()) {
logger.info("missing log file so creating empty file at path: " + filePath);
File dir = file.getParentFile();
dir.mkdirs();
if (file.createNewFile())
logger.info("successfully created empty file at path: " + filePath);
}
logFiles.add(file);
} catch (IOException e) {
logger.error("error creating log file at path: " + filePath);
}
}
return logFiles;
}
static final class AgentPauseListener implements GCEventListener<CMSType, CMSHeap> {
private final Channel channel;
private final GcEventsPersister.Builder builder;
private byte state;
private AgentPauseListener(Channel channel,
GcEventsPersister.Builder builder) {
this.channel = channel;
this.builder = builder;
}
#Override
public void onPause(PauseDetail<CMSType> pauseDetail) {
logger.info("onPause");
checkState(state == 0x00 || state == 0x01);
builder
.collectionType(pauseDetail.getType() == null
? null : pauseDetail.getType().toString())
.start(new Instant(pauseDetail.getStartTimestamp()))
.end(new Instant(pauseDetail.getEndTimestamp()))
.pause(pauseDetail.getType() == null
? false : pauseDetail.getType().isPause())
.duration(pauseDetail.getPauseMillis());
if (state == 0x00)
channel.write(builder.build());
else
state |= 0x02;
}
#Override
public void onHeapBefore(HeapDetail<CMSHeap> details) {
logger.info("onHeapBefore");
checkState(state == 0x00);
builder.heapBefore(used(details)).heapBeforeTotal(total(details));
state |= 0x01;
}
#Override
public void onHeapAfter(HeapDetail<CMSHeap> details) {
logger.info("onHeapAfter");
checkState(state == 0x03);
builder.heapAfter(used(details)).heapAfterTotal(total(details));
channel.write(builder.build());
state = 0x00;
}
private final long used(HeapDetail<CMSHeap> details) {
return used(details, CMSHeap.PAR_NEW_GENERATION)
+ used(details, CMSHeap.CMS_GENERATION)
+ used(details, CMSHeap.CMS_PERM_GENERATION);
}
private final long used(HeapDetail<CMSHeap> heapDetail,
CMSHeap gen) {
final Map<CMSHeap, HeapDetail.HeapMetric> sizes = heapDetail.getSizes();
final long used = sizes.get(gen).getUsed();
logger.info("used = " + used);
return used;
}
private final long total(HeapDetail<CMSHeap> details) {
return total(details, CMSHeap.PAR_NEW_GENERATION)
+ total(details, CMSHeap.CMS_GENERATION)
+ total(details, CMSHeap.CMS_PERM_GENERATION);
}
private final long total(HeapDetail<CMSHeap> heapDetail,
CMSHeap gen) {
final Map<CMSHeap, HeapDetail.HeapMetric> sizes = heapDetail.getSizes();
return sizes.get(gen).getTotal();
}
}
static final class ExceptionListener implements ParseExceptionListener {
#Override
public void onParseError(int lineNo, String input, String error) {
logger.error("error parsing: " + lineNo + " - " + input + " - " + error);
}
}
static final class ReconnectTask implements TimerTask {
private final ClientBootstrap bootstrap;
ReconnectTask(ClientBootstrap bootstrap) {
this.bootstrap = bootstrap;
}
#Override
public void run(Timeout timeout) throws Exception {
bootstrap.connect();
}
}
static class AgentProcessHandler extends SimpleChannelHandler {
private final ClientBootstrap bootstrap;
private final List<File> logFiles;
private final GCParserFactory parserFactory;
private final Set<Tailer> tailers = new HashSet<Tailer>(4);
public AgentProcessHandler(ClientBootstrap bootstrap,
List<File> logFiles,
GCParserFactory parserFactory) {
this.bootstrap = bootstrap;
this.logFiles = logFiles;
this.parserFactory = parserFactory;
}
#Override
public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e)
throws Exception {
logger.info("channel connected");
for (File logFile : logFiles) {
logger.info("setting up scraper for logfile: " + logFile);
GCParser parser = parserFactory.getParser();
GcEventsPersister.Builder builder =
new GcEventsPersister.Builder(logFile.getName(), localHostName);
GCEventListener gcEventListener =
new AgentPauseListener(e.getChannel(), builder);
TailerListener listener =
new LogLineListener(parser, gcEventListener, exceptionListener);
Tailer tailer = Tailer.create(logFile, listener, 1000L, true);
tailers.add(tailer);
}
}
#Override
public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e)
throws Exception {
logger.error("channel disconnected");
stopTailers();
scheduleReconnect();
}
private void scheduleReconnect() {
timer.newTimeout(new ReconnectTask(bootstrap), 5L, TimeUnit.SECONDS);
}
private final void stopTailers() {
for (Tailer tailer : tailers) {
tailer.stop();
}
tailers.clear();
}
#Override
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e)
throws Exception {
final Throwable cause = e.getCause();
logger.error(cause);
if (cause instanceof ConnectException) {
stopTailers();
scheduleReconnect();
}
}
};
private static class LogLineListener extends TailerListenerAdapter {
private final GCParser parser;
private final GCEventListener pauseListener;
private final ParseExceptionListener exceptionLister;
public LogLineListener(GCParser parser,
GCEventListener listener,
ParseExceptionListener exceptionLister) {
this.parser = parser;
this.pauseListener = listener;
this.exceptionLister = exceptionLister;
}
#Override
public void handle(String line) {
logger.info("handle(String line)");
parser.matchLine(line, pauseListener, exceptionLister);
}
}
private static final class GcParserSupplier
implements Supplier<GCParser<CMSType, CMSHeap>> {
#Override public GCParser<CMSType, CMSHeap> get() {
return new CMSParser();
}
}
private static final class TailManager implements FileHandler {
private final Set<File> discoveredSet;
private final ConcurrentMap<File, Tailer> tailerMap;
private final Supplier<GCParser<CMSType, CMSHeap>> parserSupplier;
private final ParseExceptionListener exceptionListener;
private final String host;
private volatile boolean go;
private TailManager(final Set<File> discoveredSet,
final ConcurrentMap<File, Tailer> tailerMap,
final Supplier<GCParser<CMSType, CMSHeap>> parserSupplier,
final ParseExceptionListener exceptionListener,
final String host) {
this.discoveredSet = discoveredSet;
this.tailerMap = tailerMap;
this.parserSupplier = parserSupplier;
this.exceptionListener = exceptionListener;
this.host = host;
}
public void stop() {
go = false;
for (Tailer tailer : tailerMap.values())
tailer.stop();
tailerMap.clear();
}
public void start() {
go = true;
}
#Override public void onNew(final File file,
final Channel channel) {
checkState(go);
GCParser<CMSType, CMSHeap> parser = parserSupplier.get();
String fileName = file.getName();
GcEventsPersister.Builder builder =
new GcEventsPersister.Builder(fileName, host);
AgentPauseListener eventListener =
new AgentPauseListener(channel, builder);
Function<Void, Void> removeTail = new Function<Void, Void>() {
#Override
public Void apply(#Nullable final Void input) {
final Tailer tailer = tailerMap.remove(file);
tailer.stop();
discoveredSet.remove(file);
return null;
}
};
GcTailAdaptor listener =
new GcTailAdaptor(logger, parser, eventListener, exceptionListener, removeTail);
tailerMap.put(file, Tailer.create(file, listener, 1000L, true));
}
#Override public void onDelete(File file, Channel channel) {
checkState(go);
final Tailer tailer = tailerMap.remove(file);
tailer.stop();
}
}
static class EfxAgentHandler extends SimpleChannelHandler {
private final ClientBootstrap bootstrap;
private final TailManager tailManager;
private final FileWatcher fileWatcher;
private final Timer timer;
public EfxAgentHandler(ClientBootstrap bootstrap,
TailManager tailManager,
FileWatcher fileWatcher,
Timer timer) {
this.bootstrap = bootstrap;
this.tailManager = tailManager;
this.fileWatcher = fileWatcher;
this.timer = timer;
}
#Override public void channelConnected(ChannelHandlerContext ctx,
ChannelStateEvent e) throws Exception {
logger.info("channel connected");
tailManager.start();
fileWatcher.start(e.getChannel());
}
#Override public void channelDisconnected(ChannelHandlerContext ctx,
ChannelStateEvent e) throws Exception {
logger.error("channel disconnected");
tailManager.stop();
fileWatcher.stop();
scheduleReconnect();
}
private void scheduleReconnect() {
timer.newTimeout(new ReconnectTask(bootstrap), 5L, TimeUnit.SECONDS);
}
#Override public void exceptionCaught(ChannelHandlerContext ctx,
ExceptionEvent e) throws Exception {
final Throwable cause = e.getCause();
logger.error(cause);
if (cause instanceof ConnectException) {
tailManager.stop();
scheduleReconnect();
}
}
}
static final class GcTailAdaptor extends TailerListenerAdapter {
private final Logger logger;
private final GCParser parser;
private final GCEventListener eventListener;
private final ParseExceptionListener exceptionListener;
private final Function<Void, Void> removeTail;
private volatile long lastTail;
GcTailAdaptor(final Logger logger,
final GCParser parser,
final GCEventListener eventListener,
final ParseExceptionListener exceptionListener,
final Function<Void, Void> removeTail) {
this.logger = logger;
this.parser = parser;
this.eventListener = eventListener;
this.exceptionListener = exceptionListener;
this.removeTail = removeTail;
}
#Override public void handle(String line) {
lastTail();
parser.matchLine(line, eventListener, exceptionListener);
}
private final void lastTail() {
final long t = System.currentTimeMillis();
if (lastTail == 0L) {
lastTail = t;
return;
}
if ((t-lastTail)>=1800000L)
removeTail.apply(null);
}
#Override public void handle(Exception ex) {
logger.error(ex);
}
}
#VisibleForTesting
final static class DetectionTask implements Runnable {
private final File directory;
private final Set<File> discovered;
private final FileHandler handler;
private volatile Channel channel;
DetectionTask(final File directory,
final Set<File> discovered,
final FileHandler handler) {
this.discovered = discovered;
this.handler = handler;
this.directory = directory;
}
public void setChannel(Channel channel) {
this.channel = channel;
}
public boolean removeStaleFile(File file) {
checkArgument(discovered.contains(file),
"file is not discovered so cannot be stale");
return discovered.remove(file);
}
public void run() {
final File[] files = directory.listFiles();
for (int i=0, n=files.length; i<n; i++) {
final File file = files[i];
synchronized (discovered) {
if (!discovered.contains(file)) {
discovered.add(file);
handler.onNew(file, channel);
}
}
}
final ImmutableSet<File> logFiles = ImmutableSet.copyOf(files);
final ImmutableSet<File> diff = Sets.difference(discovered, logFiles).immutableCopy();
for (File file : diff) {
discovered.remove(file);
handler.onDelete(file, channel);
}
}
}
#VisibleForTesting static interface FileHandler {
void onNew(File file, Channel channel);
void onDelete(File file, Channel channel);
}
#VisibleForTesting
final static class FileWatcher {
private final ScheduledExecutorService executor;
private final DetectionTask detectionTask;
private final int frequency;
private volatile ScheduledFuture<?> task;
#VisibleForTesting
FileWatcher(ScheduledExecutorService executor,
DetectionTask detectionTask,
int frequency) {
this.executor = executor;
this.detectionTask = detectionTask;
this.frequency = frequency;
}
public void start(Channel chanel) {
task = executor.scheduleAtFixedRate(detectionTask, 0L, frequency, TimeUnit.SECONDS);
detectionTask.setChannel(chanel);
}
public void stop() {
task.cancel(true);
detectionTask.setChannel(null);
}
public static FileWatcher on(File directory,
FileHandler handler,
int frequency) {
checkNotNull(directory);
checkNotNull(handler);
checkArgument(directory.isDirectory(), "file is not a directory");
checkArgument(directory.canRead(), "no read access to directory");
checkArgument(0 < frequency, "frequency must be > 0");
final HashSet<File> objects = Sets.newHashSet();
final DetectionTask task = new DetectionTask(directory, objects, handler);
final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
return new FileWatcher(executorService, task, frequency);
}
}
}

I have incorrect code somewhere.
Correct. Specifically, you almost certainly have different ObjectInput/OutputStream lifetimes at server and client. Use the same streams for the life of the socket, and don't do any I/O over those sockets via any other means.

I discovered why this happens. I am using a deprecated ObjectDecoder that's not compatible with my client's ObjectEncoder. I am just sending a ByteBuffer instead and it's fine now.