We Keep Coding

Classic word count topology in Storm using 2 RabbitMQ queues

I have to write a simple "Word Count" Topology in Java and Storm. In particular, i have an external data source generating CSV (comma separated) string like
Daniel, 0.5654, 144543, user, 899898, Comment,,,
These strings are inserted into a RabbitMQ queue called "input". This datasource works well, and i can see the strings in the queue.
Now, i modified the classic topology adding the RabbitMQSpout. The goal is to do a word count for the first field of every CSV line, and publish results into a new queue called "output". The problem is that i cannot see any tuple inside the new queue, but the topology was submitted and RUNNING.
So, summing up:
external data source puts items into the input queue
RabbitMQSpout takes items from input queue and insert them into topology
classic word-count topology isperformed
last bolt puts results into output queue
Problem:
i can see items inside input queue, but nothing into output, even if i used same method to send item into the queue in the external data source (and it works) and in RabbitMQExporter (does not work...)
Some code below
RabbitMQSpout
public class RabbitMQSpout extends BaseRichSpout {
public static final String DATA = "data";
private SpoutOutputCollector _collector;
private RabbitMQManager rabbitMQManager;
#Override
public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
_collector = _collector;
rabbitMQManager = new RabbitMQManager("localhost", "rabbitmq", "rabbitmq", "test");
}
#Override
public void nextTuple() {
Utils.sleep(1000);
String data = rabbitMQManager.receive("input");
if (data != null) {
_collector.emit(new Values(data));
}
}
#Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields(DATA));
}
}
SplitBolt
public class SplitBolt extends BaseRichBolt {
private OutputCollector _collector;
public SplitSentenceBolt() { }
#Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this._collector = collector;
this.SPACE = Pattern.compile(",");
}
#Override
public void execute(Tuple input) {
String sentence = input.getStringByField(RabbitMQSpout.DATA);
String[] words = sentence.split(",");
if (words.length > 0)
_collector.emit(new Values(words[0]));
}
#Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word"));
}
#Override
public Map<String, Object> getComponentConfiguration() {
return null;
}
}
WordCountBolt
public class WordCountBolt extends BaseBasicBolt {
Map<String, Integer> counts = new HashMap<String, Integer>();
#Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
String word = tuple.getString(0);
Integer count = counts.get(word);
if (count == null)
count = 0;
count++;
counts.put(word, count);
System.out.println(count);
collector.emit(new Values(word, count));
}
#Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word", "count"));
}
}
RabbitMQExporterBolt
public RabbitMQExporterBolt(String rabbitMqHost, String rabbitMqUsername, String rabbitMqPassword,
String defaultQueue) {
super();
this.rabbitMqHost = rabbitMqHost;
this.rabbitMqUsername = rabbitMqUsername;
this.rabbitMqPassword = rabbitMqPassword;
this.defaultQueue = defaultQueue;
}
#Override
public void prepare(#SuppressWarnings("rawtypes") Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
this.collector=outputCollector;
this.rabbitmq = new RabbitMQManager(rabbitMqHost, rabbitMqUsername, rabbitMqPassword, defaultQueue);
}
#Override
public void execute(Tuple tuple) {
String word = tuple.getString(0);
Integer count = tuple.getInteger(1);
String output = word + " " + count;
rabbitmq.send(output);
}
#Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields("word"));
}
}
Topology
public class WordCountTopology {
private static final String RABBITMQ_HOST = "rabbitmq";
private static final String RABBITMQ_USER = "rabbitmq";
private static final String RABBITMQ_PASS = "rabbitmq";
private static final String RABBITMQ_QUEUE = "output";
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("spout", new RabbitMQSpout(), 1);
builder.setBolt("split", new SplitSentenceBolt(), 1)
.shuffleGrouping("spout");
builder.setBolt("count", new WordCountBolt(), 1)
.fieldsGrouping("split", new Fields("word"));
Config conf = new Config();
conf.setDebug(true);
if (args != null && args.length > 0) {
builder.setBolt("exporter",
new RabbitMQExporterBolt(
RABBITMQ_HOST, RABBITMQ_USER,
RABBITMQ_PASS, RABBITMQ_QUEUE ),
1)
.shuffleGrouping("count");
conf.setNumWorkers(3);
StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
} else {
conf.setMaxTaskParallelism(3);
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("word-count", conf, builder.createTopology());
Thread.sleep(10000);
cluster.shutdown();
}
}
}
RabbitMQManager
public class RabbitMQManager {
private String host;
private String username;
private String password;
private ConnectionFactory factory;
private Connection connection;
private String defaultQueue;
public RabbitMQManager(String host, String username, String password, String queue) {
super();
this.host = host;
this.username = username;
this.password = password;
this.factory = null;
this.connection = null;
this.defaultQueue = queue;
this.initialize();
this.initializeQueue(defaultQueue);
}
private void initializeQueue(String queue){
ConnectionFactory factory = new ConnectionFactory();
factory.setHost(host);
factory.setUsername(username);
factory.setPassword(password);
Connection connection;
try {
connection = factory.newConnection();
Channel channel = connection.createChannel();
boolean durable = false;
boolean exclusive = false;
boolean autoDelete = false;
channel.queueDeclare(queue, durable, exclusive, autoDelete, null);
channel.close();
connection.close();
} catch (IOException | TimeoutException e) {
e.printStackTrace();
}
}
private void initialize(){
factory = new ConnectionFactory();
factory.setHost(host);
factory.setUsername(username);
factory.setPassword(password);
try {
connection = factory.newConnection();
} catch (IOException | TimeoutException e) {
e.printStackTrace();
}
}
public void terminate(){
if (connection != null && connection.isOpen()){
try {
connection.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
private boolean reopenConnectionIfNeeded(){
try {
if (connection == null){
connection = factory.newConnection();
return true;
}
if (!connection.isOpen()){
connection = factory.newConnection();
}
} catch (IOException | TimeoutException e) {
e.printStackTrace();
return false;
}
return true;
}
public boolean send(String message){
return this.send(defaultQueue, message);
}
public boolean send(String queue, String message){
try {
reopenConnectionIfNeeded();
Channel channel = connection.createChannel();
channel.basicPublish("", queue, null, message.getBytes());
channel.close();
return true;
} catch (IOException | TimeoutException e) {
e.printStackTrace();
}
return false;
}
public String receive(String queue) {
try {
reopenConnectionIfNeeded();
Channel channel = connection.createChannel();
Consumer consumer = new DefaultConsumer(channel);
return channel.basicConsume(queue, true, consumer);
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
}

Coroutines in Java

I am reading this page about coroutines in Python and this Wikipedia page. I saw that there are a few libraries in Java implementing coroutines.
My question is: is there any known reason why the Java designers decided not to implement coroutines so far and is there any plan to include it in a future version of Java?
Thanks.

Actually the concept of a co-routine was the first design of the Java threading system. The wait/notify mechanism is a simplistic form of co-routine where notify is equivalent to yield etc.
Since then much has been done, particularly to make structures thread-safe rather than algorithms. This derives from the realization that it is not the code that must synchronize/yield but the data structure used to communicate between the threads that must be thread-safe.

Project Loom
Continuations and Coroutines will come to Java in the nearer future and they’ll be called virtual threads (also referred to as fibers). There’s a project called Loom:
Project Loom is intended to explore, incubate and deliver Java VM features and APIs built on top of them for the purpose of supporting easy-to-use, high-throughput lightweight concurrency and new programming models on the Java platform. This is accomplished by the addition of the following constructs:
Virtual threads
Delimited continuations
Tail-call elimination
Further reading: https://cr.openjdk.java.net/~rpressler/loom/Loom-Proposal.html
To quote that document:
It is the goal of this project to add a public delimited continuation (or coroutine) construct to the Java platform. However, this goal is secondary to fibers …
Preliminary builds of Project Loom are available now, based on early-access Java 16.

On the "are there any plans ..." part of the question, the answer is:
Not at this stage
The JEP list (http://openjdk.java.net/jeps/0) does not make any mention of coroutines. The list covers features added in Java 8, added or targeted for Java 9, or proposed for future releases.
Interestingly, there was an RFE submitted in March 2013 (https://bugs.openjdk.java.net/browse/JDK-8029988). The RFE only got one vote, and it was closed 9 months with the suggestion to submit a JEP. Nobody has bothered to take the idea any further, which to me is telling.

It's synced with Java 15 build 7.
Link

There's an another choice is here for Java6+
A pythonic coroutine implementation:
import java.lang.ref.WeakReference;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
class CorRunRAII {
private final List<WeakReference<? extends CorRun>> resources = new ArrayList<>();
public CorRunRAII add(CorRun resource) {
if (resource == null) {
return this;
}
resources.add(new WeakReference<>(resource));
return this;
}
public CorRunRAII addAll(List<? extends CorRun> arrayList) {
if (arrayList == null) {
return this;
}
for (CorRun corRun : arrayList) {
add(corRun);
}
return this;
}
#Override
protected void finalize() throws Throwable {
super.finalize();
for (WeakReference<? extends CorRun> corRunWeakReference : resources) {
CorRun corRun = corRunWeakReference.get();
if (corRun != null) {
corRun.stop();
}
}
}
}
class CorRunYieldReturn<ReceiveType, YieldReturnType> {
public final AtomicReference<ReceiveType> receiveValue;
public final LinkedBlockingDeque<AtomicReference<YieldReturnType>> yieldReturnValue;
CorRunYieldReturn(AtomicReference<ReceiveType> receiveValue, LinkedBlockingDeque<AtomicReference<YieldReturnType>> yieldReturnValue) {
this.receiveValue = receiveValue;
this.yieldReturnValue = yieldReturnValue;
}
}
interface CorRun<ReceiveType, YieldReturnType> extends Runnable, Callable<YieldReturnType> {
boolean start();
void stop();
void stop(final Throwable throwable);
boolean isStarted();
boolean isEnded();
Throwable getError();
ReceiveType getReceiveValue();
void setResultForOuter(YieldReturnType resultForOuter);
YieldReturnType getResultForOuter();
YieldReturnType receive(ReceiveType value);
ReceiveType yield();
ReceiveType yield(YieldReturnType value);
<TargetReceiveType, TargetYieldReturnType> TargetYieldReturnType yieldFrom(final CorRun<TargetReceiveType, TargetYieldReturnType> another);
<TargetReceiveType, TargetYieldReturnType> TargetYieldReturnType yieldFrom(final CorRun<TargetReceiveType, TargetYieldReturnType> another, final TargetReceiveType value);
}
abstract class CorRunSync<ReceiveType, YieldReturnType> implements CorRun<ReceiveType, YieldReturnType> {
private ReceiveType receiveValue;
public final List<WeakReference<CorRun>> potentialChildrenCoroutineList = new ArrayList<>();
// Outside
private AtomicBoolean isStarted = new AtomicBoolean(false);
private AtomicBoolean isEnded = new AtomicBoolean(false);
private Throwable error;
private YieldReturnType resultForOuter;
#Override
public boolean start() {
boolean isStarted = this.isStarted.getAndSet(true);
if ((! isStarted)
&& (! isEnded())) {
receive(null);
}
return isStarted;
}
#Override
public void stop() {
stop(null);
}
#Override
public void stop(Throwable throwable) {
isEnded.set(true);
if (throwable != null) {
error = throwable;
}
for (WeakReference<CorRun> weakReference : potentialChildrenCoroutineList) {
CorRun child = weakReference.get();
if (child != null) {
child.stop();
}
}
}
#Override
public boolean isStarted() {
return isStarted.get();
}
#Override
public boolean isEnded() {
return isEnded.get();
}
#Override
public Throwable getError() {
return error;
}
#Override
public ReceiveType getReceiveValue() {
return receiveValue;
}
#Override
public void setResultForOuter(YieldReturnType resultForOuter) {
this.resultForOuter = resultForOuter;
}
#Override
public YieldReturnType getResultForOuter() {
return resultForOuter;
}
#Override
public synchronized YieldReturnType receive(ReceiveType value) {
receiveValue = value;
run();
return getResultForOuter();
}
#Override
public ReceiveType yield() {
return yield(null);
}
#Override
public ReceiveType yield(YieldReturnType value) {
resultForOuter = value;
return receiveValue;
}
#Override
public <TargetReceiveType, TargetYieldReturnType> TargetYieldReturnType yieldFrom(CorRun<TargetReceiveType, TargetYieldReturnType> another) {
return yieldFrom(another, null);
}
#Override
public <TargetReceiveType, TargetYieldReturnType> TargetYieldReturnType yieldFrom(CorRun<TargetReceiveType, TargetYieldReturnType> another, TargetReceiveType value) {
if (another == null || another.isEnded()) {
throw new RuntimeException("Call null or isEnded coroutine");
}
potentialChildrenCoroutineList.add(new WeakReference<CorRun>(another));
synchronized (another) {
boolean isStarted = another.start();
boolean isJustStarting = ! isStarted;
if (isJustStarting && another instanceof CorRunSync) {
return another.getResultForOuter();
}
return another.receive(value);
}
}
#Override
public void run() {
try {
this.call();
}
catch (Exception e) {
e.printStackTrace();
stop(e);
return;
}
}
}
abstract class CorRunThread<ReceiveType, YieldReturnType> implements CorRun<ReceiveType, YieldReturnType> {
private final ExecutorService childExecutorService = newExecutorService();
private ExecutorService executingOnExecutorService;
private static final CorRunYieldReturn DUMMY_COR_RUN_YIELD_RETURN = new CorRunYieldReturn(new AtomicReference<>(null), new LinkedBlockingDeque<AtomicReference>());
private final CorRun<ReceiveType, YieldReturnType> self;
public final List<WeakReference<CorRun>> potentialChildrenCoroutineList;
private CorRunYieldReturn<ReceiveType, YieldReturnType> lastCorRunYieldReturn;
private final LinkedBlockingDeque<CorRunYieldReturn<ReceiveType, YieldReturnType>> receiveQueue;
// Outside
private AtomicBoolean isStarted = new AtomicBoolean(false);
private AtomicBoolean isEnded = new AtomicBoolean(false);
private Future<YieldReturnType> future;
private Throwable error;
private final AtomicReference<YieldReturnType> resultForOuter = new AtomicReference<>();
CorRunThread() {
executingOnExecutorService = childExecutorService;
receiveQueue = new LinkedBlockingDeque<>();
potentialChildrenCoroutineList = new ArrayList<>();
self = this;
}
#Override
public void run() {
try {
self.call();
}
catch (Exception e) {
stop(e);
return;
}
stop();
}
#Override
public abstract YieldReturnType call();
#Override
public boolean start() {
return start(childExecutorService);
}
protected boolean start(ExecutorService executorService) {
boolean isStarted = this.isStarted.getAndSet(true);
if (!isStarted) {
executingOnExecutorService = executorService;
future = (Future<YieldReturnType>) executingOnExecutorService.submit((Runnable) self);
}
return isStarted;
}
#Override
public void stop() {
stop(null);
}
#Override
public void stop(final Throwable throwable) {
if (throwable != null) {
error = throwable;
}
isEnded.set(true);
returnYieldValue(null);
// Do this for making sure the coroutine has checked isEnd() after getting a dummy value
receiveQueue.offer(DUMMY_COR_RUN_YIELD_RETURN);
for (WeakReference<CorRun> weakReference : potentialChildrenCoroutineList) {
CorRun child = weakReference.get();
if (child != null) {
if (child instanceof CorRunThread) {
((CorRunThread)child).tryStop(childExecutorService);
}
}
}
childExecutorService.shutdownNow();
}
protected void tryStop(ExecutorService executorService) {
if (this.executingOnExecutorService == executorService) {
stop();
}
}
#Override
public boolean isEnded() {
return isEnded.get() || (
future != null && (future.isCancelled() || future.isDone())
);
}
#Override
public boolean isStarted() {
return isStarted.get();
}
public Future<YieldReturnType> getFuture() {
return future;
}
#Override
public Throwable getError() {
return error;
}
#Override
public void setResultForOuter(YieldReturnType resultForOuter) {
this.resultForOuter.set(resultForOuter);
}
#Override
public YieldReturnType getResultForOuter() {
return this.resultForOuter.get();
}
#Override
public YieldReturnType receive(ReceiveType value) {
LinkedBlockingDeque<AtomicReference<YieldReturnType>> yieldReturnValue = new LinkedBlockingDeque<>();
offerReceiveValue(value, yieldReturnValue);
try {
AtomicReference<YieldReturnType> takeValue = yieldReturnValue.take();
return takeValue == null ? null : takeValue.get();
} catch (InterruptedException e) {
e.printStackTrace();
}
return null;
}
#Override
public ReceiveType yield() {
return yield(null);
}
#Override
public ReceiveType yield(final YieldReturnType value) {
returnYieldValue(value);
return getReceiveValue();
}
#Override
public <TargetReceiveType, TargetYieldReturnType> TargetYieldReturnType yieldFrom(final CorRun<TargetReceiveType, TargetYieldReturnType> another) {
return yieldFrom(another, null);
}
#Override
public <TargetReceiveType, TargetYieldReturnType> TargetYieldReturnType yieldFrom(final CorRun<TargetReceiveType, TargetYieldReturnType> another, final TargetReceiveType value) {
if (another == null || another.isEnded()) {
throw new RuntimeException("Call null or isEnded coroutine");
}
boolean isStarted = false;
potentialChildrenCoroutineList.add(new WeakReference<CorRun>(another));
synchronized (another) {
if (another instanceof CorRunThread) {
isStarted = ((CorRunThread)another).start(childExecutorService);
}
else {
isStarted = another.start();
}
boolean isJustStarting = ! isStarted;
if (isJustStarting && another instanceof CorRunSync) {
return another.getResultForOuter();
}
TargetYieldReturnType send = another.receive(value);
return send;
}
}
#Override
public ReceiveType getReceiveValue() {
setLastCorRunYieldReturn(takeLastCorRunYieldReturn());
return lastCorRunYieldReturn.receiveValue.get();
}
protected void returnYieldValue(final YieldReturnType value) {
CorRunYieldReturn<ReceiveType, YieldReturnType> corRunYieldReturn = lastCorRunYieldReturn;
if (corRunYieldReturn != null) {
corRunYieldReturn.yieldReturnValue.offer(new AtomicReference<>(value));
}
}
protected void offerReceiveValue(final ReceiveType value, LinkedBlockingDeque<AtomicReference<YieldReturnType>> yieldReturnValue) {
receiveQueue.offer(new CorRunYieldReturn(new AtomicReference<>(value), yieldReturnValue));
}
protected CorRunYieldReturn<ReceiveType, YieldReturnType> takeLastCorRunYieldReturn() {
try {
return receiveQueue.take();
} catch (InterruptedException e) {
e.printStackTrace();
}
return null;
}
protected void setLastCorRunYieldReturn(CorRunYieldReturn<ReceiveType,YieldReturnType> lastCorRunYieldReturn) {
this.lastCorRunYieldReturn = lastCorRunYieldReturn;
}
protected ExecutorService newExecutorService() {
return Executors.newCachedThreadPool(getThreadFactory());
}
protected ThreadFactory getThreadFactory() {
return new ThreadFactory() {
#Override
public Thread newThread(final Runnable runnable) {
Thread thread = new Thread(runnable);
thread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
#Override
public void uncaughtException(Thread thread, Throwable throwable) {
throwable.printStackTrace();
if (runnable instanceof CorRun) {
CorRun self = (CorRun) runnable;
self.stop(throwable);
thread.interrupt();
}
}
});
return thread;
}
};
}
}
Now you can use pythonic coroutines in this way
(e.g. fibonacci numbers)
Thread Version:
class Fib extends CorRunThread<Integer, Integer> {
#Override
public Integer call() {
Integer times = getReceiveValue();
do {
int a = 1, b = 1;
for (int i = 0; times != null && i < times; i++) {
int temp = a + b;
a = b;
b = temp;
}
// A pythonic "yield", i.e., it returns `a` to the caller and waits `times` value from the next caller
times = yield(a);
} while (! isEnded());
setResultForOuter(Integer.MAX_VALUE);
return getResultForOuter();
}
}
class MainRun extends CorRunThread<String, String> {
#Override
public String call() {
// The fib coroutine would be recycled by its parent
// (no requirement to call its start() and stop() manually)
// Otherwise, if you want to share its instance and start/stop it manually,
// please start it before being called by yieldFrom() and stop it in the end.
Fib fib = new Fib();
String result = "";
Integer current;
int times = 10;
for (int i = 0; i < times; i++) {
// A pythonic "yield from", i.e., it calls fib with `i` parameter and waits for returned value as `current`
current = yieldFrom(fib, i);
if (fib.getError() != null) {
throw new RuntimeException(fib.getError());
}
if (current == null) {
continue;
}
if (i > 0) {
result += ",";
}
result += current;
}
setResultForOuter(result);
return result;
}
}
Sync(non-thread) version:
class Fib extends CorRunSync<Integer, Integer> {
#Override
public Integer call() {
Integer times = getReceiveValue();
int a = 1, b = 1;
for (int i = 0; times != null && i < times; i++) {
int temp = a + b;
a = b;
b = temp;
}
yield(a);
return getResultForOuter();
}
}
class MainRun extends CorRunSync<String, String> {
#Override
public String call() {
CorRun<Integer, Integer> fib = null;
try {
fib = new Fib();
} catch (Exception e) {
e.printStackTrace();
}
String result = "";
Integer current;
int times = 10;
for (int i = 0; i < times; i++) {
current = yieldFrom(fib, i);
if (fib.getError() != null) {
throw new RuntimeException(fib.getError());
}
if (current == null) {
continue;
}
if (i > 0) {
result += ",";
}
result += current;
}
stop();
setResultForOuter(result);
if (Utils.isEmpty(result)) {
throw new RuntimeException("Error");
}
return result;
}
}
Execution(Both versions will work):
// Run the entry coroutine
MainRun mainRun = new MainRun();
mainRun.start();
// Wait for mainRun ending for 5 seconds
long startTimestamp = System.currentTimeMillis();
while(!mainRun.isEnded()) {
if (System.currentTimeMillis() - startTimestamp > TimeUnit.SECONDS.toMillis(5)) {
throw new RuntimeException("Wait too much time");
}
}
// The result should be "1,1,2,3,5,8,13,21,34,55"
System.out.println(mainRun.getResultForOuter());

WorkStealingPool exits unexpectedly

I submitted some Runnables to an ExecutorService. Inside these Runnables, wait() and notify() are called. The code works with newFixedThreadPool as the ExecutorService. With newWorkStealingPool, the process exits unexpectedly without any error message.
import java.net.URL;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.IOException;
// For regular expressions
import java.util.regex.Matcher;
import java.util.regex.MatchResult;
import java.util.regex.Pattern;
import java.util.*;
import java.util.concurrent.*;
public class TestPipeline {
public static void main(String[] args) {
runAsThreads();
}
private static void runAsThreads() {
final BlockingQueue<String> urls = new OneItemQueue<String>();
final BlockingQueue<Webpage> pages = new OneItemQueue<Webpage>();
final BlockingQueue<Link> refPairs = new OneItemQueue<Link>();
final BlockingQueue<Link> uniqRefPairs = new OneItemQueue<Link>();
final ExecutorService executor = Executors.newWorkStealingPool(6);
// final ExecutorService executor = Executors.newFixedThreadPool(6);
executor.submit(new UrlProducer(urls));
executor.submit(new PageGetter(urls, pages));
executor.submit(new LinkScanner(pages,refPairs));
executor.submit(new Uniquifier<Link>(refPairs,uniqRefPairs));
executor.submit(new LinkPrinter(uniqRefPairs));
}
}
class UrlProducer implements Runnable {
private final BlockingQueue<String> output;
public UrlProducer(BlockingQueue<String> output) {
this.output = output;
}
public void run() {
System.out.println("in producer");
for (int i=0; i<urls.length; i++)
output.put(urls[i]);
}
private static final String[] urls =
{ "http://www.itu.dk", "http://www.di.ku.dk", "http://www.miele.de",
"http://www.microsoft.com", "http://www.cnn.com", "http://www.dr.dk",
"http://www.vg.no", "http://www.tv2.dk", "http://www.google.com",
"http://www.ing.dk", "http://www.dtu.dk", "http://www.bbc.co.uk"
};
}
class PageGetter implements Runnable {
private final BlockingQueue<String> input;
private final BlockingQueue<Webpage> output;
public PageGetter(BlockingQueue<String> input, BlockingQueue<Webpage> output) {
this.input = input;
this.output = output;
}
public void run() {
while (true) {
System.out.println("in pagegetter");
String url = input.take();
// System.out.println("PageGetter: " + url);
try {
String contents = getPage(url, 200);
output.put(new Webpage(url, contents));
} catch (IOException exn) { System.out.println(exn); }
}
}
public static String getPage(String url, int maxLines) throws IOException {
// This will close the streams after use (JLS 8 para 14.20.3):
try (BufferedReader in
= new BufferedReader(new InputStreamReader(new URL(url).openStream()))) {
StringBuilder sb = new StringBuilder();
for (int i=0; i<maxLines; i++) {
String inputLine = in.readLine();
if (inputLine == null)
break;
else
sb.append(inputLine).append("\n");
}
return sb.toString();
}
}
}
class Uniquifier<T> implements Runnable{
private final Set<T> set = new HashSet<T>();
private final BlockingQueue<T> input;
private final BlockingQueue<T> output;
public Uniquifier(BlockingQueue<T> input, BlockingQueue<T> output){
this.input = input;
this.output = output;
}
public void run(){
while(true){
System.out.println("in uniquifier");
T item = input.take();
if(!set.contains(item)){
set.add(item);
output.put(item);
}
}
}
}
class LinkScanner implements Runnable {
private final BlockingQueue<Webpage> input;
private final BlockingQueue<Link> output;
public LinkScanner(BlockingQueue<Webpage> input,
BlockingQueue<Link> output) {
this.input = input;
this.output = output;
}
private final static Pattern urlPattern
= Pattern.compile("a href=\"(\\p{Graph}*)\"");
public void run() {
while (true) {
System.out.println("in link scanner");
Webpage page = input.take();
// System.out.println("LinkScanner: " + page.url);
// Extract links from the page's <a href="..."> anchors
Matcher urlMatcher = urlPattern.matcher(page.contents);
while (urlMatcher.find()) {
String link = urlMatcher.group(1);
output.put(new Link(page.url, link));
}
}
}
}
class LinkPrinter implements Runnable {
private final BlockingQueue<Link> input;
public LinkPrinter(BlockingQueue<Link> input) {
this.input = input;
}
public void run() {
while (true) {
System.out.println("in link printer");
Link link = input.take();
// System.out.println("LinkPrinter: " + link.from);
System.out.printf("%s links to %s%n", link.from, link.to);
}
}
}
class Webpage {
public final String url, contents;
public Webpage(String url, String contents) {
this.url = url;
this.contents = contents;
}
}
class Link {
public final String from, to;
public Link(String from, String to) {
this.from = from;
this.to = to;
}
// Override hashCode and equals so can be used in HashSet<Link>
public int hashCode() {
return (from == null ? 0 : from.hashCode()) * 37
+ (to == null ? 0 : to.hashCode());
}
public boolean equals(Object obj) {
Link that = obj instanceof Link ? (Link)obj : null;
return that != null
&& (from == null ? that.from == null : from.equals(that.from))
&& (to == null ? that.to == null : to.equals(that.to));
}
}
// Different from java.util.concurrent.BlockingQueue: Allows null
// items, and methods do not throw InterruptedException.
interface BlockingQueue<T> {
void put(T item);
T take();
}
class OneItemQueue<T> implements BlockingQueue<T> {
private T item;
private boolean full = false;
public void put(T item) {
synchronized (this) {
while (full) {
try { this.wait(); }
catch (InterruptedException exn) { }
}
full = true;
this.item = item;
this.notifyAll();
}
}
public T take() {
synchronized (this) {
while (!full) {
try { this.wait(); }
catch (InterruptedException exn) { }
}
full = false;
this.notifyAll();
return item;
}
}
}

Because the Pool is allocating threads dynamically, there are no threads alive after runAsThreads exits because that's the end of the main thread. There needs to be at least on thread running to keep the application alive. Adding a call to awaitTermination is needed. It's not needed for the fixed pool because that will always have active threads until it is explicitly shut down as noted in the JavaDocs.

MVEL executeExpression function cannot be concurrent

Run the main function in File2 , the problem is : threads stuck at "rval=MVEL.executeExpression(compiledExpression, vars);" , 10 threads run in sequential order, not parallel , I wanna know why this happened.
PS: I'm using MVEL 2.2 , the latest version
File1:MVELHelper.java
public class MVELHelper {
private static ParserContext _ctx = new ParserContext(false);
//public static Object execute(String expression, Map<String, Object> vars, Databus databus) throws Exception {
public static Object execute(String expression, Map<String, Object> vars) throws Exception {
Object rval = null;
try {
if(vars == null) {
rval = MVEL.eval(expression, new HashMap<String,Object>());
}
else {
rval = MVEL.eval(expression, vars);
}
return rval;
}
catch(Exception e) {
throw new Exception("MVEL FAILED:"+expression,e);
}
}
public static Serializable compile(String text, ParserContext ctx)
throws Exception {
if(ctx == null) {
//ctx = _ctx;
ctx=new ParserContext(false);
}
Serializable exp = null;
try {
exp = MVEL.compileExpression(text, ctx);
//exp = MVEL.compileExpression(text);
}
catch (Exception e) {
throw new Exception("failed to compile expression.", e);
}
return exp;
}
public static Object compileAndExecute(String expression, Map<String, Object> vars) throws Exception {
Object rval = null;
try {
Serializable compiledExpression=compile(expression,null);
System.out.println("[COMPILE OVER, Thread Id="+Thread.currentThread().getId()+"] ");
if(vars == null) {
rval=MVEL.executeExpression(compiledExpression, new HashMap<String,Object>());
//rval = MVEL.eval(exp, new HashMap<String,Object>());
}
else {
//rval=MVEL.executeExpression(compiledExpression, vars,(VariableResolverFactory)null);
rval=MVEL.executeExpression(compiledExpression, vars);
//rval = MVEL.eval(expression, vars);
}
return rval;
}
catch(Exception e) {
throw new Exception("MVEL FAILED:"+expression,e);
}
}
}
File2:ExecThread3.java
public class ExecThread3 implements Runnable{
Map dataMap=null;
public Map getDataMap() {
return dataMap;
}
public void setDataMap(Map dataMap) {
this.dataMap = dataMap;
}
#Override
public void run() {
Map varsMap = new HashMap();
Map dataMap=new HashMap();
dataMap.put("count",100);
varsMap.put("dataMap", dataMap);
String expression="System.out.println(\"[BEFORE Thread Id=\"+Thread.currentThread().getId()+\"] \"+dataMap.get(\"count\"));"+
"Thread.sleep(3000);"+
"System.err.println(\"[AFTER Thread Id=\"+Thread.currentThread().getId()+\"] \"+dataMap.get(\"count\"));";
try {
//MVEL.compileExpression(expression);
MVELHelper.compileAndExecute(expression, varsMap);
}
catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static void main(String[] args) {
for(int k=0;k<10;k++){
ExecThread3 execThread=new ExecThread3();
new Thread(execThread).start();
}
}
}

What is advantage of locks over wait/notify?

What is advantage of locks over wait/notify?
Code is very similar.
private Object full = new Object();
private Object empty = new Object();
private Object data = null;
public static void main(String[] args) {
Test test = new Test();
new Thread(test.new Producer()).start();
new Thread(test.new Consumer()).start();
}
public void push(Object d) {
synchronized (full) {
while (data != null)
try {
full.wait();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
data = d;
System.out.println("push");
synchronized (empty) {
if (data != null)
empty.notify();
}
}
public Object pop() {
synchronized (empty) {
while (data == null)
try {
empty.wait();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
Object o = data;
data = null;
System.out.println("pop");
synchronized (full) {
if (data == null)
full.notify();
}
return o;
}
class Producer implements Runnable {
public void run() {
while (true) {
push(new Object());
}
}
}
class Consumer implements Runnable {
public void run() {
while (true) {
pop();
}
}
}
and
private final ReentrantLock lock = new ReentrantLock();
private final Condition fullState = lock.newCondition();
private final Condition emptyState = lock.newCondition();
private Object data = null;
public static void main(String[] args) {
Test test = new Test();
new Thread(test.new Producer()).start();
new Thread(test.new Consumer()).start();
}
public void push(Object d) {
lock.lock();
try {
while (data != null)
try {
fullState.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
data = d;
System.out.println("push");
emptyState.signal();
} finally {
lock.unlock();
}
}
public Object pop() {
Object result;
lock.lock();
try {
while (data == null)
try {
emptyState.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
result = data;
data = null;
System.out.println("pop");
fullState.signal();
} finally {
lock.unlock();
}
return result;
}
class Producer implements Runnable {
public void run() {
while (true) {
push(new Object());
}
}
}
class Consumer implements Runnable {
public void run() {
while (true) {
pop();
}
}
}

Check out the JavaDoc for ReeentratLock and your question will be answered.
"A reentrant mutual exclusion Lock with the same basic behavior and semantics as the implicit monitor lock accessed using synchronized methods and statements, but with extended capabilities."
http://docs.oracle.com/javase/1.5.0/docs/api/java/util/concurrent/locks/ReentrantLock.html