We Keep Coding

Concurrency issue with threads

I have a simple code, consisting of 4 threads (2 modify the data and 2 read the data). I just wrote this sample code to play around with Semaphor and I am not sure why I get ava.base/java.util.ArrayList$Itr.checkForComodification exception? Here are the source code and thanks for any insights.
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Semaphore;
public class SemaphoreExample {
public static class InnerWriterSemaphoreThread implements Runnable {
private final List<String> fList;
private final Semaphore fWriteSem;
InnerWriterSemaphoreThread(List<String> list, Semaphore w) {
fList = list;
fWriteSem = w;
}
private void prune() {
System.out.println(Thread.currentThread().getName()+" in prune()..");
for (String s : fList) {
fList.remove(s);
}
}
#Override
public void run() {
String name = Thread.currentThread().getName();
String text;
while (true) {
text = RandomTextGenerator.getRandomSNumbertring();
try {
while(!fWriteSem.tryAcquire()){
System.out.println(name+" waiting to accquire semaphore to write..");
Thread.sleep(0L,4);
}
if (fList.size() > 10) {
prune();
}
fList.add(text);
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
fWriteSem.release();// notify readers that write has completed
System.out.println(name+" finished writing, releasing semaphore..");
}
}//while()
}//run()
}//WriterSemaphoreThread
public static class InnerReaderSemaphoreThread implements Runnable {
private final List<String> fList;
private final Semaphore fWriteSem;
InnerReaderSemaphoreThread(List<String> list,Semaphore w) {
fList = list;
fWriteSem = w;
}
private void sleep(){
try{
Thread.sleep(0L, 4);
}catch(InterruptedException e){
e.printStackTrace();
}
}
#Override
public void run() {
String name = Thread.currentThread().getName();
while (true) {
System.out.println(name + " in run()..");
try {
while(fList.isEmpty()){
System.out.println(name+" list is empty, going to sleep..");
sleep();
}
while(!fWriteSem.tryAcquire()){
System.out.println(name+" waiting to accquire semaphor to read..");
Thread.sleep(0l,4);
}
for (String text : fList) {
System.out.println(name + " reading from list " + text);
}
}catch (InterruptedException e) {
e.printStackTrace();
}
finally{
fWriteSem.release(); //Notify threads who want to write to the list
System.out.println(name+" finished reading, releasing semaphore and going to sleep..");
sleep();
}
}
}
}//ReaderSemaphoreThread
/**
* #param args the command line arguments
*/
public static void main(String[] args) {
List<String> list = new ArrayList<>();
Semaphore r = new Semaphore(1);
Thread th1 = new Thread(new InnerWriterSemaphoreThread(list, r), "Thread 1");
Thread th2 = new Thread(new InnerReaderSemaphoreThread(list, r), "Thread 2");
Thread th3 = new Thread(new InnerWriterSemaphoreThread(list, r), "Thread 3");
Thread th4 = new Thread(new InnerReaderSemaphoreThread(list, r), "Thread 4");
th2.start();
th4.start();
th1.start();
th3.start();
}
}
Above is the sample source code

As #assylias mentioned in comment it happens when you remove elements from list in foreach loop. Just replace
for (String s : fList) {
fList.remove(s);
}
with
fList.clear();

Multithreaded Web Crawler in Java

I am trying to write a Multithreaded Web Crawler in Java using Jsoup.I have a Java Class "Master" which creates 6 threads(5 for crawling and 1 for maintenance of queues) ,and 3 queues namely "to_do","to_do_next"(to be done in next iteration) and "done"(final links).
I am using sunchronized locks on shared queues.The idea is as soon as all the 5 threads find the "to_do" queue empty they notify a maintenance thread which does some work and notify these threads back.But the problem is the program is getting blocked sometimes (so i assume there is some race condition I am not able to take care of)....also upon checking I found that not all threads are getting notified by maintenace thread.so is it possible that some notify signals might be lost??
Code for Master class
private Queue<String> to_do = new LinkedList<String>();
private Queue<String> done= new LinkedList<String>();
private Queue<String> to_do_next = new LinkedList<String>();
private int level = 1;
private Object lock1 = new Object();
private Object lock2 = new Object();
private Object lock3 = new Object();
private static Thread maintenance;
public static Master mref;
public static Object wait1 = new Object();
public static Object wait2 = new Object();
public static Object wait3 = new Object();
public static int flag = 5;
public static int missedSignals = -1;
public boolean checkToDoEmpty(){
return to_do.isEmpty();
}
public int getLevel() {
return level;
}
public void incLevel() {
this.level++;
}
public static void interrupt() {
maintenance.interrupt();
}
public void transfer() {
to_do = to_do_next;
}
public String accessToDo() {
synchronized(lock1){
String tmp = to_do.peek();
if(tmp != null)
tmp = to_do.remove();
return tmp;
}
}
public void addToDoNext(String url){
synchronized(lock2){
to_do_next.add(url);
}
}
public void addDone(String string) {
synchronized(lock3){
done.add(string);
}
}
public static void main(String[] args){
Master m = new Master();
mref = m;
URL startUrl = null;
try {
startUrl = new URL("http://cse.iitkgp.ac.in");
}catch (MalformedURLException e1) {
e1.printStackTrace();
}
Thread t1 = new Thread(new Worker(1));
Thread t2 = new Thread(new Worker(2));
Thread t3 = new Thread(new Worker(3));
Thread t4 = new Thread(new Worker(4));
Thread t5 = new Thread(new Worker(5));
maintenance = new Thread(new MaintenanceThread());
m.to_do.add(startUrl.toString());
maintenance.start();
t1.start();
t2.start();
t3.start();
t4.start();
t5.start();
try {
t1.join();
t2.join();
t3.join();
t4.join();
t5.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
/*for(String s:m.done)
System.out.println(s);
for(String s:m.to_do)
System.out.println(s);*/
}
Code for Worker threads
public void run() {
while(Master.mref.getLevel() != 3){
if(!Master.mref.checkToDoEmpty()){
String url = Master.mref.accessToDo();
if(url != null && url.contains("iitkgp") && url.contains("http://")){
try {
Document doc = Jsoup.connect(url).get();
org.jsoup.select.Elements links = doc.select("a[href]");
for(org.jsoup.nodes.Element l: links){
Master.mref.addToDoNext(l.attr("abs:href").toString());
}
Master.mref.addDone(url);
} catch (IOException e) {
System.out.println(url);
e.printStackTrace();
}
continue;
}
}
//System.out.println("thread " + id + " about to notify on wait1");
synchronized(Master.wait1){
Master.wait1.notify();
Master.missedSignals++;
}
synchronized(Master.wait2){
try {
Master.wait2.wait();
System.out.println("thread " + id + " coming out of wait2");
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
System.out.println("Terminating " + id + " thread");
Master.flag--;
if(Master.flag == 0)
Master.interrupt();
}
Code for Maintenace thread
while(Master.flag != 0){
try {
synchronized(Master.wait1){
if(Master.missedSignals != -1){
count += Master.missedSignals;
Master.missedSignals = -1;
}
while(count != 5){
Master.wait1.wait();
if(Master.missedSignals != -1)
count += Master.missedSignals;
Master.missedSignals = -1;
count++;
}
count = 0;
}
//System.out.println("in between");
Master.mref.incLevel();
Master.mref.transfer();
synchronized(Master.wait2){
Master.wait2.notifyAll();
}
} catch (InterruptedException e) {
break;
}
}
System.out.println("Mainta thread gone");

Your design is way too complicated
i suggest using for your to_do queue the following: LinkedBlockingQueue
This is a blocking queue, which means that your threads will ask for an object from the queue and only when one will appear they will get the object, till then they will stay blocking.
Just use the following methods to put and take objects in the queue: put() & take()
Please look at the following two links for more explanations on this special queue:
http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/LinkedBlockingQueue.html
http://tutorials.jenkov.com/java-util-concurrent/linkedblockingqueue.html
Now, your only concern is killing the threads when they are finished with their work, for that I suggest the following:
boolean someThreadStillAlive = true;
while (someThreadStillAlive) {
someThreadStillAlive = false;
Thread.sleep(200);
for (Thread t : fetchAndParseThreads) {
someThreadStillAlive = someThreadStillAlive || t.isAlive();
}
}
This will occur in your main code block, where it will loop & sleep till all threads are finished.
Ohh, instead of take(), you can use poll(int timeout...) where it will wait for the timeout to finish and if no new object is inserted into the queue it will kill the thread.
All of the above, were used successfully in my own crawler.

Tracking Executing Threads

I am trying to figure out how I can track all the threads that my application is spawning. Initially, I thought I had it figured out using a CyclicBarrier, however I am seeing threads executing after my await call.
Below is the working pseudo code:
public class ThreadTesterRunner {
public static void main(String[] args) throws InterruptedException {
final CyclicBarrier cb = new CyclicBarrier(1);
ThreadRunner tr = new ThreadRunner(cb);
Thread t = new Thread(tr, "Thread Runner");
t.start();
boolean process = true;
// wait until all threads process, then print reports
while (process){
if(tr.getIsFinished()){
System.out.println("Print metrics");
process = false;
}
Thread.sleep(1000);
}
}
}
class ThreadRunner implements Runnable {
static int timeOutTime = 2;
private ExecutorService executorService = Executors.newFixedThreadPool(10);
private final CyclicBarrier barrier;
private boolean isFinished=false;
public ThreadRunner(CyclicBarrier cb) {
this.barrier = cb;
}
public void run(){
try {
boolean stillLoop = true; int i = 0;
while (stillLoop){
int size;
Future<Integer> future = null;
try {
future = executorService.submit(new Reader()); // sleeps
size = future.get();
} catch (InterruptedException | ExecutionException ex) {
// handle Errs
}
if(i == 3){
stillLoop = false;
this.barrier.await();
this.isFinished=true;
}
//System.out.println("i = "+i+" Size is: "+size+"\r");
i++;
}
} catch (InterruptedException | BrokenBarrierException e1) {
e1.printStackTrace();
}
}
public boolean getIsFinished(){
return this.isFinished;
}
}
class Reader implements Callable {
private ExecutorService executorService = Executors.newFixedThreadPool(1);
#Override
public Object call() throws Exception {
System.out.println("Reading...");
Thread.sleep(2000);
executorService.submit(new Writer());
return 1000;
}
}
class Writer implements Callable {
#Override
public Void call() throws Exception {
Thread.sleep(4000);
System.out.println("Wrote");
return null;
}
}
Can anyone suggest a way to ONLY print "print metrics" after all threads have run?

It doesn't seem like you're doing anything to coordinate with your Reader and Writer threads, which are the ones you want to wait for. If you pass your synchronization barrier through to those threads so that they can register and signal when they are done, it works just fine.
Here's a version rewritten to do so, using a Phaser instead of a CyclicBarrier. Note that each Reader and Writer registers itself upon construction, and notifies the synchronization barrier when it is done executing:
public class ThreadTesterRunner {
public static void main(String[] args) throws InterruptedException {
final Phaser cb = new Phaser();
ThreadRunner tr = new ThreadRunner(cb);
Thread t = new Thread(tr, "Thread Runner");
t.start();
boolean process = true;
// wait until all threads process, then print reports
while (process){
if(tr.getIsFinished()){
System.out.println("Print metrics");
process = false;
}
//else {
// System.out.println("Waiting: registered=" + cb.getRegisteredParties() + ", arrived=" + cb.getArrivedParties() + ", unarrived=" + cb.getUnarrivedParties());
//}
Thread.sleep(1000);
}
}
}
class ThreadRunner implements Runnable {
static int timeOutTime = 2;
private ExecutorService executorService = Executors.newFixedThreadPool(10);
private final Phaser barrier;
private boolean isFinished=false;
public ThreadRunner(Phaser phaser) {
this.barrier = phaser;
}
public void run(){
try {
boolean stillLoop = true; int i = 0;
while (stillLoop){
int size;
Future<Integer> future = null;
try {
future = executorService.submit(new Reader(this.barrier)); // sleeps
size = future.get();
} catch (InterruptedException | ExecutionException ex) {
// handle Errs
}
if(i == 3){
stillLoop = false;
this.barrier.awaitAdvance(0);
this.isFinished=true;
}
//System.out.println("i = "+i+" Size is: "+size+"\r");
i++;
}
} catch (Exception e1) {
e1.printStackTrace();
}
}
public boolean getIsFinished(){
return this.isFinished;
}
}
class Reader implements Callable {
private Phaser barrier;
private ExecutorService executorService = Executors.newFixedThreadPool(1);
public Reader(Phaser phase) {
phase.register();
this.barrier = phase;
}
#Override
public Object call() throws Exception {
System.out.println("Reading...");
Thread.sleep(2000);
executorService.submit(new Writer(this.barrier));
this.barrier.arrive();
return 1000;
}
}
class Writer implements Callable {
private Phaser barrier;
public Writer(Phaser phase) {
phase.register();
this.barrier = phase;
}
#Override
public Void call() throws Exception {
Thread.sleep(4000);
System.out.println("Wrote");
this.barrier.arrive();
return null;
}
}

From what I can see you aren't waiting for the Writer to finish in the Reader. Is that the problem you are seeing?
You are also accessing isFinished from more than one thread without synchronization (which however, merely may delay the termination of the loop in this situation).
I don't see CyclicBarrier doing anything.
Not sure what you are trying to do, but I'd think about how simpler I can make it. For example, can Reader and Writer be combined into one task? Then, waiting for them to finish would merely be:
executorService.invokeAll(tasks);
System.out.println("Print metrics");
where tasks is a collection of tasks (see also this javadoc)

Text is not getting printed once the Threads are done [duplicate]

This question already has answers here:
How to wait for all threads to finish, using ExecutorService?
(27 answers)
Closed 8 years ago.
Please have a look at the following code.
public class BigFileWholeProcessor {
private static final int NUMBER_OF_THREADS = 2;
public void processFile(String fileName) {
BlockingQueue<String> fileContent = new LinkedBlockingQueue<String>();
BigFileReader bigFileReader = new BigFileReader(fileName, fileContent);
BigFileProcessor bigFileProcessor = new BigFileProcessor(fileContent);
ExecutorService es = Executors.newFixedThreadPool(NUMBER_OF_THREADS);
es.execute(bigFileReader);
es.execute(bigFileProcessor);
es.shutdown();
if(es.isTerminated())
{
System.out.println("Completed Work");
}
}
}
public class BigFileReader implements Runnable {
private final String fileName;
int a = 0;
public static final String SENTINEL = "SENTINEL";
private final BlockingQueue<String> linesRead;
public BigFileReader(String fileName, BlockingQueue<String> linesRead) {
this.fileName = fileName;
this.linesRead = linesRead;
}
#Override
public void run() {
try {
//since it is a sample, I avoid the manage of how many lines you have read
//and that stuff, but it should not be complicated to accomplish
BufferedReader br = new BufferedReader(new FileReader(new File("E:/Amazon HashFile/Hash.txt")));
String str = "";
while((str=br.readLine())!=null)
{
linesRead.put(str);
System.out.println(a);
a++;
}
linesRead.put(SENTINEL);
} catch (Exception ex) {
ex.printStackTrace();
}
System.out.println("Completed");
}
}
public class BigFileProcessor implements Runnable {
private final BlockingQueue<String> linesToProcess;
public BigFileProcessor (BlockingQueue<String> linesToProcess) {
this.linesToProcess = linesToProcess;
}
#Override
public void run() {
String line = "";
try {
while ( (line = linesToProcess.take()) != null) {
//do what you want/need to process this line...
if(line==BigFileReader.SENTINEL)
{
break;
}
String [] pieces = line.split("(...)/g");
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
I want to print the text "completed work" in BigFileWholeProcessor once all the thread work is done. But instead, it is not getting printed. Why is this? How to identify that all the threads are done and need printing?

shutdown() only signal ES to shutdown, you need
awaitTermination(long timeout, TimeUnit unit)
before print message

Use submit() method instead of execute(). The get() method can be used if you want to wait for the thread to finish at any point of time. Read documentation on use of Future object for further details.
ExecutorService es = Executors.newFixedThreadPool(2);
Future<?> f = es.submit(new Thread(new TestRun()));
f.get(); // Wait for result... (i.e similar to `join()` in this case)
es.shutdown(); // Shutdown ExecutorService
System.out.println("Done.");
I have defined a TestRun class implementing Runnable, not shown here. The Future object makes more sense in other scenarios.

How do we know threadPoolExecutor has finished execution

I have a parent thread that sends messages to MQ and it manages a ThreadPoolExecutor for worker threads which listen to MQ and writes message to output file. I manage a threadpool of size 5. So when I run my program, I have 5 files with messages. Everything works fine until here. I now need to merge these 5 files in my parent thread.
How do I know ThreadPoolExecutor finished processing so I can start merging files.
public class ParentThread {
private MessageSender messageSender;
private MessageReciever messageReciever;
private Queue jmsQueue;
private Queue jmsReplyQueue;
ExecutorService exec = Executors.newFixedThreadPool(5);
public void sendMessages() {
System.out.println("Sending");
File xmlFile = new File("c:/filename.txt");
List<String> lines = null;
try {
lines = FileUtils.readLines(xmlFile, null);
} catch (IOException e) {
e.printStackTrace();
}
for (String line : lines){
messageSender.sendMessage(line, this.jmsQueue, this.jmsReplyQueue);
}
int count = 0;
while (count < 5) {
messageSender.sendMessage("STOP", this.jmsQueue, this.jmsReplyQueue);
count++;
}
}
public void listenMessages() {
long finishDate = new Date().getTime();
for (int i = 0; i < 5; i++) {
Worker worker = new Worker(i, this.messageReciever, this.jmsReplyQueue);
exec.execute(worker);
}
exec.shutdown();
if(exec.isTerminated()){ //PROBLEM is HERE. Control Never gets here.
long currenttime = new Date().getTime() - finishDate;
System.out.println("time taken: "+currenttime);
mergeFiles();
}
}
}
This is my worker class
public class Worker implements Runnable {
private boolean stop = false;
private MessageReciever messageReciever;
private Queue jmsReplyQueue;
private int processId;
private int count = 0;
private String message;
private File outputFile;
private FileWriter outputFileWriter;
public Worker(int processId, MessageReciever messageReciever,
Queue jmsReplyQueue) {
this.processId = processId;
this.messageReciever = messageReciever;
this.jmsReplyQueue = jmsReplyQueue;
}
public void run() {
openOutputFile();
listenMessages();
}
private void listenMessages() {
while (!stop) {
String message = messageReciever.receiveMessage(null,this.jmsReplyQueue);
count++;
String s = "message: " + message + " Recieved by: "
+ processId + " Total recieved: " + count;
System.out.println(s);
writeOutputFile(s);
if (StringUtils.isNotEmpty(message) && message.equals("STOP")) {
stop = true;
}
}
}
private void openOutputFile() {
try {
outputFile = new File("C:/mahi/Test", "file." + processId);
outputFileWriter = new FileWriter(outputFile);
} catch (IOException e) {
System.out.println("Exception while opening file");
stop = true;
}
}
private void writeOutputFile(String message) {
try {
outputFileWriter.write(message);
outputFileWriter.flush();
} catch (IOException e) {
System.out.println("Exception while writing to file");
stop = true;
}
}
}
How will I know when the ThreadPool has finished processing so I can do my other clean up work?
Thanks

If you Worker class implements Callable instead of Runnable, then you'd be able to see when your threads complete by using a Future object to see if the Thread has returned some result (e.g. boolean which would tell you whether it has finished execution or not).
Take a look in section "8. Futures and Callables" # website below, it has exactly what you need imo:
http://www.vogella.com/articles/JavaConcurrency/article.html
Edit: So after all of the Futures indicate that their respective Callable's execution is complete, its safe to assume your executor has finished execution and can be shutdown/terminated manually.

Something like this:
exec.shutdown();
// waiting for executors to finish their jobs
while (!exec.awaitTermination(50, TimeUnit.MILLISECONDS));
// perform clean up work

You can use a thread for monitoring ThreadPoolExecutor like that
import java.util.concurrent.ThreadPoolExecutor;
public class MyMonitorThread implements Runnable {
private ThreadPoolExecutor executor;
private int seconds;
private boolean run=true;
public MyMonitorThread(ThreadPoolExecutor executor, int delay)
{
this.executor = executor;
this.seconds=delay;
}
public void shutdown(){
this.run=false;
}
#Override
public void run()
{
while(run){
System.out.println(
String.format("[monitor] [%d/%d] Active: %d, Completed: %d, Task: %d, isShutdown: %s, isTerminated: %s",
this.executor.getPoolSize(),
this.executor.getCorePoolSize(),
this.executor.getActiveCount(),
this.executor.getCompletedTaskCount(),
this.executor.getTaskCount(),
this.executor.isShutdown(),
this.executor.isTerminated()));
try {
Thread.sleep(seconds*1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
And add
MyMonitorThread monitor = new MyMonitorThread(executorPool, 3);
Thread monitorThread = new Thread(monitor);
monitorThread.start();
to your class where ThreadPoolExecutor is located.
It will show your threadpoolexecutors states in every 3 seconds.