We Keep Coding

Semaphore in Java. producer-consumer problem printer

package producerconsumer;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.Random;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
public class Buffer {
int t;
private final Semaphore notEmptyy = new Semaphore(0); // prevent underflow
private final Semaphore notFulll = new Semaphore(10); // prevent overflow
private int itemn;
private int itemb;
int count=0;
int buffersize =10 ;
private final BlockingQueue<Integer> queue = new ArrayBlockingQueue<Integer>(buffersize);
private final Semaphore mutex = new Semaphore(1); // control buffer access
private final Semaphore notEmpty = new Semaphore(0); // prevent underflow
private final Semaphore notFull = new Semaphore(10); // prevent overflow
public Buffer(){
}
public void add( int x) throws InterruptedException{
while(count== buffersize )
notFulll.acquire();
System.out.println("user printer-request,: " +Thread.currentThread().getName() + " " + x);
queue.offer(x);
count++;
notEmptyy.release();
}
public int take() throws InterruptedException{
while(queue.isEmpty())
notEmptyy.acquire();
t=queue.take();
count--;
notFulll.release();
return t;
}
public void put( ){
while(true){
try {
Random random = new Random();
int data = random.nextInt(100);
notFull .acquire();
mutex .acquire();
add(data);
mutex .release();
notEmpty .release();
// itemb = data;
} catch (InterruptedException e){
System.out.println("InterruptedException caught ");
}
// System.out.println("user printer-request,: " +Thread.currentThread().getName() + " " + itemb);
// this.item = item;
}
}
public void get(){
while(true){
try{
notEmpty .acquire();
mutex .acquire();
itemn = take();
mutex .release();
notFull .release();
queue.remove(itemn);
System.out.println("print request, : "+Thread.currentThread().getName()+" " + itemn );
} catch (InterruptedException e){
System.out.println("InterruptedException caught ");
}
}
}
}
package producerconsumer;
public class producers implements Runnable{
private static final int DELAY = 500;
private Buffer osLabGroup;
public producers(Buffer buffer){
osLabGroup = buffer;
}
public void run(){
// while(true){
osLabGroup.put();
try{
Thread.sleep(DELAY);
}catch (InterruptedException exception) {}
}
}
package producerconsumer;
public class consumers implements Runnable{
private static final int DELAY = 1000;
private Buffer osLabGroup;
public consumers(Buffer buffer){
osLabGroup = buffer;
}
public void run(){
// while(true){
osLabGroup.get();
try{
Thread.sleep(DELAY);
}catch (InterruptedException exception) {}
}
}
//}
package producerconsumer;
public class ProducerConsumer {
public static void main(String[] args) {
Buffer buffer = new Buffer();
producers p1 = new producers(buffer);
consumers c1 = new consumers(buffer);
producers p2 = new producers(buffer);
consumers c2 = new consumers(buffer);
producers p3 = new producers(buffer);
consumers c3 = new consumers(buffer);
Thread pr1 = new Thread(p1);
Thread co1 = new Thread(c1);
Thread pr2 = new Thread(p2);
Thread co2 = new Thread(c2);
Thread pr3 = new Thread(p3);
Thread co3 = new Thread(c3);
pr1.setName("p1");
co1.setName("c1");
pr2.setName("p2");
co2.setName("c2");
pr3.setName("p3");
co3.setName("c3");
pr1.start();
co1.start();
pr2.start();
co2.start();
pr3.start();
co3.start();
}
}
I have a program that simulates the print jobs of a printer, one job at a time: either from Producer or Consumer and it shares an object called a Buffer.
in the buffer it has Set1 of threads, producers, put data with a ½ second delay.
also it does have a Set2 of threads, consumers, read data from the buffer with one second delay.
as it stands, I am trying to follow this guy https://www.youtube.com/watch?v=nxw2y27z0V4&t=1207s
using semaphore this is my code
I problem my output is not accurate enter image description here
this my edited code
package producerconsumer;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
public class Buffer {
int t;
private final Semaphore notEmptyy = new Semaphore(0); // prevent underflow
private final Semaphore notFulll = new Semaphore(10); // prevent overflow
private int itemn;
private int itemb;
int count=0;
int buffersize =10 ;
private final ArrayList<Integer> list = new ArrayList<Integer>(buffersize);
private final LinkedList<Integer> queue = new LinkedList<Integer>(list);
private final Semaphore mutex = new Semaphore(1); // control buffer access
private final Semaphore notEmpty = new Semaphore(0); // prevent underflow
private final Semaphore notFull = new Semaphore(10); // prevent overflow
public Buffer(){
}
public void add( int x) throws InterruptedException{
while(count== buffersize )
notFulll.acquire();
System.out.println("user printer-request,: " +Thread.currentThread().getName() + " " + x);
queue.offer(x);
count++;
notEmptyy.release();
}
public int take() throws InterruptedException{
while(count ==0)
notEmptyy.acquire();
t=queue.pollFirst();
count--;
notFulll.release();
return t;
}
public void put( ){
while(true){
try {
Random random = new Random();
int data = random.nextInt(100);
notFull.acquire();
mutex.acquire();
add(data);
mutex.release();
notEmpty.release();
// itemb = data;
} catch (InterruptedException e){
System.out.println("InterruptedException caught ");
}
// System.out.println("user printer-request,: " +Thread.currentThread().getName() + " " + itemb);
// this.item = item;
}
}
public void get(){
while(true){
try{
notEmpty.acquire();
mutex.acquire();
itemn = take();
mutex.release();
notFull.release();
// queue.remove(itemn);
System.out.println("print request, : "+Thread.currentThread().getName()+" " + itemn );
} catch (InterruptedException e){
System.out.println("InterruptedException caught ");
}
}
}
}
i get this output enter image description here
and this output enter image description here

In Buffer#get, when you take an item from queue, you call queue#remove again. This is a wrong behavior. You should consume this item instead of operating the queue outside of the concurrency control code. This can lead to abnormal behaviors, such as deadlock.
try {
notEmpty .acquire();
mutex .acquire();
itemn = take();
mutex .release();
notFull .release();
// why remove from queue again? It is already taken from queue. Comment this line
// queue.remove(itemn);
System.out.println("print request, : "+Thread.currentThread().getName()+" " + itemn );
} catch (InterruptedException e){
System.out.println("InterruptedException caught ");
}
In addition, in your program, you should replace ArrayBlockingQueue with a collection class under java.util package, such as LinkedList.
ArrayBlockingQueue you use is actually a thread-safe blocking queue. You can directly use take&put to complete this program without other concurrency control.
EDIT:
As mentioned in the video introduction, there are two ways to solve the producer-consumer problem, semaphore or monitor. Your code seems to mix these two together. There is no need to do any concurrency control in your add and take method.
Code should be:
public class consumers implements Runnable{
private static final int DELAY = 1000;
private Buffer osLabGroup;
public consumers(Buffer buffer){
osLabGroup = buffer;
}
public void run(){
while (true) {
osLabGroup.get();
try{
Thread.sleep(DELAY);
}catch (InterruptedException exception) {}
}
}
}
public class producers implements Runnable{
private static final int DELAY = 500;
private Buffer osLabGroup;
public producers(Buffer buffer){
osLabGroup = buffer;
}
public void run(){
while(true) {
osLabGroup.put();
try {
Thread.sleep(DELAY);
} catch (InterruptedException exception) {
}
}
}
}
import java.util.LinkedList;
import java.util.Random;
import java.util.concurrent.Semaphore;
public class Buffer {
int buffersize =10 ;
private final LinkedList<Integer> queue = new LinkedList<Integer>();
private final Semaphore mutex = new Semaphore(1); // control buffer access
private final Semaphore notEmpty = new Semaphore(0); // prevent underflow
private final Semaphore notFull = new Semaphore(buffersize); // prevent overflow
public Buffer(){
}
public void add( int x) throws InterruptedException{
System.out.println("user printer-request,: " +Thread.currentThread().getName() + " " + x + "," + System.currentTimeMillis());
queue.add(x);
}
public int take() throws InterruptedException{
Integer first = queue.removeFirst();
System.out.println("print request, : "+Thread.currentThread().getName()+" " + first + "," + System.currentTimeMillis() );
return first;
}
public void put(){
try {
Random random = new Random();
int data = random.nextInt(100);
notFull.acquire();
mutex.acquire();
add(data);
mutex.release();
notEmpty.release();
} catch (InterruptedException e){
System.out.println("InterruptedException caught ");
}
}
public void get(){
try{
notEmpty.acquire();
mutex.acquire();
int take = take();
mutex.release();
notFull.release();
} catch (InterruptedException e){
System.out.println("InterruptedException caught ");
}
}
}

simple barrier synchronisation in java

im trying to understand the barrier problem. im still really new to programming but was presented with this problem in class to solve.
"I have to solve the barrier problem using counting semaphores. You can assume that there is a shared variable N which indicates the number of concurrent threads in the system. When the first N −1 threads arrive at the barrier, they should block until the Nth thread arrives, at which point all threads might proceed.
The shared counter variable can be used to keep track of the number of threads that have arrived and the semaphores mutex and barrier can be used to solve the synchronization problem."
import java.util.concurrent.Semaphore;
public class BarrierSynchronization extends Thread {
int N;
int count;
Semaphore mutex;
Semaphore barrier;
public BarrierSynchronization ()
{
this.N = 5;
this.count = 0;
this.mutex = new Semaphore(1);
this.barrier = new Semaphore(0);
}
public void run()
{
try {
mutex.acquire();
count = count + 1;
System.out.println(Thread.currentThread().getName() + ": " + count);
mutex.release();
if (count == N)
{
barrier.release();
System.out.println("All " + count + " threads have reached the barrier. The barrier is now open" );
} // unblock one thread
barrier.acquire();
barrier.release();
System.out.println(Thread.currentThread().getName() + " has passed the barrier");
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
i tried to implement the pseudo code from the little book of semaphores. i called on this thread in main class and ran it but it gives an error about mutex.wait() for some reason. when i remove that bit of the code it runs but shows nothing. what exactly am i supposed to do for this problem?
public class Main {
public static void main(String[] args) throws InterruptedException
{
BarrierSynchronization barrier = new BarrierSynchronization();
Thread bs1 = new Thread(barrier);
Thread bs2 = new Thread(barrier);
Thread bs3 = new Thread(barrier);
Thread bs4 = new Thread(barrier);
Thread bs5 = new Thread(barrier);
bs1.start();
bs2.start();
bs3.start();
bs4.start();
bs5.start();
}
why does it output the rare one before the barrier is unlocked for all the threads? i think im close enough to solving this problem. is a race condition or something?
CLICK TO SEE IMAGE

Might be a bit late, but here is an implementation with a driver code that works. You have to garante mutual exclusion and keep track of the number of threads that arrive at the barrier.
public class Barrier {
private int capacity;
private Semaphore s, exclusao, counter;
public Barrier(int capacity) {
this.capacity = capacity;
counter = new Semaphore(0);
s = new Semaphore(0);
exclusao = new Semaphore(1);
}
public void espera() throws InterruptedException {
exclusao.acquire();
if (counter.availablePermits() < capacity - 1) {
counter.release();
exclusao.release();
s.acquire();
} else {
exclusao.release();
System.out.println("RELEASE ALL");
for (int i = 0; i < capacity; i++) {
s.release();
}
}
}
}
class TesteThread extends Thread {
private Barrier b;
private long waitPeriod;
public TesteThread(long wait, Barrier b) {
this.b = b;
this.waitPeriod = wait;
System.out.println("Thread started" + this.getName());
}
public void espera() throws InterruptedException {
b.espera();
}
#Override
public void run() {
try {
System.out.println("Thread a dormir " + this.getName());
sleep(waitPeriod);
System.out.println("Thread a esperar " + this.getName());
espera();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
class BarrierExample {
public static void main(String[] args) throws InterruptedException {
Barrier BR = new Barrier(5);
TesteThread[] teste = new TesteThread[5];
for (int i = 0; i < teste.length; i++) {
teste[i] = new TesteThread((long) (Math.random() * 1000), BR);
teste[i].start();
}
for (int i = 0; i < teste.length; i++) {
teste[i].join();
}
}
}`package examesFSO.exame2020_normal;
import java.util.concurrent.Semaphore;
public class Barrier {
private int capacity;
private Semaphore s, exclusao, counter;
public Barrier(int capacity) {
this.capacity = capacity;
counter = new Semaphore(0);
s = new Semaphore(0);
exclusao = new Semaphore(1);
}
public void espera() throws InterruptedException {
exclusao.acquire();
if (counter.availablePermits() < capacity - 1) {
counter.release();
exclusao.release();
s.acquire();
} else {
System.out.println("RELEASE ALL");
for (int i = 0; i < capacity; i++) {
s.release();
}
}
exclusao.release();
}
}
class TesteThread extends Thread {
private Barrier b;
private long waitPeriod;
public TesteThread(long wait, Barrier b) {
this.b = b;
this.waitPeriod = wait;
System.out.println("Thread instanciada " + this.getName());
}
public void espera() throws InterruptedException {
b.espera();
}
#Override
public void run() {
try {
System.out.println("Thread a dormir " + this.getName());
sleep(waitPeriod);
System.out.println("Thread a esperar " + this.getName());
espera();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
class BarrierExample {
public static void main(String[] args) throws InterruptedException {
Barrier BR = new Barrier(5);
TesteThread[] teste = new TesteThread[5];
for (int i = 0; i < teste.length; i++) {
teste[i] = new TesteThread((long) (Math.random() * 1000), BR);
teste[i].start();
}
for (int i = 0; i < teste.length; i++) {
teste[i].join();
}
}
}

Java Semaphore Acquire Order Based on Thread Value

Good afternoon everyone,
I am working on a school project that requires me to use semaphores to control access to resources. From what I have developed so far, they are:
Semaphore 1) Waiting Area - This permits only 15 customers (Threads) to enter the waiting area, else they are rejected from the store (using TryAcquire).
Semaphore 2) ServerQueue - This permits customers (Threads) to use the only 3 servers in the restaurant once in the waiting area.
My Problem: Our professor requires the serverQueue to take the shortest order (IE, the thread with the least amount of burritosOrdered) when in the waitingArea.
Full flow of application:
Main method instantiates a serverQueue (3 servers) and a waitingArea (15 customers)
Main method instantiates and starts 20 customer threads
Each customer (Thread) run function has been overridden to attempt to get in the waiting area
Each customer in the waitingArea tries to access a server in the serverQueue
How can I tell the serverQueue to get the shortest order? Because the threads override the run, I don't have direct access to an array of all the threads to compare their values.
Thank you for taking a look!
Main
public class Main {
private static final int numCustomers = 5;
public static void main(String[] args)
{
ServerQueue serverQueue = new ServerQueue();
WaitingArea waitingArea = new WaitingArea(3, serverQueue);
Thread customers[] = new Thread[numCustomers];
for (int i = 0; i < numCustomers; i++)
{
customers[i] = new Thread(new Customer(waitingArea), "Customer " + i);
}
for (int i = 0; i < numCustomers; i++)
{
customers[i].start();
}
}
}
Customer
import java.util.Date;
import java.util.Random;
// Runnable is an interface that facilitates threads
public class Customer implements Runnable {
// The semaphore
// private ServerQueue serverQueue;
private WaitingArea waitingArea;
public int burritosOrdered;
public int burritosMade = 0;
// Constructor, allow semaphore to be passed/assigned
public Customer(WaitingArea waitingArea) {
this.waitingArea = waitingArea;
Random r = new Random();
this.burritosOrdered = r.nextInt(21);
}
public void setBurritosMade(int newBurritos) {
this.burritosMade += newBurritos;
}
// We must override the run function within Runnable
// The run function is called by threadObject.start();
#Override
public void run() {
waitingArea.seatCustomer(burritosOrdered);
}
}
waitingArea
import java.util.Date;
import java.util.concurrent.Semaphore;
public class WaitingArea {
private Semaphore semaphore;
private ServerQueue serverQueue;
private int maxCustomers;
public WaitingArea(int maxCustomers, ServerQueue serverQueue) {
semaphore = new Semaphore(maxCustomers, true);
this.serverQueue = serverQueue;
this.maxCustomers = maxCustomers;
}
public void seatCustomer(int burritosOrdered)
{
boolean hasPermit = false;
try
{
hasPermit = semaphore.tryAcquire();
if(hasPermit) {
System.out.println(new Date() + " - "
+ Thread.currentThread().getName()
+ " entered store ordering "
+ burritosOrdered + " burritos");
serverQueue.finishOrder();
} else {
System.out.println(new Date() + " - " + Thread.currentThread().getName() + " left due to full shop");
}
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
if(hasPermit) {
semaphore.release();
System.out.println(new Date() + " - "
+ Thread.currentThread().getName()
+ " left with " + burritosOrdered + " burritos made");
}
}
}
}
serverQueue
import java.util.Date;
import java.util.concurrent.Semaphore;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
public class ServerQueue {
// This Semaphore will keep track of no. of servers used at any point.
private final Semaphore semaphore;
// While checking/acquiring a free server out of three available servers, we will use this lock.
private final Lock serverLock;
// This array represents the pool of free server.
private boolean freeServers[];
public ServerQueue() {
semaphore = new Semaphore(1, true);
freeServers = new boolean[1];
serverLock = new ReentrantLock();
// Set all servers to available
for(int i=0;i<freeServers.length;i++) {
freeServers[i] = true;
}
}
public void finishOrder() throws InterruptedException {
try {
System.out.println(semaphore.getClass());
// Decrease the semaphore counter to mark a printer busy
semaphore.acquire();
// Get the server printer
int assignedServer = getServer();
Thread.sleep(3000);
// Print the job
System.out.println(new Date() + " - " + Thread.currentThread().getName()
+ " is getting service from server " + assignedServer);
//Server is done; Free the server to be used by other threads.
releaseServer(assignedServer);
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
System.out.print(new Date() + " - " + Thread.currentThread().getName() + " has been served\n");
//Increase the semaphore counter back
semaphore.release();
}
}
//Acquire a free server to finish a job
private int getServer() {
int foundServer = -1;
try {
//Get a lock here so that only one thread can go beyond this at a time
serverLock.lock();
//Check which server is free
for (int i=0; i<freeServers.length; i++)
{
//If free server found then mark it busy
if (freeServers[i])
{
foundServer = i;
freeServers[i] = false;
break;
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
//Allow other threads to check for free servers
serverLock.unlock();
}
return foundServer;
}
//Release the server
private void releaseServer(int i) {
serverLock.lock();
//Mark the server as free
freeServers[i] = true;
serverLock.unlock();
}
}

Why this ReadWriteLock example dosen't work?

LinkedList throws exception when trying to poll data. But I think i correctly use read/write lock concept. What is wrong with that code?
package sample;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
public class PingPong extends Thread {
boolean read = false;
Queue<String> queue;
static ReadWriteLock lock = new ReentrantReadWriteLock();
final static Lock readLock = lock.readLock();
final static Lock writeLock = lock.writeLock();
boolean stop;
public PingPong(boolean read, Queue<String> queue) {
this.read = read;
this.queue = queue;
}
int count = 0;
#Override
public String toString() {
return "PingPong{" +
"read=" + read +
", count=" + count +
'}';
}
#Override
public void run() {
if (read) {
while (!stop) {
readLock.lock();
// synchronized (queue) {
try {
String string = queue.poll();
if (string != null) {
count++;
}
} finally {
readLock.unlock();
}
// }
inform();
}
} else {
while (!stop) {
writeLock.lock();
// synchronized (queue) {
try {
if (queue.add("some str" + count)) {
count++;
}
} finally {
writeLock.unlock();
}
// }
inform();
}
}
}
private void inform() {
// Thread.yield();
// synchronized (queue) {
// queue.notify();
// try {
// queue.wait(1);
// } catch (InterruptedException e) {
// e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
// }
// }
}
public static void main(String[] args) throws InterruptedException {
Queue<String> queue = new LinkedList();
// queue = new ArrayBlockingQueue<String>(100);
// queue = new ConcurrentLinkedQueue<String>();
List<PingPong> pongs = new ArrayList<PingPong>();
for (int i = 0; i < 10; ++i) {
PingPong pingPong = new PingPong(i % 2 == 0, queue);
pingPong.start();
pongs.add(pingPong);
}
Thread.sleep(1000);
int sum = 0;
int read = 0;
int write = 0;
for (PingPong pp : pongs) {
pp.stop = true;
pp.join();
}
for (PingPong pp : pongs) {
System.out.println(pp);
sum += pp.count;
if (pp.read) read += pp.count;
else write += pp.count;
}
System.out.println(sum);
System.out.println("write = " + write);
System.out.println("read = " + read);
System.out.println("queue.size() = " + queue.size());
System.out.println("balance (must be zero) = " + (write - read - queue.size()));
}
}

It's because this call mutates the queue collection:
String string = queue.poll();
From Queue JavaDoc:
Retrieves and removes the head of this queue, or returns null if this queue is empty.
Read locks are meant to be used in situations where multiple threads can safely read, while writes have to be performed exclusively (no other reads and writes). Because you are using read lock to poll the queue (write operation!), you are effectively allowing multiple threads to modify non thread-safe LinkedList concurrently.
Read-write lock isn't the correct synchronization mechanism in this case.

Make an existing code in Java parallel/multithread

I have a very simple crawler. I want to make my current code run in a few threads. Could you provide me a little tutorial or article to help me achive this test?
I'm originally a .Net developer and in .Net I have no problem whatsoever running codes in multithread but unfortunately I don't know anything about threads in Java.
My crawler is a command-line software so don't worry about GUI.
Thank you in advance.

Java does multithreading through the Thread class. One of the most common ways to make existing code multithreaded is to use the Runnable interface to define what you want to call at thread start, and then start it off.
public class SomeFunctions
{
public static void FunctionA() {}
public static void FunctionB() {}
public static void FunctionC() {}
}
// ...
Thread t1 = new Thread(new Runnable() {
public void run() {
SomeFunctions.FunctionA();
}
});
t1.start();
// (rinse and repeat for the other functions)
Dry coded, but it should at least get the general concept across. Of course, as soon as you go into multithreading land, you have concurrency issues and need to make sure everything is appropriately syhchronized, etc., but any language will have those issues.
If you're worried about synchronization, you have a few tools at your disposal. The easiest is the recursive mutex functionality built into Java, the "synchronized" keyword. More classical means are also available through various classes in the java.util.concurrent and java.util.concurrent.locks packages such as Semaphore and ReadWriteLock
http://download.oracle.com/javase/6/docs/api/java/util/concurrent/package-summary.html
http://download.oracle.com/javase/6/docs/api/java/util/concurrent/locks/package-summary.html

You can take a look at my webcrawler example. Sry for the lengthiness.
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
* A web crawler with a Worker pool
*
* #author Adriaan
*/
public class WebCrawler implements Manager {
private Set<Worker> workers = new HashSet<Worker>();
private List<String> toCrawl = new ArrayList<String>();
private Set<String> crawled = new HashSet<String>();
private Set<String> hosts = new HashSet<String>();
private Set<String> results = new HashSet<String>();
private int maxResults;
public WebCrawler(String url, int numberOfWorkers, int maxResults) {
this.maxResults = maxResults;
toCrawl.add(url);
createWorkers(numberOfWorkers);
}
public void createWorkers(int numberOfWorkers) {
for (int i = 0; i < numberOfWorkers; i++) {
workers.add(new Worker(this));
}
}
private void stopWorkers() {
for (Worker worker : workers) {
worker.terminate();
}
}
public synchronized Job getNewJob() {
while (toCrawl.size() == 0) {
try {
wait();
} catch (InterruptedException e) {
// ignore
}
}
return new EmailAddressCrawlJob().setDescription(toCrawl.remove(0));
}
public synchronized void jobCompleted(Job job) {
// System.out.println("crawled: " + job.getDescription());
crawled.add(job.getDescription());
String host = getHost(job.getDescription());
boolean knownHost = hosts.contains(host);
if (!knownHost) {
System.out.println("host: " + host);
hosts.add(host);
}
for (String url : job.getNewDescriptions()) {
if (!crawled.contains(url)) {
if (knownHost) {
toCrawl.add(toCrawl.size() - 1, url);
} else {
toCrawl.add(url);
}
}
}
for (String result : job.getResults()) {
if (results.add(result)) {
System.out.println("result: " + result);
}
}
notifyAll();
if (results.size() >= maxResults) {
stopWorkers();
System.out.println("Crawled hosts:");
for (String crawledHost : hosts) {
System.out.println(crawledHost);
}
Set<String> uncrawledHosts = new HashSet<String>();
for (String toCrawlUrl : toCrawl) {
uncrawledHosts.add(getHost(toCrawlUrl));
}
System.out.println("Uncrawled hosts:");
for (String unCrawledHost : uncrawledHosts) {
System.out.println(unCrawledHost);
}
}
if (crawled.size() % 10 == 0) {
System.out.println("crawled=" + crawled.size() + " toCrawl="
+ toCrawl.size() + " results=" + results.size() + " hosts="
+ hosts.size() + " lastHost=" + host);
}
}
public String getHost(String host) {
int hostStart = host.indexOf("://") + 3;
if (hostStart > 0) {
int hostEnd = host.indexOf("/", hostStart);
if (hostEnd < 0) {
hostEnd = host.length();
}
host = host.substring(hostStart, hostEnd);
}
return host;
}
public static void main(String[] args) throws MalformedURLException {
new WebCrawler("http://www.nu.nl/", 5, 20);
}
}
Worker
**
* A Worker proactively gets a Job, executes it and notifies its manager that
* the Job is completed.
*
* #author Adriaan
*/
public class Worker extends Thread {
private final Manager manager;
private Job job = null;
private boolean isWorking;
public Worker(Manager manager) {
this.manager = manager;
isWorking = true;
start();
}
#Override
public void run() {
System.out.println("Worker " + Thread.currentThread().getId()
+ " starting ");
while (isWorking) {
job = manager.getNewJob();
job.execute();
manager.jobCompleted(job);
}
}
public void terminate() {
isWorking = false;
}
}
Manager interface
/**
* Manager interface for Workers
*
* #author Adriaan
*/
public interface Manager {
/**
* Gets a new job
*
* #return
*/
public Job getNewJob();
/**
* Indicates the job is completed
*
* #param job
*/
public void jobCompleted(Job job);
}
Job
import java.util.HashSet;
import java.util.Set;
/**
* A Job is a unit of work defined by a String (the description). During execution the
* job can obtain results and new job descriptions.
*
* #author Adriaan
*/
public abstract class Job {
private String description;
private Set<String> results = new HashSet<String>();
private Set<String> newDescriptions = new HashSet<String>();
/**
* Sets the job description
*
* #param description
* #return this for chaining
*/
public Job setDescription(String description) {
this.description = description;
return this;
}
/**
* Executes the job
*/
public abstract void execute();
/**
* Gets the results obtained
*
* #return
*/
public Set<String> getResults() {
return results;
}
/**
* Gets the now job descriptions obtained
*
* #return
*/
public Set<String> getNewDescriptions() {
return newDescriptions;
}
/**
* Gets the job description
*
* #return
*/
public String getDescription() {
return description;
}
/**
* Allows the implementation to add an obtained result
*
* #param result
*/
void addResult(String result) {
results.add(result);
}
/**
* Allows the implementation to add an obtained description
*
* #param result
*/
void addNewDescription(String newDescription) {
newDescriptions.add(newDescription);
}
}
A Job which crawls a page for email addresses:
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* A Job which crawls HTTP or HTTPS URL's for email adresses, collecting new
* URL's to crawl along the way.
*
* #author Adriaan
*/
public class EmailAddressCrawlJob extends Job {
#Override
public void execute() {
try {
URL url = new URL(getDescription());
if (url != null) {
String text = readText(url);
extractNewDescriptions(text, url);
extractResults(text);
}
} catch (MalformedURLException e) {
System.err.println("Bad url " + getDescription());
}
}
private String readText(URL url) {
URLConnection connection;
try {
connection = url.openConnection();
InputStream input = connection.getInputStream();
byte[] buffer = new byte[1000];
int num = input.read(buffer);
if (num > 0) {
StringBuilder builder = new StringBuilder();
builder.append(new String(buffer, 0, num));
while (num != -1) {
num = input.read(buffer);
if (num != -1) {
builder.append(new String(buffer, 0, num));
}
}
return builder.toString();
}
} catch (IOException e) {
//System.err.println("Could not read from " + url);
}
return "";
}
private void extractNewDescriptions(String text, URL url) {
// URL extracting code from Sun example
String lowerCaseContent = text.toLowerCase();
int index = 0;
while ((index = lowerCaseContent.indexOf("<a", index)) != -1) {
if ((index = lowerCaseContent.indexOf("href", index)) == -1) {
break;
}
if ((index = lowerCaseContent.indexOf("=", index)) == -1) {
break;
}
index++;
String remaining = text.substring(index);
StringTokenizer st = new StringTokenizer(remaining, "\t\n\r\">#");
String strLink = st.nextToken();
if (strLink.startsWith("javascript:")) {
continue;
}
URL urlLink;
try {
urlLink = new URL(url, strLink);
strLink = urlLink.toString();
} catch (MalformedURLException e) {
// System.err.println("Could not create url: " + target
// + " + " + strLink);
continue;
}
// only look at http links
String protocol = urlLink.getProtocol();
if (protocol.compareTo("http") != 0
&& protocol.compareTo("https") != 0) {
// System.err.println("Ignoring: " + protocol
// + " protocol in " + urlLink);
continue;
}
addNewDescription(urlLink.toString());
}
}
private void extractResults(String text) {
Pattern p = Pattern
.compile("([\\w\\-]([\\.\\w])+[\\w]+#([\\w\\-]+\\.)+[A-Za-z]{2,4})");
Matcher m = p.matcher(text);
while (m.find()) {
addResult(m.group(1));
}
}
}
I know this answer is a bit verbose, but I thought OP might be best helped with a working example and I happened to have made one not so long ago.

A very basic java program that will give the abstract idea of the Multi Threading..
public class MyThread extends Thread {
String word;
public MyThread(String rm){
word = rm;
}
public void run(){
try {
for(;;){
System.out.println(word);
Thread.sleep(1000);
}
} catch(InterruptedException e) {
System.out.println("sleep interrupted");
}
}
public static void main(String[] args) {
Thread t1=new MyThread("First Thread");
Thread t2=new MyThread("Second Thread");
t1.start();
t2.start();
}
}
And the Output will be..
First Thread
Second Thread
First Thread
Second Thread
First Thread
Go with this PPT it will help you with the basics..
Here