We Keep Coding

Java Pusher Client disconnects right after connecting

I am connecting to Pusher server using JAVA code and the issue is the client (using Java Client library) disconnects after few seconds of making connection to the server and it does not reconnect itself.
Also, the connection status at the java-client shows is CONNECTED, as shown in the onConnectionStateChange() callback method but in the background it seems like it stays disconnected because when I tried pushing a request from the server java client did not receive anything but on reconnect (manually) I get the request.
Code:
#Component
#Slf4j
public class PosWebSocketClient {
private Pusher pusher;
private boolean isConnected = false;
private String pusherAppKey;
private Timer timer;
private Date activeTime;
private boolean isUserLoggedIn = false;
private PusherOptions pusherOptions;
public synchronized void init(String pusherAppKey) {
log.info("Initializing Pusher");
pusher = new Pusher(pusherAppKey/* , pusherOptions() */);
this.pusherAppKey = pusherAppKey;
this.isUserLoggedIn = true;
pusher.connect(new ConnectionEventListener() {
#Override
public void onConnectionStateChange(ConnectionStateChange change) {
log.info("State changed to " + change.getCurrentState() + " from "
+ change.getPreviousState());
if (change.getCurrentState() == ConnectionState.CONNECTED) {
isConnected = true;
} else {
isConnected = false;
}
log.info("isConnected {}", isConnected);
}
#Override
public void onError(String message, String code, Exception e) {
log.info("Error while connecting to the server with {} {} {}", message, code, e.getMessage());
log.error("Exception: - ",e);
}
}, ConnectionState.ALL);
Channel channel = pusher.subscribe("*****");
channel.bind("any-event-1", sendDataListener());
channel.bind("any-event-2", receiveOrderListener());
channel.bind("any-event-3", logOutListener());
channel.bind("any-event-4", getOrderStatusListener());
activeTime = new Date();
/*
* if (timer == null) { timer = new Timer(); timer.schedule(new MyTask(), 0,
* 1000 * 60 * 2); }
*/
}
class MyTask extends TimerTask {
public void run() {
long idleTimeInMinutes = (new Date().getTime() - activeTime.getTime()) / (1000 * 60);
log.info("Pusher Idle Time {} ", idleTimeInMinutes);
if (isUserLoggedIn && idleTimeInMinutes >= 10 && StringUtils.isNotBlank(pusherAppKey)) {
log.info("Pusher idle time is greater than 10 mins, initializing again");
disconnect();
init(pusherAppKey);
}
}
}
private SubscriptionEventListener logOutListener() {
return new SubscriptionEventListener() {
#Override
public void onEvent(PusherEvent pusherEvent) {
}
};
}
private SubscriptionEventListener sendDataListener() {
return new SubscriptionEventListener() {
#Override
public void onEvent(PusherEvent pusherEvent) {
log.info("Received SendData event");
}
};
}
private SubscriptionEventListener receiveOrderListener() {
log.info("Received FetchOrder event");
return new SubscriptionEventListener() {
}
};
}
private SubscriptionEventListener getOrderStatusListener() {
log.info("Received SendStatus event");
return new SubscriptionEventListener() {
}
};
}
public synchronized void disconnect() {
// Disconnect from the service (or become disconnected my network conditions)
if (pusher != null && pusher.getConnection() != null) {
log.info("Disconnecting Pusher");
Channel channel = pusher.getChannel("*****");
channel.unbind("any-event-1", sendDataListener());
channel.unbind("any-event-2", receiveOrderListener());
channel.unbind("any-event-3", logOutListener());
channel.unbind("any-event-4", getOrderStatusListener());
pusher.unsubscribe("*****");
pusher.disconnect();
}
}
public void restart() {
if (StringUtils.isNotBlank(pusherAppKey)) {
log.info("Restarting Pusher");
disconnect();
this.init(this.pusherAppKey);
}
}
/*
* private PusherOptions pusherOptions() { if (pusherOptions == null) {
* pusherOptions = new PusherOptions();
* pusherOptions.setMaxReconnectionAttempts(1000000); } return pusherOptions; }
*/
public boolean isConnected() {
return isConnected;
}
private void setPusherStatus() {
activeTime = new Date();
}
public void userLoggedOut() {
this.isUserLoggedIn = false;
}
}
The maven dependency used is
<dependency>
<groupId>com.pusher</groupId>
<artifactId>pusher-java-client</artifactId>
<version>2.0.0</version>
</dependency>
Can anyone please have a look and let me know the issue with the code or dependency or any property I am missing while making connection to the server ? TIA.

WorkStealingPool exits unexpectedly

I submitted some Runnables to an ExecutorService. Inside these Runnables, wait() and notify() are called. The code works with newFixedThreadPool as the ExecutorService. With newWorkStealingPool, the process exits unexpectedly without any error message.
import java.net.URL;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.IOException;
// For regular expressions
import java.util.regex.Matcher;
import java.util.regex.MatchResult;
import java.util.regex.Pattern;
import java.util.*;
import java.util.concurrent.*;
public class TestPipeline {
public static void main(String[] args) {
runAsThreads();
}
private static void runAsThreads() {
final BlockingQueue<String> urls = new OneItemQueue<String>();
final BlockingQueue<Webpage> pages = new OneItemQueue<Webpage>();
final BlockingQueue<Link> refPairs = new OneItemQueue<Link>();
final BlockingQueue<Link> uniqRefPairs = new OneItemQueue<Link>();
final ExecutorService executor = Executors.newWorkStealingPool(6);
// final ExecutorService executor = Executors.newFixedThreadPool(6);
executor.submit(new UrlProducer(urls));
executor.submit(new PageGetter(urls, pages));
executor.submit(new LinkScanner(pages,refPairs));
executor.submit(new Uniquifier<Link>(refPairs,uniqRefPairs));
executor.submit(new LinkPrinter(uniqRefPairs));
}
}
class UrlProducer implements Runnable {
private final BlockingQueue<String> output;
public UrlProducer(BlockingQueue<String> output) {
this.output = output;
}
public void run() {
System.out.println("in producer");
for (int i=0; i<urls.length; i++)
output.put(urls[i]);
}
private static final String[] urls =
{ "http://www.itu.dk", "http://www.di.ku.dk", "http://www.miele.de",
"http://www.microsoft.com", "http://www.cnn.com", "http://www.dr.dk",
"http://www.vg.no", "http://www.tv2.dk", "http://www.google.com",
"http://www.ing.dk", "http://www.dtu.dk", "http://www.bbc.co.uk"
};
}
class PageGetter implements Runnable {
private final BlockingQueue<String> input;
private final BlockingQueue<Webpage> output;
public PageGetter(BlockingQueue<String> input, BlockingQueue<Webpage> output) {
this.input = input;
this.output = output;
}
public void run() {
while (true) {
System.out.println("in pagegetter");
String url = input.take();
// System.out.println("PageGetter: " + url);
try {
String contents = getPage(url, 200);
output.put(new Webpage(url, contents));
} catch (IOException exn) { System.out.println(exn); }
}
}
public static String getPage(String url, int maxLines) throws IOException {
// This will close the streams after use (JLS 8 para 14.20.3):
try (BufferedReader in
= new BufferedReader(new InputStreamReader(new URL(url).openStream()))) {
StringBuilder sb = new StringBuilder();
for (int i=0; i<maxLines; i++) {
String inputLine = in.readLine();
if (inputLine == null)
break;
else
sb.append(inputLine).append("\n");
}
return sb.toString();
}
}
}
class Uniquifier<T> implements Runnable{
private final Set<T> set = new HashSet<T>();
private final BlockingQueue<T> input;
private final BlockingQueue<T> output;
public Uniquifier(BlockingQueue<T> input, BlockingQueue<T> output){
this.input = input;
this.output = output;
}
public void run(){
while(true){
System.out.println("in uniquifier");
T item = input.take();
if(!set.contains(item)){
set.add(item);
output.put(item);
}
}
}
}
class LinkScanner implements Runnable {
private final BlockingQueue<Webpage> input;
private final BlockingQueue<Link> output;
public LinkScanner(BlockingQueue<Webpage> input,
BlockingQueue<Link> output) {
this.input = input;
this.output = output;
}
private final static Pattern urlPattern
= Pattern.compile("a href=\"(\\p{Graph}*)\"");
public void run() {
while (true) {
System.out.println("in link scanner");
Webpage page = input.take();
// System.out.println("LinkScanner: " + page.url);
// Extract links from the page's <a href="..."> anchors
Matcher urlMatcher = urlPattern.matcher(page.contents);
while (urlMatcher.find()) {
String link = urlMatcher.group(1);
output.put(new Link(page.url, link));
}
}
}
}
class LinkPrinter implements Runnable {
private final BlockingQueue<Link> input;
public LinkPrinter(BlockingQueue<Link> input) {
this.input = input;
}
public void run() {
while (true) {
System.out.println("in link printer");
Link link = input.take();
// System.out.println("LinkPrinter: " + link.from);
System.out.printf("%s links to %s%n", link.from, link.to);
}
}
}
class Webpage {
public final String url, contents;
public Webpage(String url, String contents) {
this.url = url;
this.contents = contents;
}
}
class Link {
public final String from, to;
public Link(String from, String to) {
this.from = from;
this.to = to;
}
// Override hashCode and equals so can be used in HashSet<Link>
public int hashCode() {
return (from == null ? 0 : from.hashCode()) * 37
+ (to == null ? 0 : to.hashCode());
}
public boolean equals(Object obj) {
Link that = obj instanceof Link ? (Link)obj : null;
return that != null
&& (from == null ? that.from == null : from.equals(that.from))
&& (to == null ? that.to == null : to.equals(that.to));
}
}
// Different from java.util.concurrent.BlockingQueue: Allows null
// items, and methods do not throw InterruptedException.
interface BlockingQueue<T> {
void put(T item);
T take();
}
class OneItemQueue<T> implements BlockingQueue<T> {
private T item;
private boolean full = false;
public void put(T item) {
synchronized (this) {
while (full) {
try { this.wait(); }
catch (InterruptedException exn) { }
}
full = true;
this.item = item;
this.notifyAll();
}
}
public T take() {
synchronized (this) {
while (!full) {
try { this.wait(); }
catch (InterruptedException exn) { }
}
full = false;
this.notifyAll();
return item;
}
}
}

Because the Pool is allocating threads dynamically, there are no threads alive after runAsThreads exits because that's the end of the main thread. There needs to be at least on thread running to keep the application alive. Adding a call to awaitTermination is needed. It's not needed for the fixed pool because that will always have active threads until it is explicitly shut down as noted in the JavaDocs.

Java Thread Don't kill other Threads and wont remove from vector item

I have Vector of threads, and i wanna check all items in this vector. Everyone item is connection of user to server. I wanna "clean" all dead connections.
I can't find where I'm wrong.
Here is it my code :
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package server;
import java.util.Iterator;
import java.util.Vector;
import java.util.logging.Level;
import java.util.logging.Logger;
import server.ServerCore.Clients;
/**
*
* #author pisio
*/
public class GrimReaper extends Thread {
private int timeout = LoadSettings.Init().getConfigInt("grimreaper") * 1000; // 1000 ms = 1 sec
public GrimReaper() {
super();
}
public void cleanUserThreads() {
Vector users = ServerCore.users;
if (users.size() < 1) {
return;
}
Iterator iteratr = users.iterator();
while (iteratr.hasNext()) {
Clients user = (Clients) iteratr.next();
System.out.println(user.isAlive());
if (user.getClient().isClosed()) {
user.interrupt();
if (user.isInterrupted()) {
System.out.println("Beshe kiknat");
}
iteratr.remove();
// if (PublicVaribles.Init().systemLevelMesseging() == 2) {
System.out.println("+\t Kicked user ");
// }
}//if is dead
}//while
}//cleanUserThreads;
#Override
public void run() {
try {
while (ServerCore.getServerRunning()) {
cleanUserThreads();
sleep(timeout);
System.out.println("nani na shinigami");
}
} catch (InterruptedException ex) {
Logger.getLogger(GrimReaper.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
package server;
import java.io.*;
import java.net.*;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import server.DB.DataBase;
public class ServerCore extends Thread {
private static ServerCore sc = null;
private int port = LoadSettings.Init().getConfigInt("port");
private int max_connections = LoadSettings.Init().getConfigInt("max_connections");
private String ipServer = LoadSettings.Init().getConfig("ipServer");
private ServerSocket socket;
private static boolean serverRuning = false;
public static Vector users = new Vector();
public GrimReaper shinigami = new GrimReaper();// Shinigami from Japanice is Grim Reaper!
private ServerCore() {
}
#Override
public void run() {
shinigami.start();
try {
socket = new ServerSocket(port, max_connections);
System.out.println("+++\t Server was started at address:" + socket.getLocalSocketAddress() + " with posible max users " + max_connections);
serverRuning = true;
while (serverRuning) {
Socket client = socket.accept();
shinigami.cleanUserThreads();
if (users.size() < max_connections) {
Clients cls = new Clients(client);
cls.start();
users.add(cls);
System.out.println("++\tClient was connected:" + client.toString());
} else {
Clients cls = new Clients(client);
cls.start();
cls.getOutput().println("sorry_no_avable_slot");
cls.getOutput().flush();
cls.interrupt();
}
}
} catch (IOException ex) {
// Logger.getLogger(ServerCore.class.getName()).log(Level.SEVERE, null, ex);
}
}
//run method
public void sendUserMsg() {
Scanner input = PublicVaribles.Init().inputKeyBord();
System.out.print("Enter UserID/user connection port:");
int userID = input.nextInt();
Iterator iterator = users.iterator();
while (iterator.hasNext()) {
Clients cls = (Clients) iterator.next();
/// System.out.println("Passed user:" + cls.getUserId());
if (cls.getUserId() == userID) {
System.out.print("\nEnter msg:");
String str = input.next();
cls.getOutput().println(str);
System.out.println("+\t" + cls.getUserId() + " get msg :" + str);
}
}
}
//SendUserMsg
public void stopServer() {
statusServer();
serverRuning = false;
try {
socket.close();
} catch (IOException ex) {
Logger.getLogger(ServerCore.class.getName()).log(Level.SEVERE, null, ex);
}
System.out.println("+++\t SERVER WAS STOPED !");
}
//Stop server
public void statusServer() {
if (serverRuning) {
System.out.println("+++\t Server running at port:" + port + " with connected users :" + users.size() + "/" + max_connections);
} else {
System.out.println("+++\t Server IS NOT RUNNING!");
}
}
//Status server
public static boolean getServerRunning() {
// function for GrimReaper .... R.I.P :D
return ServerCore.serverRuning;
}
public static ServerCore Init() {
if (ServerCore.sc == null) {
ServerCore.sc = new ServerCore();
}
return ServerCore.sc;
}
// SingleTon
public class Clients extends Thread {
private Socket client;
private int userID;
private Scanner input;
private PrintWriter output;
public Clients(Socket socket) {
client = socket;
userID = socket.getPort();
try {
input = new Scanner(client.getInputStream());
output = new PrintWriter(client.getOutputStream(), true);
} catch (IOException ioEx) {
System.out.println(ioEx.toString());
}
}
public int getUserId() {
return userID;
}
public Scanner getInput() {
return input;
}
public PrintWriter getOutput() {
return output;
}
public Socket getClient() {
return client;
}
}//Clients Class
}

Note: I'm assuming Clients extends Thread.
It looks like you might be using interrupt() incorrectly. After calling user.interrupt(), it's up to that thread to check that it has been interrupted by calling Thread.interrupted() and terminate itself if true. Here's an extremely basic example:
class Clients extends Thread {
#Override
public void run() {
while (!Thread.interrupted()) {
//do work
}
//thread has been interrupted and quits
}
}

GWT SerializationException__Ljava_lang_String_2V while doing RPC

Below are my service, serviceImpl and async callback interfaces. Appreciate if someone can help me understand why the submitTeam(...) isn't being called (I howeve see that isValidEmail(...) is being invoked when used). At least provide me the approach for debugging this as currently I am unable to use eclipse debugger (eclipse is not stopping at breakpoints) and the sysout/syserr statements are not being logged to the console either :-(.
#RemoteServiceRelativePath("registrationService")
public interface RegistrationService extends RemoteService
{
Boolean isValidEmail(String email);
String submitTeam(String teamName, List<Player> players);
}
public interface RegistrationServiceAsync
{
void isValidEmail(String email, AsyncCallback<Boolean> callback);
void submitTeam(String teamName, List<Player> players, AsyncCallback<String> callback);
}
public class SubmitTeamCallback implements AsyncCallback<String> {
private final Label status;
public SubmitTeamCallback(Label s) {
status = s;
}
public void onFailure(Throwable caught) {
status.setText(caught.getMessage());
}
public void onSuccess(String result) {
System.out.println("" + getClass().getName() + " : " + result);
status.setText(result);
}
}
public class RegistrationCallback implements AsyncCallback<Boolean>
{
private final Label status;
private final PlayerWidget playerWidget;
private Boolean isValidSharedFlag;
public RegistrationCallback(Label s, PlayerWidget pw, Boolean isValid)
{
status = s;
playerWidget = pw;
setIsValidSharedFlag(isValid);
}
public void onFailure(Throwable caught)
{
status.setText(caught.getMessage());
}
public void onSuccess(Boolean result)
{
if (result.equals(Boolean.FALSE))
{
playerWidget.invalidEmail();
String oldText = status.getText();
status.setText(oldText + "Please specify a valid email address for the captain.");
setIsValidSharedFlag(Boolean.FALSE);
} else
{
playerWidget.validEmail();
String newText = status.getText().replace("Please specify a valid email address for the captain.", "");
status.setText(newText);
setIsValidSharedFlag(Boolean.TRUE);
}
}
public void setIsValidSharedFlag(Boolean isValidSharedFlag)
{
this.isValidSharedFlag = isValidSharedFlag;
}
public Boolean getIsValidSharedFlag()
{
return isValidSharedFlag;
}
}
public class RegistrationServiceImpl extends RemoteServiceServlet implements
RegistrationService {
Logger logger = Logger.getLogger("");
private final Emailer emailer = new Emailer();
private final EmailValidator validator = new EmailValidator();
public Boolean isValidEmail(String email) {
return validator.validate(email);
}
public String submitTeam(String teamName, List<Player> players) {
logger.log(Level.SEVERE, "This is a test log");
System.out.println("This is a test log");
boolean emailSent = false;
for (Player p : players) {
System.out.println("Emailing to captain");
if (p instanceof Captain) {
if (!validator.validate(p.getEmail())) {
return "Please specify a valid email";
}
System.out.println("Emailing to captain");
emailSent = emailer.email(p);
}
}
if (emailSent) {
System.out.println("Email sent successfully");
return teamName
+ " has been successfully registered. Please pay the registration fee to confirm registration. Thank you.";
} else {
return "Unable to send email. Please email the team details [Team name, minimum of 6 players, captain's email address and phone number] to funpluscharity#gmail.com";
}
}
}
Below method is going to invoke the RPC calls
private void registerTeam() {
System.out.println("Received request to dubmit team");
StringBuilder statusText = new StringBuilder();
try {
Boolean isValid = true;
RegistrationServiceAsync service = (RegistrationServiceAsync) GWT
.create(RegistrationService.class);
ServiceDefTarget serviceDef = (ServiceDefTarget) service;
System.err.println(".........." + GWT.getModuleBaseURL());
serviceDef
.setServiceEntryPoint("/services/registrationService");
if (teamName.getText() == null
|| teamName.getText().length() == 0) {
isValid = false;
statusText.append("Please specify team name. ");
}
Player captain = getCaptain();
if (!atleast6PlayersAreSpecified()) {
isValid = false;
statusText.append("Please specify atleast 6 players. ");
}
if (captain == null) {
isValid = false;
statusText.append("Please designate a captain. ");
}
System.out.println("Is request valid? " + isValid);
if (isValid.equals(Boolean.TRUE)) {
System.out.println("RPC - submitTeam start ");
System.out.println("" + getPlayers());
SubmitTeamCallback callback = new SubmitTeamCallback(status);
service.submitTeam(teamName.getText(), getPlayers(),
callback);
System.out.println("RPC - submitTeam end");
} else {
status.setText(statusText.toString());
}
} catch (Exception e) {
e.printStackTrace();
System.err.println(e.getMessage());
}
}
Made some progress after adding Window.alert(...) based on #Pistol suggestion and it is going to onFailure(Throwable t) method of the SubmitTeamCallback class.
unknown.com_google_gwt_user_client_rpc_SerializationException_SerializationException__Ljava_lang_String_2V(Unknown Source)

An alternative method to debug client side code is using Window.alert(...). For example placing it in your callback onFailure() method to see caught messages for a start. Or you can use FireBug or something similar to see if the rpc-call is actually being made?
Edit :
Are you sure your Player class implements IsSerializable and have an empty constructor? Check out Serializable User-Defined Classes.

Figured this out. The POJO's should have a default constructor and the one's I am using didn't have one. GWT compiler didn't complain either. Not sure why.

Make an existing code in Java parallel/multithread

I have a very simple crawler. I want to make my current code run in a few threads. Could you provide me a little tutorial or article to help me achive this test?
I'm originally a .Net developer and in .Net I have no problem whatsoever running codes in multithread but unfortunately I don't know anything about threads in Java.
My crawler is a command-line software so don't worry about GUI.
Thank you in advance.

Java does multithreading through the Thread class. One of the most common ways to make existing code multithreaded is to use the Runnable interface to define what you want to call at thread start, and then start it off.
public class SomeFunctions
{
public static void FunctionA() {}
public static void FunctionB() {}
public static void FunctionC() {}
}
// ...
Thread t1 = new Thread(new Runnable() {
public void run() {
SomeFunctions.FunctionA();
}
});
t1.start();
// (rinse and repeat for the other functions)
Dry coded, but it should at least get the general concept across. Of course, as soon as you go into multithreading land, you have concurrency issues and need to make sure everything is appropriately syhchronized, etc., but any language will have those issues.
If you're worried about synchronization, you have a few tools at your disposal. The easiest is the recursive mutex functionality built into Java, the "synchronized" keyword. More classical means are also available through various classes in the java.util.concurrent and java.util.concurrent.locks packages such as Semaphore and ReadWriteLock
http://download.oracle.com/javase/6/docs/api/java/util/concurrent/package-summary.html
http://download.oracle.com/javase/6/docs/api/java/util/concurrent/locks/package-summary.html

You can take a look at my webcrawler example. Sry for the lengthiness.
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
* A web crawler with a Worker pool
*
* #author Adriaan
*/
public class WebCrawler implements Manager {
private Set<Worker> workers = new HashSet<Worker>();
private List<String> toCrawl = new ArrayList<String>();
private Set<String> crawled = new HashSet<String>();
private Set<String> hosts = new HashSet<String>();
private Set<String> results = new HashSet<String>();
private int maxResults;
public WebCrawler(String url, int numberOfWorkers, int maxResults) {
this.maxResults = maxResults;
toCrawl.add(url);
createWorkers(numberOfWorkers);
}
public void createWorkers(int numberOfWorkers) {
for (int i = 0; i < numberOfWorkers; i++) {
workers.add(new Worker(this));
}
}
private void stopWorkers() {
for (Worker worker : workers) {
worker.terminate();
}
}
public synchronized Job getNewJob() {
while (toCrawl.size() == 0) {
try {
wait();
} catch (InterruptedException e) {
// ignore
}
}
return new EmailAddressCrawlJob().setDescription(toCrawl.remove(0));
}
public synchronized void jobCompleted(Job job) {
// System.out.println("crawled: " + job.getDescription());
crawled.add(job.getDescription());
String host = getHost(job.getDescription());
boolean knownHost = hosts.contains(host);
if (!knownHost) {
System.out.println("host: " + host);
hosts.add(host);
}
for (String url : job.getNewDescriptions()) {
if (!crawled.contains(url)) {
if (knownHost) {
toCrawl.add(toCrawl.size() - 1, url);
} else {
toCrawl.add(url);
}
}
}
for (String result : job.getResults()) {
if (results.add(result)) {
System.out.println("result: " + result);
}
}
notifyAll();
if (results.size() >= maxResults) {
stopWorkers();
System.out.println("Crawled hosts:");
for (String crawledHost : hosts) {
System.out.println(crawledHost);
}
Set<String> uncrawledHosts = new HashSet<String>();
for (String toCrawlUrl : toCrawl) {
uncrawledHosts.add(getHost(toCrawlUrl));
}
System.out.println("Uncrawled hosts:");
for (String unCrawledHost : uncrawledHosts) {
System.out.println(unCrawledHost);
}
}
if (crawled.size() % 10 == 0) {
System.out.println("crawled=" + crawled.size() + " toCrawl="
+ toCrawl.size() + " results=" + results.size() + " hosts="
+ hosts.size() + " lastHost=" + host);
}
}
public String getHost(String host) {
int hostStart = host.indexOf("://") + 3;
if (hostStart > 0) {
int hostEnd = host.indexOf("/", hostStart);
if (hostEnd < 0) {
hostEnd = host.length();
}
host = host.substring(hostStart, hostEnd);
}
return host;
}
public static void main(String[] args) throws MalformedURLException {
new WebCrawler("http://www.nu.nl/", 5, 20);
}
}
Worker
**
* A Worker proactively gets a Job, executes it and notifies its manager that
* the Job is completed.
*
* #author Adriaan
*/
public class Worker extends Thread {
private final Manager manager;
private Job job = null;
private boolean isWorking;
public Worker(Manager manager) {
this.manager = manager;
isWorking = true;
start();
}
#Override
public void run() {
System.out.println("Worker " + Thread.currentThread().getId()
+ " starting ");
while (isWorking) {
job = manager.getNewJob();
job.execute();
manager.jobCompleted(job);
}
}
public void terminate() {
isWorking = false;
}
}
Manager interface
/**
* Manager interface for Workers
*
* #author Adriaan
*/
public interface Manager {
/**
* Gets a new job
*
* #return
*/
public Job getNewJob();
/**
* Indicates the job is completed
*
* #param job
*/
public void jobCompleted(Job job);
}
Job
import java.util.HashSet;
import java.util.Set;
/**
* A Job is a unit of work defined by a String (the description). During execution the
* job can obtain results and new job descriptions.
*
* #author Adriaan
*/
public abstract class Job {
private String description;
private Set<String> results = new HashSet<String>();
private Set<String> newDescriptions = new HashSet<String>();
/**
* Sets the job description
*
* #param description
* #return this for chaining
*/
public Job setDescription(String description) {
this.description = description;
return this;
}
/**
* Executes the job
*/
public abstract void execute();
/**
* Gets the results obtained
*
* #return
*/
public Set<String> getResults() {
return results;
}
/**
* Gets the now job descriptions obtained
*
* #return
*/
public Set<String> getNewDescriptions() {
return newDescriptions;
}
/**
* Gets the job description
*
* #return
*/
public String getDescription() {
return description;
}
/**
* Allows the implementation to add an obtained result
*
* #param result
*/
void addResult(String result) {
results.add(result);
}
/**
* Allows the implementation to add an obtained description
*
* #param result
*/
void addNewDescription(String newDescription) {
newDescriptions.add(newDescription);
}
}
A Job which crawls a page for email addresses:
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* A Job which crawls HTTP or HTTPS URL's for email adresses, collecting new
* URL's to crawl along the way.
*
* #author Adriaan
*/
public class EmailAddressCrawlJob extends Job {
#Override
public void execute() {
try {
URL url = new URL(getDescription());
if (url != null) {
String text = readText(url);
extractNewDescriptions(text, url);
extractResults(text);
}
} catch (MalformedURLException e) {
System.err.println("Bad url " + getDescription());
}
}
private String readText(URL url) {
URLConnection connection;
try {
connection = url.openConnection();
InputStream input = connection.getInputStream();
byte[] buffer = new byte[1000];
int num = input.read(buffer);
if (num > 0) {
StringBuilder builder = new StringBuilder();
builder.append(new String(buffer, 0, num));
while (num != -1) {
num = input.read(buffer);
if (num != -1) {
builder.append(new String(buffer, 0, num));
}
}
return builder.toString();
}
} catch (IOException e) {
//System.err.println("Could not read from " + url);
}
return "";
}
private void extractNewDescriptions(String text, URL url) {
// URL extracting code from Sun example
String lowerCaseContent = text.toLowerCase();
int index = 0;
while ((index = lowerCaseContent.indexOf("<a", index)) != -1) {
if ((index = lowerCaseContent.indexOf("href", index)) == -1) {
break;
}
if ((index = lowerCaseContent.indexOf("=", index)) == -1) {
break;
}
index++;
String remaining = text.substring(index);
StringTokenizer st = new StringTokenizer(remaining, "\t\n\r\">#");
String strLink = st.nextToken();
if (strLink.startsWith("javascript:")) {
continue;
}
URL urlLink;
try {
urlLink = new URL(url, strLink);
strLink = urlLink.toString();
} catch (MalformedURLException e) {
// System.err.println("Could not create url: " + target
// + " + " + strLink);
continue;
}
// only look at http links
String protocol = urlLink.getProtocol();
if (protocol.compareTo("http") != 0
&& protocol.compareTo("https") != 0) {
// System.err.println("Ignoring: " + protocol
// + " protocol in " + urlLink);
continue;
}
addNewDescription(urlLink.toString());
}
}
private void extractResults(String text) {
Pattern p = Pattern
.compile("([\\w\\-]([\\.\\w])+[\\w]+#([\\w\\-]+\\.)+[A-Za-z]{2,4})");
Matcher m = p.matcher(text);
while (m.find()) {
addResult(m.group(1));
}
}
}
I know this answer is a bit verbose, but I thought OP might be best helped with a working example and I happened to have made one not so long ago.

A very basic java program that will give the abstract idea of the Multi Threading..
public class MyThread extends Thread {
String word;
public MyThread(String rm){
word = rm;
}
public void run(){
try {
for(;;){
System.out.println(word);
Thread.sleep(1000);
}
} catch(InterruptedException e) {
System.out.println("sleep interrupted");
}
}
public static void main(String[] args) {
Thread t1=new MyThread("First Thread");
Thread t2=new MyThread("Second Thread");
t1.start();
t2.start();
}
}
And the Output will be..
First Thread
Second Thread
First Thread
Second Thread
First Thread
Go with this PPT it will help you with the basics..
Here