We Keep Coding

How to gracefully wait to job task finish in BlockingQueue java

I am writing a job queue using BlockingQueue and ExecutorService. It basically waiting new data in the queue, if there are any data put into the queue, executorService will fetch data from queue. But the problem is that i am using a loop that loops to wait the queue to have data and thus the cpu usage is super high.
I am new to use this api. Not sure how to improve this.
ExecutorService mExecutorService = Executors.newSingleThreadExecutor();
BlockingQueue<T> mBlockingQueue = new ArrayBlockingQueue();
public void handleRequests() {
Future<T> future = mExecutorService.submit(new WorkerHandler(mBlockingQueue, mQueueState));
try {
value = future.get();
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
if (mListener != null && returnedValue != null) {
mListener.onNewItemDequeued(value);
}
}
}
private static class WorkerHandler<T> implements Callable<T> {
private final BlockingQueue<T> mBlockingQueue;
private PollingQueueState mQueueState;
PollingRequestHandler(BlockingQueue<T> blockingQueue, PollingQueueState state) {
mBlockingQueue = blockingQueue;
mQueueState = state;
}
#Override
public T call() throws Exception {
T value = null;
while (true) { // problem is here, this loop takes full cpu usage if queue is empty
if (mBlockingQueue.isEmpty()) {
mQueueState = PollingQueueState.WAITING;
} else {
mQueueState = PollingQueueState.FETCHING;
}
if (mQueueState == PollingQueueState.FETCHING) {
try {
value = mBlockingQueue.take();
break;
} catch (InterruptedException e) {
Log.e(TAG, e.getMessage(), e);
break;
}
}
}
Any suggestions on how to improve this would be much appreciated!

You don't need to test for the queue to be empty, you just take(), so the thread blocks until data is available.
When an element is put on the queue the thread awakens an value is set.
If you don't need to cancel the task you just need:
#Override
public T call() throws Exception {
T value = mBlockingQueue.take();
return value;
}
If you want to be able to cancel the task :
#Override
public T call() throws Exception {
T value = null;
while (value==null) {
try {
value = mBlockingQueue.poll(50L,TimeUnit.MILLISECONDS);
break;
} catch (InterruptedException e) {
Log.e(TAG, e.getMessage(), e);
break;
}
}
return value;
}

if (mBlockingQueue.isEmpty()) {
mQueueState = PollingQueueState.WAITING;
} else {
mQueueState = PollingQueueState.FETCHING;
}
if (mQueueState == PollingQueueState.FETCHING)
Remove these lines, the break;, and the matching closing brace.

Why the threads stop when the other start to execute?

I'm new to Java concurrent API and I've searched but didn't find an answer to my question.
Well, I have a code that look for every file inside directories and their subdirectories and another that copy every found file that match a specified pattern.
I separate this codes in one Runnable implementation called DirSearch and one Callable implementation called FileSearch and submit them using an ExecutorService.
That's the code:
private boolean execute() {
ExecutorService executor = Executors.newFixedThreadPool(threadsNumber);
BlockingQueue<File> dirQueue = new LinkedBlockingQueue<>();
BlockingQueue<File> fileQueue = new LinkedBlockingQueue<>(10000);
boolean isFinished = false;
try {
for(int i = 0; i < dirThreads; i++) {
executor.submit(new DirSearch(dirQueue, fileQueue, count, dirThreads);
}
count.incrementAndGet();
dirQueue.add(baseDir);
Future<Boolean> future = executor.submit(new FileSearch(filequeue, outputDirectory, filename));
isFinished = future.get();
} catch(ExecutionException | InterruptedException | RuntimeException ex) {
ex.printStackTrace();
} finally {
executor.shutdownNow();
}
return isFinished;
}
...
private void copyFile(File in, File out) {
Path inPath = Paths.get(in.getAbsolutePath());
Path outPath = Paths.get(out.getAbsolutePath(), in.getName());
try {
main.updateCurrentLabel(outPath.toString());
switch(mode) {
case "1":
Files.copy(inPath, outPath, StandardCopyOption.REPLACE_EXISTING);
break;
case "2":
Files.move(inPath, outPath, StandardCopyOption.REPLACE_EXISTING);
break;
default:
break;
}
main.updateCopiedLabel(String.valueOf(countCpFiles.incrementAndGet()));
} catch(IOException ex) {
ex.printStackTrace();
}
}
...
private class DirSearch implements Runnable {
...
#Override
public void run() {
try {
File dir = dirQueue.take();
while(dir != new File("")) {
File[] elements = dir.listFiles();
if(elements != null) {
for(File element : elements) {
if(element.isDirectory()) {
count.incrementAndGet();
dirQueue.put(element);
} else {
fileQueue.put(element);
}
}
}
if(count.decrementAndGet() == 0) {
end();
}
dir = dirQueue.take();
}
} catch(InterruptedException ex) {
ex.printStackTrace();
}
}
...
}
...
private class FileSearch implements Callable<Boolean> {
...
#Override
public Boolean call() {
boolean isFinished = false;
try {
File file = fileQueue.take();
while(file != new File("")) {
incrementAnalyzed();
String foundFile = file.getName().toLowerCase();
if(foundFile.matches(filename.replace("?", ".?").replace("*", ".*?"))) {
copyFile(file, outputDirectory);
}
file = fileQueue.take();
}
isFinished = true;
} catch(InterruptedException ex) {
ex.printStackTrace();
}
return isFinished;
}
}
The problem is: when the FileSearch start to copy files, the other threads (DirSearch) stop and don't look for any new file until the copy is completed. Why this is happening? Am I doing anything wrong or this is not the correct approach?

Two possible answers which came to my mind and which i cant guarantee they apply to your specific situation:
1. Java VM gets only one core from your CPU which means it can only run one thread at a time.
2. your threads both use the same variable which means only one is allowed to really manipulate it at a time. For this specific problem look up java keyword "synchronized".
I guess the root of the problem tends to be #1

Java WatchService watches not terminated

I'm working on an application that uses the Java watchservice (Java 8) under Linux Mint. One interesting problem I am encountering is running out of inotify watches.
I'm developing under Eclipse and the behavior is as follows:
When the app starts, it recurses a directory structure, putting a watch on each directory found. The current test case uses 13,660 paths. My maximum is set to 16384.
If I stop and restart the app several (20+ times), it seems to function normally. Eventually, however, I will get a cascade of system errors indicating the maximum number of watches has been reached. However, if I restart Eclipse, the issue goes away.
Obviously, the Watch Service isn't releasing all of it's resources, but of the 13,660 watches it acquires, only a few (I'm guessing less than a hundred) are retained. It appears they aren't released unless I shut down Eclipse's Java instance and restart it.
To address this, I've ensured the watch service's close method is called when the application shuts down and the watch service task is cancelled.
The only other thing that I'm doing differently is I'm running two separate watch services for two different purposes. I'm told that you shouldn't need to run more than one, and perhaps this is the problem, but I'd rather not run one watch service if I can help it.
That said, are there any thoughts or suggestions on how I might be able to determine the cause of this bug?
Apologies for the massive code posting. This is my implementation of the WatchService class.
A few notes:
The pathFinder runs in a separate thread and is just a file visitor - walking the directory tree and returning paths to all dirs / files found.
Register is called only when changes are posted to the pathsChanged property (from the pathFinder's onSucceeded callback).
The pathsChanged property is always updated by a setAll() call. It only posts the latest changes and is not meant to be cumulative. Beyond the watchservice, other classes listen to these properties and respond accordingly.
public final class LocalWatchService extends BaseTask {
private final static String TAG = "LocalWatchService";
//watch service task
private WatchService watcher;
//path finding task and associated executor
private LocalPathFinder finder;
//root path where the watch service begins
private final Path mRootPath;
private final ExecutorService pathFinderExecutor =
createExecutor ("pathFinder", false);
//class hash map which keys watched paths to generated watch keys
private final Map<WatchKey, Path> keys = new HashMap<WatchKey, Path>();
//reference to model property of watched paths.
private final SimpleListProperty <SyncPath> mChangedPaths =
new SimpleListProperty <SyncPath>
(FXCollections.<SyncPath> observableArrayList());
public LocalWatchService (String rootPath) {
super ();
mRootPath = Paths.get(rootPath);
//create the watch service
try {
this.watcher = FileSystems.getDefault().newWatchService();
} catch (IOException e) {
e.printStackTrace();
}
setOnCancelled(new EventHandler() {
#Override
public void handle(Event arg0) {
pathFinderExecutor.shutdown();
}
});
mChangedPaths.addListener(new ListChangeListener <SyncPath> (){
#Override
public void onChanged(
javafx.collections.ListChangeListener.Change<? extends SyncPath>
arg0) {
for (SyncPath path: arg0.getList()) {
//call register only when a directory is found
if (path.getFile() == null) {
try {
register (path.getPath());
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
}
});
};
public SimpleListProperty<SyncPath> changedPaths() { return mChangedPaths; }
public void initializeWatchPaths() {
ArrayList <Path> paths = new ArrayList <Path> ();
//create a DirectoryStream filter that finds only directories
//and symlinks
DirectoryStream.Filter<Path> filter =
new DirectoryStream.Filter<Path>() {
public boolean accept(Path file) throws IOException {
return (Files.isDirectory(file) ||
Files.isSymbolicLink(file));
}
};
//apply the filter to a directory stream opened on the root path
//and save everything returned.
paths.addAll(utils.getFiles(mRootPath, filter));
runPathFinder (paths);
}
private void runPathFinder (ArrayList <Path> paths) {
//need to add blocking code / mechanism in case a path finder is
//currently running (rare case)
finder = new LocalPathFinder();
finder.setPaths (paths);
//callbacks on successful completion of pathfinder
EventHandler <WorkerStateEvent> eh =
new EventHandler <WorkerStateEvent> () {
ArrayList <SyncPath> paths = new ArrayList <SyncPath>();
#Override
public void handle(WorkerStateEvent arg0) {
for (Path p: finder.getPaths()) {
paths.add(
new SyncPath(mRootPath, p, SyncType.SYNC_NONE));
}
addPaths(paths);
}
};
finder.setOnSucceeded(eh);
pathFinderExecutor.execute (finder);
}
private void addPath(Path path, SyncType syncType) {
mChangedPaths.setAll(new SyncPath(mRootPath, path, syncType));
}
private void addPaths(ArrayList<SyncPath> paths) {
mChangedPaths.setAll(paths);
}
/**
* Register the given directory with the WatchService
* #throws InterruptedException
*/
public final void register(Path dir)
throws IOException, InterruptedException {
//register the key with the watch service
WatchKey key =
dir.register (watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
if (!keys.isEmpty()) {
Path prev = keys.get(key);
if (prev == null) {
//This is a new key
}
else if (!dir.equals(prev)) {
//This is an update
}
}
keys.put(key, dir);
}
private void processWatchEvent (WatchKey key, Path dir) throws IOException, InterruptedException {
for (WatchEvent<?> event: key.pollEvents()) {
WatchEvent.Kind kind = event.kind();
// TBD - provide example of how OVERFLOW event is handled
if (kind == OVERFLOW) {
System.out.println ("Overflow encountered");
}
WatchEvent<Path> ev = (WatchEvent<Path>)event;
Path target = dir.resolve(ev.context());
if (kind == ENTRY_DELETE) {
ArrayList <Path> finderList = new ArrayList <Path> ();
if (Files.isDirectory(target)) {
//directory deletion is not implemented apart from
//file deletion
}
else
addPath (target, SyncType.SYNC_DELETE);
} else if (kind == ENTRY_CREATE) {
/*
* Added paths are passed to the pathfinder service for
* subdirectory discovery. Path and subpaths are then added
* to the AddedPaths property via an event listener on
* service's onSucceeded() event.
*
* Added files are added directly to the AddedPaths property
*/
ArrayList <Path> finderList = new ArrayList <Path> ();
if (Files.isDirectory(target)) {
finderList.add (target);
runPathFinder (finderList);
}
//add files directly to the addedPaths property
else {
//a newly created file may not be immediately readable
if (Files.isReadable(target)) {
addPath (target, SyncType.SYNC_CREATE);
}
else
System.err.println ("File " + target + " cannot be read");
}
} else if (kind == ENTRY_MODIFY) {
System.out.println ("File modified: " + target.toString());
}
boolean valid = key.reset();
if (!valid)
break;
}
}
#SuppressWarnings("unchecked")
<T> WatchEvent<T> cast(WatchEvent<?> event) {
return (WatchEvent<T>)event;
}
#Override
protected Void call () throws IOException, InterruptedException {
boolean interrupted = false;
register (mRootPath);
initializeWatchPaths();
try {
// enter watch cycle
while (!interrupted) {
//watch for a key change. Thread blocks until a change occurs
WatchKey key = null;
interrupted = isCancelled();
//thread blocks until a key change occurs
// (whether a new path is processed by finder or a watched item changes otherwise)
try {
key = watcher.take();
} catch (InterruptedException e) {
interrupted = true;
try {
watcher.close();
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
// fall through and retry
}
Path dir = keys.get (key);
if (dir == null) {
System.out.println ("Null directory key encountered.");
continue;
}
//process key change once it occurs
processWatchEvent(key, dir);
// reset key and remove from set if directory no longer accessible
if (!key.reset()) {
keys.remove(key);
// all directories are inaccessible
if (keys.isEmpty())
break;
}
}
} finally {
if (interrupted)
Thread.currentThread().interrupt();
}
keys.clear();
watcher.close();
return null;
};
}

Issue invoking WatchService inside a singleton bean

we wanted to watch a file periodically for changes, we are using jboss 7 . Following is my code snippet. I initialized the watcher in the postconstruct method of singleton bean and scheduled a method to poll watch events. I could observe the changes when i modify the file very first time, however the subsequent modifications to the file are not recieved . Can anyone please let me know what could be the issue
#Startup
#ConcurrencyManagement(ConcurrencyManagementType.BEAN)
#Interceptors(NonThrowingPostConstructInterceptor.class)
#Singleton
#Service
#LocalBinding(jndiBinding=IHeartBeatProducerService.JNDI_LOCAL_BINDING)
public class HeartBeatProducerService extends EMSingletonService implements IHeartBeatProducerService{
#EJB(mappedName=IMessageService.JNDI_LOCAL_BINDING)
public IMessageService messageService;
#EJB(mappedName=ICommandExecutionService.JNDI_LOCAL_BINDING)
public ICommandExecutionService commandService;
private final static String LAST_OPERATION_COMPLETED="Last Operation Completed";
private final static String STATUS="Status";
private WatchService watcher;
private Path dir;
private String concServer;
public static final String TOPIC="foo";
private IMLogger logger = new IMLogger("foo");
private String content=null;
#PostConstruct
#Override
public void init() {
// TODO Auto-generated method stub
super.init();
try {
watcher = FileSystems.getDefault().newWatchService();
dir=Paths.get("/shared/foo");
dir.register(watcher, StandardWatchEventKinds.ENTRY_MODIFY);
logger.entering(0, IHeartBeatProducerService.class.getSimpleName(), "Initializing Heart Beat", new String[]{"Entered"});
} catch (IOException e) {
e.printStackTrace();
}
}
#Schedule(second="*/10", minute = "*", hour="*")
private void checkStatus()
{
logger.entering(0, IHeartBeatProducerService.class.getSimpleName(), "Checking Status", new String[]{"Entered"});
final String[] command={"pidof","server"};
commandService.run(command, null, false);
concServer=(commandService.getExitCode()==0)?"UP":"DOWN";
if(concServer.equals("UP"))
{
watch();
}
else
{
content="foo:Failed";
}
produce();
}
public void watch()
{
logger.entering(0, IHeartBeatProducerService.class.getSimpleName(), "Entering watch()", new String[]{"Entered"});
WatchKey key = null;
try
{
key = watcher.take();
}
catch (InterruptedException e)
{
logger.error(HeartBeatProducerService.class.getSimpleName(),"Interupted Exception " + e.getMessage());
}
for ( WatchEvent<?> event: key.pollEvents())
{
WatchEvent.Kind kind = event.kind();
logger.info(HeartBeatProducerService.class.getSimpleName(),"Watch Event :" + kind.name());
if(kind.name().equals("OVERFLOW"))
{
continue;
}
if(kind.name().equals("ENTRY_MODIFY"))
{
Path concLog = (Path) event.context();
logger.info(HeartBeatProducerService.class.getSimpleName(),"Modified File Name:" + concLog.getFileName());
if(concLog.endsWith("current_status.txt"))
{
logger.info(HeartBeatProducerService.class.getSimpleName(), "Reading Status");
readStatus();
}
}
}
boolean valid = key.reset();
if ( !valid)
{
logger.error(HeartBeatProducerService.class.getSimpleName(),"Key Unregistered");
}
}
private void parse(String output)
{
// parse file contents
}
private void readStatus() {
//read status and parse()
}
private void produce()
{
try {
messageService.publish(TOPIC, content, PublishType.ASync);
} catch (MessageException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
There is already a link explaining the same with #Asynchronous tag (EJB 3.1 and NIO2: Monitoring the file system) . however I need to know what could be wrong in this approach.

Your watch method needs to run in an infinite loop. What's happening now is that after
try {
key = watcher.take();
}
you process the event and then the watch() method is finished. Try the effect of
for(;;) {
before the above lines, ending the for block after the validity check. Did you see the example at The Java Tutorials?

Directory watching for changes in java

I am using WatchService to watch change in directory, in particular creation of new file in directory. Below is my code -
package watcher;
import java.nio.file.*;
import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
import static java.nio.file.StandardWatchEventKinds.OVERFLOW;
import java.io.*;
public class Watch {
public static void main(String[] args) throws IOException {
Path dir = Paths.get("c:\\mk\\");
WatchService service = FileSystems.getDefault().newWatchService();
WatchKey key = dir.register(service, ENTRY_CREATE);
System.out.println("Watching directory: "+dir.toString());
for(;;){
WatchKey key1;
try {
key1 = service.take();
} catch (InterruptedException x) {
break;
}
for (WatchEvent<?> event: key1.pollEvents()) {
WatchEvent.Kind<?> kind = event.kind();
if (kind == OVERFLOW) {
continue;
}
WatchEvent<Path> ev = (WatchEvent<Path>)event;
Path filename = ev.context();
Path child = dir.resolve(filename);
System.out.println("New file: "+child.toString()+" created.");
try{
FileInputStream in = new FileInputStream(child.toFile());
System.out.println("File opened for reading");
in.close();
System.out.println("File Closed");
}catch(Exception x){
x.printStackTrace();
}
}
boolean valid = key.reset();
if (!valid) {
break;
}
}
}
}
When I create file inside "mk" directory, I am getting notification for that. But when I copy some file in this directory, I am getting exception on opening that copied file.
My guess is Windows Copier dialog has still locked that file and I was unable to open that file. So basically I want to know is how to get notified for file has been closed by other process.
Output of above code is like -
Watching directory: c:\mk
New file: c:\mk\New Text Document (2).txt created.
File opened for reading
File Closed
New file: c:\mk\Config.class created.
java.io.FileNotFoundException: c:\mk\Config.class (The process cannot access the file because it is being used by another process)
at java.io.FileInputStream.open(Native Method)
at java.io.FileInputStream.<init>(FileInputStream.java:138)
at watcher.Watch.main(Watch.java:36)
New file: c:\mk\New Text Document (3).txt created.
File opened for reading
File Closed
Files "New Text Document (2).txt" and "New Text Document (3).txt" I have created but file "Config.class" I have copied from other directory.
Please help me on this.

I got this working by implementing algorithm: Watcher thread will put file names in BlockingQueue and other thread will poll this queue, takes file names, try few times to open file. If file gets opened, Windows Copier has released file lock and we can proceed. So when other threads finds file has been unlocked, other thread will put this file name in processed queue, from where my application will retrieve file name. Also the other thread while checking for file unlock by opening file, if it runs long time for unlocking file, we can place back this file name in BlockingQueue and process other file names, former can be processed later.
Solution: Hope this may help to other:
package dirwatch;
import java.nio.file.*;
import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
import static java.nio.file.StandardWatchEventKinds.OVERFLOW;
import static java.nio.file.LinkOption.*;
import java.nio.file.attribute.*;
import java.io.*;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
public class WatchDir {
private final WatchService watcher;
private final Map<WatchKey,Path> keys;
private final boolean recursive;
private boolean trace = false;
private BlockingQueue<String> fileProcessingQueue;
//******* processedFileQueue **** will be used by other threads to retrive unlocked files.. so I have
// kept as public final
public final BlockingQueue<String> processedFileQueue;
private volatile boolean closeProcessingThread;
private volatile boolean closeWatcherThread;
private void processFiles(){
System.out.println("DirWatchProcessingThread Started");
String fileName;
outerLoop: while(!closeProcessingThread || !fileProcessingQueue.isEmpty()){
try{
fileName = fileProcessingQueue.poll(1000, TimeUnit.MILLISECONDS);
}catch(InterruptedException ie){
fileName = null;
}
if(fileName == null || fileName.equals("")){
continue outerLoop;
}
long startTime = System.currentTimeMillis();
innerLoop: while(true){
FileInputStream fis = null;
File file = new File(fileName);
try{
fis = new FileInputStream(fileName);
break innerLoop;
}catch(FileNotFoundException fnfe){
if(!file.exists() || file.isDirectory()){
System.out.println("File: '"+fileName+"has been deleted in file system or it is not file. Not processing this file.");
continue outerLoop;
}
try{
Thread.sleep(WatchDirParameters.millisToPuaseForFileLock);
}catch(InterruptedException ie){
}
if((System.currentTimeMillis() - startTime) > WatchDirParameters.millisToSwapFileForUnlocking){
if(fileProcessingQueue.offer(fileName)){
continue outerLoop;
}else{
startTime = System.currentTimeMillis();
continue innerLoop;
}
}
}finally{
if(fis != null){
try{
fis.close();
}catch(IOException ioe){
ioe.printStackTrace();
}
}
}
}
System.out.println("Queuing File: "+fileName);
processedLoop:while(true){
try{
if(processedFileQueue.offer(fileName, 1000, TimeUnit.MILLISECONDS)){
break processedLoop;
}
}catch(InterruptedException ie){
//ie.printStackTrace();
}
}
}
closeWatcherThread = true;
closeProcessingThread = true;
System.out.println("DirWatchProcessingThread Exited");
}
/**
* Process all events for keys queued to the watcher
*/
private void processEvents(){
System.out.println("DirWatcherThread started.");
while(!closeWatcherThread) {
// wait for key to be signalled
WatchKey key;
try {
key = watcher.take();
} catch (InterruptedException x) {
// if we are returning from these method, it means we no longer wants to watch directory
// we must close thread which may be waiting for file names in queue
continue;
}catch(ClosedWatchServiceException cwse){
break;
}
Path dir = keys.get(key);
if (dir == null) {
System.err.println("WatchKey not recognized!!");
continue;
}
try{
for (WatchEvent<?> event: key.pollEvents()) {
WatchEvent.Kind kind = event.kind();
if (kind == OVERFLOW) {
continue;
}
// Context for directory entry event is the file name of entry
WatchEvent<Path> ev = cast(event);
Path name = ev.context();
Path child = dir.resolve(name);
if(kind.equals(ENTRY_CREATE)){
// if directory is created, and watching recursively, then
// register it and its sub-directories
if (recursive) {
try {
if (Files.isDirectory(child, NOFOLLOW_LINKS)) {
registerAll(child);
continue;
}
} catch (IOException x) {
// ignore to keep sample readbale
}
}
while(true){
if(fileProcessingQueue.remainingCapacity() < 2){
// if only one last can be inserted then don't queue this we need 1 empty space in queue
// for swaping file names..
// sleep for some time so processing thread may have made some rooms to queue in fileQueue
// this logic will not create any problems as only one this thread is inserting in queue
try{
Thread.sleep(200);
}catch(InterruptedException ie){
}
continue;
}
if(!fileProcessingQueue.offer(child.toString())){
// couldn't queue this element by whatever reason.. we will try to enqueue again by continuing loop
continue;
}else{
// file name has been queued in queue
break;
}
}
}
}
// reset key and remove from set if directory no longer accessible
boolean valid = key.reset();
if (!valid) {
keys.remove(key);
// all directories are inaccessible
if (keys.isEmpty()) {
break;
}
}
}catch(ClosedWatchServiceException cwse){
break;
}
}
closeProcessingThread = true;
closeWatcherThread = true;
System.out.println("DirWatcherThread exited.");
}
public void stopWatching(){
try{
watcher.close();
}catch(IOException ioe){
}
closeProcessingThread = true;
closeWatcherThread = true;
}
public static WatchDir watchDirectory(String dirName, boolean recursive) throws InvalidPathException, IOException, Exception{
try{
Path dir = Paths.get(dirName);
final WatchDir watchDir = new WatchDir(dir, recursive);
watchDir.closeProcessingThread = false;
watchDir.closeWatcherThread = false;
new Thread(new Runnable() {
public void run() {
watchDir.processFiles();
}
}, "DirWatchProcessingThread").start();
new Thread(new Runnable() {
public void run() {
watchDir.processEvents();
}
}, "DirWatcherThread").start();
return watchDir;
}catch(InvalidPathException ipe){
throw ipe;
}catch(IOException ioe){
throw ioe;
}catch(Exception e){
throw e;
}
}
#SuppressWarnings("unchecked")
private static <T> WatchEvent<T> cast(WatchEvent<?> event) {
return (WatchEvent<T>)event;
}
/**
* Register the given directory with the WatchService
*/
private void register(Path dir) throws IOException {
//WatchKey key = dir.register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
WatchKey key = dir.register(watcher, ENTRY_CREATE);
if (trace) {
Path prev = keys.get(key);
if (prev == null) {
System.out.format("register: %s\n", dir);
} else {
if (!dir.equals(prev)) {
System.out.format("update: %s -> %s\n", prev, dir);
}
}
}
keys.put(key, dir);
}
/**
* Register the given directory, and all its sub-directories, with the
* WatchService.
*/
private void registerAll(final Path start) throws IOException {
// register directory and sub-directories
Files.walkFileTree(start, new SimpleFileVisitor<Path>() {
#Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
register(dir);
return FileVisitResult.CONTINUE;
}
});
}
/**
* Creates a WatchService and registers the given directory
*/
private WatchDir(Path dir, boolean recursive) throws IOException {
fileProcessingQueue = new ArrayBlockingQueue<String>(WatchDirParameters.fileQueueSize, false);
processedFileQueue = new ArrayBlockingQueue<String>(WatchDirParameters.fileQueueSize, false);
this.watcher = FileSystems.getDefault().newWatchService();
this.keys = new HashMap<WatchKey,Path>();
this.recursive = recursive;
//CreateTxtFile.createFile(dir, 1);
if (recursive) {
System.out.format("Scanning %s ...\n", dir);
registerAll(dir);
System.out.println("Done.");
} else {
register(dir);
}
// enable trace after initial registration
this.trace = true;
}
}
Parameter Class:
package dirwatch;
public class WatchDirParameters {
public static final int millisToPuaseForFileLock = 200;
public static final int fileQueueSize = 500;
public static final int millisToSwapFileForUnlocking = 2000;
}

Made an updated version of the file provided by #UDPLover that is built for use in a high rate of file access environment I converted the to process queue into a HashMap<String, WatchEvent> to carry the watch event over to pass to an abstract method inside of the file blocking checker itself. Also made a print() method that allows anything printed to console by the WatchCore to be enabled or disabled. The file polling for loop from the original example was updated to use the JDK8 function for loop, made all parts threaded/interrupted. This is untested as of yet, will update with fixes when I can test it.
package filewatcher;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.ClosedWatchServiceException;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import static java.nio.file.LinkOption.NOFOLLOW_LINKS;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
import static java.nio.file.StandardWatchEventKinds.OVERFLOW;
import java.nio.file.WatchEvent;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
*
* #author Nackloose
* http://stackoverflow.com/questions/13998379/directory-watching-for-changes-in-java
*/
public abstract class WatchCore extends Thread {
//make class a thread by default
/**
* After the WatchCore recieves an event for a file and deems it unlocked,
* it will be passed to this function
*
* #param e WatchEvent for the file, after it has been affirmed to be
* unlocked.
*/
public abstract void onEventAndUnlocked(WatchEvent e);
private final WatchService watcher;
private final Map<WatchKey, Path> keys;
private final boolean recursive;
private boolean trace = false;
//converted to HashMap to remove the limitation as I need this in a high rate of file access enviroment.
//as well as to carry the event passed for that folder into the block check itself.
//got rid of the finished queue and made events pass to the abstract void above
private final HashMap<String, WatchEvent> fileProcessingQueue;
//create a varible to keep track of the thread checking the file blocking, so we can start and stop it.
private final WatchBlocker blocker;
public WatchCore(String dir) throws IOException {
//defaultly dont recurse
this(dir, false);
}
public WatchCore(String dir, boolean recursive) throws IOException {
this(Paths.get(dir), recursive);
}
public WatchCore(Path dir) throws IOException {
//defaultly dont recurse
this(dir, false);
}
public WatchCore(Path dir, boolean recursive) throws IOException {
fileProcessingQueue = new HashMap<>();
this.watcher = FileSystems.getDefault().newWatchService();
this.keys = new HashMap<>();
this.recursive = recursive;
//CreateTxtFile.createFile(dir, 1);
if (recursive) {
print("Scanning %s ...", dir);
registerAll(dir);
print("Done.");
} else {
register(dir);
}
// enable trace after initial registration
this.trace = true;
//start the thread to process files to be checked for file blocking
blocker = new WatchBlocker();
}
#SuppressWarnings("unchecked")
private static <T> WatchEvent<T> cast(WatchEvent<?> event) {
return (WatchEvent<T>) event;
}
#Override
public synchronized void start() {
//start up our thread _FIRST_
super.start();
//then start the blocking thread
blocker.start();
}
#Override
public void interrupt() {
//Everything backwards, stop the blocker _FIRST_
blocker.interrupt();
//then stop our thread.
super.interrupt();
}
/**
* Register the given directory with the WatchService
*/
private void register(Path dir) throws IOException {
//WatchKey key = dir.register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
WatchKey key = dir.register(watcher, ENTRY_CREATE);
if (trace) {
Path prev = keys.get(key);
if (prev == null) {
print("register: %s\n", dir);
} else {
if (!dir.equals(prev)) {
print("update: %s -> %s\n", prev, dir);
}
}
}
keys.put(key, dir);
}
/**
* Register the given directory, and all its sub-directories, with the
* WatchService.
*/
private void registerAll(final Path start) throws IOException {
// register directory and sub-directories
Files.walkFileTree(start, new SimpleFileVisitor<Path>() {
#Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
register(dir);
return FileVisitResult.CONTINUE;
}
});
}
/**
* Process all events for keys queued to the watcher
*/
#Override
public void run() {
//this was previous called processEvents()
//pruned any un-nessicary continues, labels, and labels on breaks, a lot of them
//were redundant
print("DirWatcherThread started.");
//as long as we're not interrupted we keep working
while (!interrupted()) {
// wait for key to be signalled
WatchKey key;
try {
key = watcher.take();
} catch (InterruptedException x) {
// if we are returning from these method, it means we no longer wants to watch directory
// we must close thread which may be waiting for file names in queue
continue;
} catch (ClosedWatchServiceException cwse) {
break;
}
Path dir = keys.get(key);
if (dir == null) {
printe("WatchKey not recognized!!");
continue;
}
try {
//converted to functional for loop.
key.pollEvents().stream().filter((event) -> {
WatchEvent.Kind kind = event.kind();
return !(kind == OVERFLOW); //make sure we do the filter
}).forEach((event) -> {
WatchEvent.Kind kind = event.kind();
// Context for directory entry event is the file name of entry
WatchEvent<Path> ev = cast(event);
Path name = ev.context();
Path child = dir.resolve(name);
if (kind.equals(ENTRY_CREATE)) {
// if directory is created, and watching recursively, then
// register it and its sub-directories
if (recursive) {
try {
if (Files.isDirectory(child, NOFOLLOW_LINKS)) {
registerAll(child);
return; //continue;
}
} catch (IOException x) {
// ignore to keep sample readbale
}
}
fileProcessingQueue.put(child.toString(), ev);
}
});
// reset key and remove from set if directory no longer accessible
boolean valid = key.reset();
if (!valid) {
keys.remove(key);
// all directories are inaccessible
if (keys.isEmpty()) {
break;
}
}
} catch (ClosedWatchServiceException cwse) {
break;
}
}
print("DirWatcherThread exited.");
}
/**
*
* #author
* http://stackoverflow.com/questions/13998379/directory-watching-for-changes-in-java
* Nackloose
*/
private class WatchBlocker extends Thread {
#Override
public synchronized void start() {
//get it going
super.start();
}
#Override
public void interrupt() {
//interupt our thread
super.interrupt();
}
#Override
public void run() {
//this was perviously processFiles()
//pruned any un-nessicary continues, labels, and labels on breaks, a lot of them
//were redundant
print("DirWatchProcessingThread Started");
Entry<String, WatchEvent> fileEvent;
outerLoop:
//as long as we're not interrupted we keep working
while (!interrupted()) {
if (fileProcessingQueue.isEmpty()) {
try {
Thread.sleep(WatchCoreParameters.timeToIdle);
} catch (InterruptedException ex) {
Logger.getLogger(WatchCore.class.getName()).log(Level.SEVERE, null, ex);
}
continue;
}
fileEvent = fileProcessingQueue.entrySet().iterator().next();
fileProcessingQueue.remove(fileEvent.getKey());
long startTime = System.currentTimeMillis();
while (true) {
FileInputStream fis = null;
File file = new File(fileEvent.getKey());
try {
fis = new FileInputStream(fileEvent.getKey());
break;
} catch (FileNotFoundException fnfe) {
if (!file.exists() || file.isDirectory()) {
print("File: '" + fileEvent + "has been deleted in file system or it is not file. Not processing this file.");
continue outerLoop;
}
try {
Thread.sleep(WatchCoreParameters.millisToPauseForFileLock);
} catch (InterruptedException ie) {
}
if ((System.currentTimeMillis() - startTime) > WatchCoreParameters.millisToSwapFileForUnlocking) {
fileProcessingQueue.put(fileEvent.getKey(), fileEvent.getValue());
}
} finally {
if (fis != null) {
try {
fis.close();
} catch (IOException ioe) {
ioe.printStackTrace();
}
}
}
}
print("Queuing File: " + fileEvent);
//pass the unlocked file event to the abstract method
onEventAndUnlocked(fileEvent.getValue());
}
print("DirWatchProcessingThread Exited");
}
}
/**
*
* #author
* http://stackoverflow.com/questions/13998379/directory-watching-for-changes-in-java
* Nackloose
*/
public static class WatchCoreParameters {
public static int timeToIdle = 2000, millisToPauseForFileLock = 200,
millisToSwapFileForUnlocking = 2000;
public static boolean verbose = false;
}
//<editor-fold defaultstate="collapsed" desc="Printing methods">
private void print(String s) {
//defaultly we're not writing an error
print(s, false);
}
public static final void print(String s, boolean error) {
//check verbosity, exit if none.
if (!WatchCoreParameters.verbose) {
return;
}
//if this is an error, assign System.err to a temp varible
//otherise assign System.out for normal printing
PrintStream out = (!error ? System.out : System.err);
if (s.contains("\n")) { // check to see if theirs a new line
out.print(s); //print accordingly
} else {
out.println(s); //print accordingly
}
}
public static final void printe(String s) {
//shortcut/convenience method for printing an error
print(s, true);
}
public static final void print(String s, Object... formatObj) {
//check verbosity, exit if none.
if (!WatchCoreParameters.verbose) {
return;
}
//format the object into the string, and if no newline is there, add it.
System.out.format(s + (s.contains("\n") ? "" : "\n"), formatObj);
}
//</editor-fold>
}