We Keep Coding

Why does Java file write consume CPU?

I am writing data to file using a queue on a separate thread, but the process consumes around 25% of CPU, as shown in this test main.
Is there something I can do to resolve this issue?
Perhaps I should be using flush() somewhere?
The test shows the main method start and run the queue thread and then send created data to it. The queue thread writes the data to a BufferedWriter which handles writing the data to a file.
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.logging.Level;
import java.util.logging.Logger;
import uk.co.moonsit.utils.timing.Time;
public class OutputFloatQueueReceiver extends Thread {
private static final Logger LOG = Logger.getLogger(OutputFloatQueueReceiver.class.getName());
private ConcurrentLinkedQueue<List<Float>> queue = null;
private boolean running = true;
private final BufferedWriter outputWriter;
private int ctr = 0;
private final int LIMIT = 1000;
public OutputFloatQueueReceiver(String outputFile, String header, ConcurrentLinkedQueue<List<Float>> q) throws IOException {
queue = q;
File f = new File(outputFile);
FileWriter fstream = null;
if (!f.exists()) {
try {
f.getParentFile().mkdirs();
if (!f.createNewFile()) {
throw new IOException("Exception when trying to create file " + f.getAbsolutePath());
}
fstream = new FileWriter(outputFile, false);
} catch (IOException ex) {
//Logger.getLogger(ControlHierarchy.class.getName()).log(Level.SEVERE, null, ex);
throw new IOException("Exception when trying to create file " + f.getAbsolutePath());
}
}
fstream = new FileWriter(outputFile, true);
outputWriter = new BufferedWriter(fstream);
outputWriter.append(header);
}
public synchronized void setRunning(boolean running) {
this.running = running;
}
#Override
public void run() {
while (running) {
while (queue.peek() != null) {
if (ctr++ % LIMIT == 0) {
LOG.log(Level.INFO, "Output Queue size = {0} '{'ctr={1}'}'", new Object[]{queue.size(), ctr});
}
List<Float> list = queue.poll();
if (list == null) {
continue;
}
try {
StringBuilder sbline = new StringBuilder();
Time t = new Time(list.get(0));
sbline.append(t.HMSS()).append(",");
for (Float f : list) {
sbline.append(f).append(",");
}
sbline.append("\n");
outputWriter.write(sbline.toString());
} catch (IOException ex) {
LOG.info(ex.toString());
break;
}
}
}
if (outputWriter != null) {
try {
outputWriter.close();
LOG.info("Closed outputWriter");
} catch (IOException ex) {
Logger.getLogger(OutputFloatQueueReceiver.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
public static void main(String[] args) {
try {
String outputFile = "c:\\tmp\\qtest.csv";
File f = new File(outputFile);
f.delete();
StringBuilder header = new StringBuilder();
header.append("1,2,3,4,5,6,7,8,9");
header.append("\n");
ConcurrentLinkedQueue<List<Float>> outputQueue = null;
OutputFloatQueueReceiver outputQueueReceiver = null;
outputQueue = new ConcurrentLinkedQueue<>();
outputQueueReceiver = new OutputFloatQueueReceiver(outputFile, header.toString(), outputQueue);
outputQueueReceiver.start();
for (int i = 1; i < 100000; i++) {
List<Float> list = new ArrayList<>();
//list.set(0, (float) i); // causes exception
list.add((float) i);
for (int j = 1; j < 10; j++) {
list.add((float) j);
}
outputQueue.add(list);
}
try {
Thread.sleep(5000);
} catch (InterruptedException ex) {
Logger.getLogger(OutputFloatQueueReceiver.class.getName()).log(Level.SEVERE, null, ex);
}
outputQueueReceiver.setRunning(false);
} catch (IOException ex) {
Logger.getLogger(OutputFloatQueueReceiver.class.getName()).log(Level.SEVERE, null, ex);
}
}
}

This code is the reason while your code is using so much CPU:
while (running) {
while (queue.peek() != null) {
// logging
List<Float> list = queue.poll();
if (list == null) {
continue;
}
// do stuff with list
}
}
Basically, your code is busy-waiting, repeatedly "peeking" until a queue entry becomes available. It is probably spinning there in a tight loop.
You should replace your queue class with a BlockingQueue, and simply use take() ... like this:
while (running) {
List<Float> list = queue.take();
// do stuff with list
}
The take() call block indefinitely, only returning once there is an element available, and returning that element as the result. If blocking indefinitely is a problem, you could either use poll(...) with a timeout, or you could arrange that some other thread interrupts the thread that is blocked.

Splitting huge CSV by custom filter?

I have huge (>5GB) CSV file in format:
username,transaction
I want to have as an output separate CSV file for each user with only all of his transactions in the same format. I have few ideas in mind, but i want to hear other ideas for effective (fast and memory efficient) implementation.
Here is what i done up to now. First test is read/process/write in single thread, second test is with many threads. Performance is not that good, so i think i'm doing something wrong. Please correct me.
public class BatchFileReader {
private ICsvBeanReader beanReader;
private double total;
private String[] header;
private CellProcessor[] processors;
private DataTransformer<HashMap<String, List<LoginDto>>> processor;
private boolean hasMoreRecords = true;
public BatchFileReader(String file, DataTransformer<HashMap<String, List<LoginDto>>> processor) {
try {
this.processor = processor;
this.beanReader = new CsvBeanReader(new FileReader(file), CsvPreference.STANDARD_PREFERENCE);
header = CSVUtils.getHeader(beanReader.getHeader(true));
processors = CSVUtils.getProcessors();
} catch (IOException e) {
e.printStackTrace();
}
}
public void read() {
try {
readFile();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (beanReader != null) {
try {
beanReader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
private void readFile() throws IOException {
while (hasMoreRecords) {
long start = System.currentTimeMillis();
HashMap<String, List<LoginDto>> usersBatch = readBatch();
long end = System.currentTimeMillis();
System.out.println("Reading batch for " + ((end - start) / 1000f) + " seconds.");
total +=((end - start)/ 1000f);
if (processor != null && !usersBatch.isEmpty()) {
processor.transform(usersBatch);
}
}
System.out.println("total = " + total);
}
private HashMap<String, List<LoginDto>> readBatch() throws IOException {
HashMap<String, List<LoginDto>> users = new HashMap<String, List<LoginDto>>();
int readLoginCount = 0;
while (readLoginCount < CONFIG.READ_BATCH_SIZE) {
LoginDto login = beanReader.read(LoginDto.class, header, processors);
if (login != null) {
if (!users.containsKey(login.getUsername())) {
List<LoginDto> logins = new LinkedList<LoginDto>();
users.put(login.getUsername(), logins);
}
users.get(login.getUsername()).add(login);
readLoginCount++;
} else {
hasMoreRecords = false;
break;
}
}
return users;
}
}
public class BatchFileWriter {
private final String file;
private final List<T> processedData;
public BatchFileWriter(final String file, List<T> processedData) {
this.file = file;
this.processedData = processedData;
}
public void write() {
try {
writeFile(file, processedData);
} catch (IOException e) {
e.printStackTrace();
} finally {
}
}
private void writeFile(final String file, final List<T> processedData) throws IOException {
System.out.println("START WRITE " + " " + file);
FileWriter writer = new FileWriter(file, true);
long start = System.currentTimeMillis();
for (T record : processedData) {
writer.write(record.toString());
writer.write("\n");
}
writer.flush();
writer.close();
long end = System.currentTimeMillis();
System.out.println("Writing in file " + file + " complete for " + ((end - start) / 1000f) + " seconds.");
}
}
public class LoginsTest {
private static final ExecutorService executor = Executors.newSingleThreadExecutor();
private static final ExecutorService procExec = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() + 1);
#Test
public void testSingleThreadCSVtoCSVSplit() throws InterruptedException, ExecutionException {
long start = System.currentTimeMillis();
DataTransformer<HashMap<String, List<LoginDto>>> simpleSplitProcessor = new DataTransformer<HashMap<String, List<LoginDto>>>() {
#Override
public void transform(HashMap<String, List<LoginDto>> data) {
for (String field : data.keySet()) {
new BatchFileWriter<LoginDto>(field + ".csv", data.get(field)).write();
}
}
};
BatchFileReader reader = new BatchFileReader("loadData.csv", simpleSplitProcessor);
reader.read();
long end = System.currentTimeMillis();
System.out.println("TOTAL " + ((end - start)/ 1000f) + " seconds.");
}
#Test
public void testMultiThreadCSVtoCSVSplit() throws InterruptedException, ExecutionException {
long start = System.currentTimeMillis();
System.out.println(start);
final DataTransformer<HashMap<String, List<LoginDto>>> simpleSplitProcessor = new DataTransformer<HashMap<String, List<LoginDto>>>() {
#Override
public void transform(HashMap<String, List<LoginDto>> data) {
System.out.println("transform");
processAsync(data);
}
};
final CountDownLatch readLatch = new CountDownLatch(1);
executor.execute(new Runnable() {
#Override
public void run() {
BatchFileReader reader = new BatchFileReader("loadData.csv", simpleSplitProcessor);
reader.read();
System.out.println("read latch count down");
readLatch.countDown();
}});
System.out.println("read latch before await");
readLatch.await();
System.out.println("read latch after await");
procExec.shutdown();
executor.shutdown();
long end = System.currentTimeMillis();
System.out.println("TOTAL " + ((end - start)/ 1000f) + " seconds.");
}
private void processAsync(final HashMap<String, List<LoginDto>> data) {
procExec.execute(new Runnable() {
#Override
public void run() {
for (String field : data.keySet()) {
writeASync(field, data.get(field));
}
}
});
}
private void writeASync(final String field, final List<LoginDto> data) {
procExec.execute(new Runnable() {
#Override
public void run() {
new BatchFileWriter<LoginDto>(field + ".csv", data).write();
}
});
}
}

Would it not be better to use unix commands to sort and then split the original file?
Something like: cat txn.csv | sort > txn-sorted.csv
From there get a listing of the unique usernames via grep and then grep the sorted file for each username

If you know Camel already, I'd write a simple Camel route to:
Read line from file
Parse the line
Write to the correct output file
Its a very simple route but if you want it as fast as possible it is then trivially easy make it multithreaded
eg your route would look something like:
from("file:/myfile.csv")
.beanRef("lineParser")
.to("seda:internal-queue");
from("seda:internal-queue")
.concurrentConsumers(5)
.to("fileWriter");
If you don't know Camel then its not worth learning some this one task. However you are probably going to need to make it multithreaded to get the maximum performance. You'll have to experiment where best to put the threading as it will depend on what parts of the operation are slowest.
The multithreading will use up more memory so you'll need to balance memory efficiency against performance.

I would open/append a new output file for each user. If you wanted to minimize memory usage and incur more I/O overhead, you could do something like the following, though you'd probably want to use a real CSV parser like Super CSV (http://supercsv.sourceforge.net/index.html):
Scanner s = new Scanner(new File("/my/dir/users-and-transactions.txt"));
while (s.hasNextLine()) {
String line = s.nextLine();
String[] tokens = line.split(",");
String user = tokens[0];
String transaction = tokens[1];
PrintStream out = new PrintStream(new FileOutputStream("/my/dir/" + user, true));
out.println(transaction);
out.close();
}
s.close();
If you've got a reasonable amount of memory, you could create a Map of user name to OutputStream. Each time you see a user string, you could get the existing OutputStream for that user name or create a new one if none exists.

Watchservice not being triggered [duplicate]

This code works fine in Linux but not in Windows 7: to get file contents update I have to click on the output file. Where is the trick?
I am using Windows 7 prof, NetBeans IDE 8.0 RC1 (Build 201402242200) updated to version NetBeans 8.0 Patch 1.1, JDK 1.8
package watchfilethreadmod;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.file.Files;
import static java.nio.file.LinkOption.NOFOLLOW_LINKS;
import java.nio.file.Path;
import java.nio.file.Paths;
import static java.nio.file.StandardWatchEventKinds.*;
import java.nio.file.WatchEvent;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
public class WatchFileThreadMod {
static class WatchFile {
String fileName;
long lastFilePos;
RandomAccessFile file;
public WatchFile(String _fileName, RandomAccessFile _file) {
fileName = _fileName;
lastFilePos = 0;
file = _file;
}
}
public static void shutDownListener(Thread thread) {
Thread thr = thread;
if (thr != null) {
thr.interrupt();
}
}
private static class MyWatchQueueReader implements Runnable {
/**
* the watchService that is passed in from above
*/
private WatchService myWatcher;
public ArrayList<WatchFile> threadFileToWatch;
public String dirPath;
public MyWatchQueueReader(String _dirPath, WatchService myWatcher, ArrayList<WatchFile> _threadFileToWatch) {
this.myWatcher = myWatcher;
this.threadFileToWatch = _threadFileToWatch;
this.dirPath = _dirPath;
}
private void openFile(WatchFile obj) {
try {
System.out.println("Open file "+obj.fileName);
obj.file = new RandomAccessFile(dirPath + "/" + obj.fileName, "r");
} catch (FileNotFoundException e) {
obj.file = null;
System.out.println("filename " + obj.fileName + " non trovato");
}
obj.lastFilePos = 0;
}
private void process(WatchEvent evt) {
String thisLine;
ArrayList<WatchFile> auxList = threadFileToWatch;
for (WatchFile obj : auxList) {
if (obj.fileName.equals(evt.context().toString())) {
if (obj.file == null) {
openFile(obj);
}
try {
obj.file.seek(obj.lastFilePos);
} catch (IOException e) {
System.err.println("Seek error: " + e);
}
try {
thisLine = obj.file.readLine();
if ((thisLine == null)&&(evt.kind() == ENTRY_MODIFY)) {
System.out.printf("---> thisLine == null received %s event for file: %s\n",
evt.kind(), evt.context());
obj.file.close();
System.out.println("Close file "+obj.fileName);
openFile(obj);
thisLine = obj.file.readLine();
}
while (thisLine != null) { // while loop begins here
if (thisLine.length() > 0) {
if (thisLine.substring(thisLine.length() - 1).equals("*")) {
obj.lastFilePos = obj.file.getFilePointer();
System.out.println(obj.fileName + ": " + thisLine);
}
}
thisLine = obj.file.readLine();
} // end while
} // end try
catch (IOException e) {
System.err.println("Error: " + e);
}
}
}
}
/**
* In order to implement a file watcher, we loop forever ensuring
* requesting to take the next item from the file watchers queue.
*/
#Override
public void run() {
try {
// get the first event before looping
WatchKey key = myWatcher.take();
while (key != null) {
// we have a polled event, now we traverse it and
// receive all the states from it
for (WatchEvent event : key.pollEvents()) {
WatchEvent.Kind eventType = event.kind();
if (eventType == OVERFLOW) {
continue;
}
process(event);
}
key.reset();
key = myWatcher.take();
}
} catch (InterruptedException e) {
ArrayList<WatchFile> auxList = threadFileToWatch;
for (WatchFile obj : auxList) {
if (obj.file != null) {
try {
obj.file.close();
System.out.println("chiusura file " + obj.fileName);
} catch (IOException ex) {
System.out.println("errore in chiusura file");
Logger.getLogger(WatchFileThreadMod.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
//e.printStackTrace();
}
System.out.println("Stopping thread");
}
}
public static void main(String[] args) throws Exception {
// get the directory we want to watch, using the Paths singleton class
//Path toWatch = Paths.get(DIRECTORY_TO_WATCH);
ArrayList<WatchFile> fileToWatch = new ArrayList<>();
String filename;
RandomAccessFile file;
fileToWatch.add(new WatchFile("EURUSD.rlt", new RandomAccessFile(args[0] + "/EURUSD.rlt", "r")));
filename = "EURCHF2.rlt";
try {
file = new RandomAccessFile(args[0] + "/" + filename, "r");
} catch (FileNotFoundException e) {
file = null;
System.out.println("filename " + filename + " non trovato");
}
fileToWatch.add(new WatchFile(filename, file));
fileToWatch = fileToWatch;
Path toWatch = Paths.get(args[0]);
if (toWatch == null) {
throw new UnsupportedOperationException("Directory not found");
}
// Sanity check - Check if path is a folder
try {
Boolean isFolder = (Boolean) Files.getAttribute(toWatch,
"basic:isDirectory", NOFOLLOW_LINKS);
if (!isFolder) {
throw new IllegalArgumentException("Path: " + toWatch + " is not a folder");
}
} catch (IOException ioe) {
// Folder does not exists
ioe.printStackTrace();
}
// make a new watch service that we can register interest in
// directories and files with.
WatchService myWatcher = toWatch.getFileSystem().newWatchService();
// start the file watcher thread below
MyWatchQueueReader fileWatcher = new MyWatchQueueReader(args[0], myWatcher, fileToWatch);
Thread processingThread = new Thread(fileWatcher, "FileWatcher");
processingThread.start();
toWatch.register(myWatcher, ENTRY_CREATE, ENTRY_MODIFY);
}
}
Edit: reduced code as requested.
Edit 2: file path
Edit 3: Metatrader code I am using to write data
#property strict
int file_handle;
string InpFileName = _Symbol + ".rlt"; // File name
input string InpDirectoryName = "Data"; // Folder name
int OnInit()
{
ResetLastError();
file_handle = FileOpen(InpDirectoryName + "//" + InpFileName, FILE_SHARE_READ|FILE_WRITE|FILE_TXT|FILE_ANSI);
if(file_handle == INVALID_HANDLE) {
PrintFormat("Failed to open %s file, Error code = %d", InpFileName, GetLastError());
ExpertRemove();
}
return INIT_SUCCEEDED;
}
void OnTick()
{
// file_handle = FileOpen(InpDirectoryName + "//" + InpFileName, FILE_SHARE_READ|FILE_WRITE|FILE_TXT|FILE_ANSI);
// Datetime), Bid, Volume
// string s = FileRead()
string s = TimeToStr(TimeGMT()) + "|" + Bid + "|" + Volume[0];
FileWriteString(file_handle, s + "|*\r\n");
FileFlush(file_handle);
//FileClose(file_handle);
}
void OnDeinit(const int reason)
{
FileClose(file_handle);
}
Edit 4: Screencast to better show my issue: data updates only when I click on the output file
Watch Service does not update

First of all, a premise : I'm answering this question primarily for future users of WatchService, which (like me) could experience this problem (i.e. on some systems events are signaled way after they occur).
The problem is that the implementation of this feature in Java is native, so it is platform-dependant (take a look at https://docs.oracle.com/javase/7/docs/api/java/nio/file/WatchService.html, under the section 'platform dependencies').
In particular, on Windows 7 (and MacOSX afaict) the implementation uses polling to retrieve the changes from the filesystem, so you can't rely on the 'liveliness' of notifications from a WatchService. The notifications will eventually be signaled, but there are no guarantees on when it will happen.
I don't have a rigorous solution to this problem, but after a lot of trial and error I can describe what works for me :
First, when writing to a file that is registered (i.e. 'watched'), I try to flush the content every time I can and update the 'last modified' attribute on the file, e.g. :
try (FileWriter writer = new FileWriter(outputFile)) {
writer.write("The string to write");
outputFile.setLastModified(System.currentTimeMillis());
writer.flush();
}
Second, I try to 'trigger' the refresh from code (I know it's not good code, but in this case, I'm just happy it works 99% of the time)
Thread.sleep(2000);
// in case I've just created a file and I'm watching the ENTRY_CREATE event on outputDir
outputDir.list();
or (if watching ENTRY_MODIFY on a particular file in outputDir)
Thread.sleep(2000);
outputFile.length();
In both cases, a sleep call simply 'gives the time' to the mechanism underlying the WatchService to trigger, even though 2 seconds are probably a lot more than it is needed.

Probably missing quotes on file path.

Watchservice in windows 7 does not work

This code works fine in Linux but not in Windows 7: to get file contents update I have to click on the output file. Where is the trick?
I am using Windows 7 prof, NetBeans IDE 8.0 RC1 (Build 201402242200) updated to version NetBeans 8.0 Patch 1.1, JDK 1.8
package watchfilethreadmod;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.file.Files;
import static java.nio.file.LinkOption.NOFOLLOW_LINKS;
import java.nio.file.Path;
import java.nio.file.Paths;
import static java.nio.file.StandardWatchEventKinds.*;
import java.nio.file.WatchEvent;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
public class WatchFileThreadMod {
static class WatchFile {
String fileName;
long lastFilePos;
RandomAccessFile file;
public WatchFile(String _fileName, RandomAccessFile _file) {
fileName = _fileName;
lastFilePos = 0;
file = _file;
}
}
public static void shutDownListener(Thread thread) {
Thread thr = thread;
if (thr != null) {
thr.interrupt();
}
}
private static class MyWatchQueueReader implements Runnable {
/**
* the watchService that is passed in from above
*/
private WatchService myWatcher;
public ArrayList<WatchFile> threadFileToWatch;
public String dirPath;
public MyWatchQueueReader(String _dirPath, WatchService myWatcher, ArrayList<WatchFile> _threadFileToWatch) {
this.myWatcher = myWatcher;
this.threadFileToWatch = _threadFileToWatch;
this.dirPath = _dirPath;
}
private void openFile(WatchFile obj) {
try {
System.out.println("Open file "+obj.fileName);
obj.file = new RandomAccessFile(dirPath + "/" + obj.fileName, "r");
} catch (FileNotFoundException e) {
obj.file = null;
System.out.println("filename " + obj.fileName + " non trovato");
}
obj.lastFilePos = 0;
}
private void process(WatchEvent evt) {
String thisLine;
ArrayList<WatchFile> auxList = threadFileToWatch;
for (WatchFile obj : auxList) {
if (obj.fileName.equals(evt.context().toString())) {
if (obj.file == null) {
openFile(obj);
}
try {
obj.file.seek(obj.lastFilePos);
} catch (IOException e) {
System.err.println("Seek error: " + e);
}
try {
thisLine = obj.file.readLine();
if ((thisLine == null)&&(evt.kind() == ENTRY_MODIFY)) {
System.out.printf("---> thisLine == null received %s event for file: %s\n",
evt.kind(), evt.context());
obj.file.close();
System.out.println("Close file "+obj.fileName);
openFile(obj);
thisLine = obj.file.readLine();
}
while (thisLine != null) { // while loop begins here
if (thisLine.length() > 0) {
if (thisLine.substring(thisLine.length() - 1).equals("*")) {
obj.lastFilePos = obj.file.getFilePointer();
System.out.println(obj.fileName + ": " + thisLine);
}
}
thisLine = obj.file.readLine();
} // end while
} // end try
catch (IOException e) {
System.err.println("Error: " + e);
}
}
}
}
/**
* In order to implement a file watcher, we loop forever ensuring
* requesting to take the next item from the file watchers queue.
*/
#Override
public void run() {
try {
// get the first event before looping
WatchKey key = myWatcher.take();
while (key != null) {
// we have a polled event, now we traverse it and
// receive all the states from it
for (WatchEvent event : key.pollEvents()) {
WatchEvent.Kind eventType = event.kind();
if (eventType == OVERFLOW) {
continue;
}
process(event);
}
key.reset();
key = myWatcher.take();
}
} catch (InterruptedException e) {
ArrayList<WatchFile> auxList = threadFileToWatch;
for (WatchFile obj : auxList) {
if (obj.file != null) {
try {
obj.file.close();
System.out.println("chiusura file " + obj.fileName);
} catch (IOException ex) {
System.out.println("errore in chiusura file");
Logger.getLogger(WatchFileThreadMod.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
//e.printStackTrace();
}
System.out.println("Stopping thread");
}
}
public static void main(String[] args) throws Exception {
// get the directory we want to watch, using the Paths singleton class
//Path toWatch = Paths.get(DIRECTORY_TO_WATCH);
ArrayList<WatchFile> fileToWatch = new ArrayList<>();
String filename;
RandomAccessFile file;
fileToWatch.add(new WatchFile("EURUSD.rlt", new RandomAccessFile(args[0] + "/EURUSD.rlt", "r")));
filename = "EURCHF2.rlt";
try {
file = new RandomAccessFile(args[0] + "/" + filename, "r");
} catch (FileNotFoundException e) {
file = null;
System.out.println("filename " + filename + " non trovato");
}
fileToWatch.add(new WatchFile(filename, file));
fileToWatch = fileToWatch;
Path toWatch = Paths.get(args[0]);
if (toWatch == null) {
throw new UnsupportedOperationException("Directory not found");
}
// Sanity check - Check if path is a folder
try {
Boolean isFolder = (Boolean) Files.getAttribute(toWatch,
"basic:isDirectory", NOFOLLOW_LINKS);
if (!isFolder) {
throw new IllegalArgumentException("Path: " + toWatch + " is not a folder");
}
} catch (IOException ioe) {
// Folder does not exists
ioe.printStackTrace();
}
// make a new watch service that we can register interest in
// directories and files with.
WatchService myWatcher = toWatch.getFileSystem().newWatchService();
// start the file watcher thread below
MyWatchQueueReader fileWatcher = new MyWatchQueueReader(args[0], myWatcher, fileToWatch);
Thread processingThread = new Thread(fileWatcher, "FileWatcher");
processingThread.start();
toWatch.register(myWatcher, ENTRY_CREATE, ENTRY_MODIFY);
}
}
Edit: reduced code as requested.
Edit 2: file path
Edit 3: Metatrader code I am using to write data
#property strict
int file_handle;
string InpFileName = _Symbol + ".rlt"; // File name
input string InpDirectoryName = "Data"; // Folder name
int OnInit()
{
ResetLastError();
file_handle = FileOpen(InpDirectoryName + "//" + InpFileName, FILE_SHARE_READ|FILE_WRITE|FILE_TXT|FILE_ANSI);
if(file_handle == INVALID_HANDLE) {
PrintFormat("Failed to open %s file, Error code = %d", InpFileName, GetLastError());
ExpertRemove();
}
return INIT_SUCCEEDED;
}
void OnTick()
{
// file_handle = FileOpen(InpDirectoryName + "//" + InpFileName, FILE_SHARE_READ|FILE_WRITE|FILE_TXT|FILE_ANSI);
// Datetime), Bid, Volume
// string s = FileRead()
string s = TimeToStr(TimeGMT()) + "|" + Bid + "|" + Volume[0];
FileWriteString(file_handle, s + "|*\r\n");
FileFlush(file_handle);
//FileClose(file_handle);
}
void OnDeinit(const int reason)
{
FileClose(file_handle);
}
Edit 4: Screencast to better show my issue: data updates only when I click on the output file
Watch Service does not update

First of all, a premise : I'm answering this question primarily for future users of WatchService, which (like me) could experience this problem (i.e. on some systems events are signaled way after they occur).
The problem is that the implementation of this feature in Java is native, so it is platform-dependant (take a look at https://docs.oracle.com/javase/7/docs/api/java/nio/file/WatchService.html, under the section 'platform dependencies').
In particular, on Windows 7 (and MacOSX afaict) the implementation uses polling to retrieve the changes from the filesystem, so you can't rely on the 'liveliness' of notifications from a WatchService. The notifications will eventually be signaled, but there are no guarantees on when it will happen.
I don't have a rigorous solution to this problem, but after a lot of trial and error I can describe what works for me :
First, when writing to a file that is registered (i.e. 'watched'), I try to flush the content every time I can and update the 'last modified' attribute on the file, e.g. :
try (FileWriter writer = new FileWriter(outputFile)) {
writer.write("The string to write");
outputFile.setLastModified(System.currentTimeMillis());
writer.flush();
}
Second, I try to 'trigger' the refresh from code (I know it's not good code, but in this case, I'm just happy it works 99% of the time)
Thread.sleep(2000);
// in case I've just created a file and I'm watching the ENTRY_CREATE event on outputDir
outputDir.list();
or (if watching ENTRY_MODIFY on a particular file in outputDir)
Thread.sleep(2000);
outputFile.length();
In both cases, a sleep call simply 'gives the time' to the mechanism underlying the WatchService to trigger, even though 2 seconds are probably a lot more than it is needed.

Probably missing quotes on file path.

Locking files while using Java Logger

I am creating a logger that will log things throughout my program. It seems to work fine. This is what the class looks like.
public class TESTLogger {
protected static File file;
protected static Logger logger = Logger.getLogger("");
public TESTLogger(String logName) {
setupLogger(logName);
}
private static void setupLogger(String logName) {
String basePath = Utils.getBasePath();
File logsDir = new File(basePath);
if(logsDir.exists() == false) {
logsDir.mkdir();
}
String filePath = basePath + File.separator + logName + ".%g.log";
file = new File(filePath);
try {
FileHandler fileHandler = new FileHandler(filePath, 5242880, 5, true);
fileHandler.setFormatter(new java.util.logging.Formatter() {
#Override
public String format(LogRecord logRecord) {
if(logRecord.getLevel() == Level.INFO) {
return "[INFO " + createDateTimeLog() + "] " + logRecord.getMessage() + "\r\n";
} else if(logRecord.getLevel() == Level.WARNING) {
return "[WARN " + createDateTimeLog() + "] " + logRecord.getMessage() + "\r\n";
} else if(logRecord.getLevel() == Level.SEVERE) {
return "[ERROR " + createDateTimeLog() + "] " + logRecord.getMessage() + "\r\n";
} else {
return "[OTHER " + createDateTimeLog() + "] " + logRecord.getMessage() + "\r\n";
}
}
});
logger.addHandler(fileHandler);
} catch (IOException e) {
}
}
private static void writeToFile(Level level, String writeThisToFile) {
logger.log(level, writeThisToFile);
}
private static String createDateTimeLog() {
String dateTime = "";
Date date = new Date();
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd H:mm:ss");
dateTime = simpleDateFormat.format(date);
return dateTime;
}
public void error(String message) {
writeToFile(Level.SEVERE, message);
}
public void warn(String message) {
writeToFile(Level.WARNING, message);
}
public void info(String message) {
writeToFile(Level.INFO, message);
}
}
When my application starts it creats the TESTLogger object. Then whenever I log I run logger.info / logger.warn / logger.error with my log message. That is working great. However, multiple instances of my jar can be running at the same time. When that happens, it creates a new instance of the log. IE: I could have myLog.0.log. When the second instance of the jar logs something it will go under myLog.0.log.1, then myLog.0.log.2 and so on.
I don't want to create all these different instances of my log file. I thought I might use a File Lock (from java.nio.channels package). However, I have not been able to figure out how to do that with the Java Logger class I am using (java.util.logging).
Any ideas how to prevent this from happening would be great. Thanks in advance.
EDIT:
Ok. So I have rewritten writeToFile and it seems to work a little better. However, every now and again I still get a .1 log. It doesn't happen as much as it used to. And it NEVER gets to .2 (it used to get all the way up to .100). I would still like to prevent this .1, though.
This is what my code looks like now:
private static void writeToFile(Level level, String writeThisToFile) {
try {
File file = new File("FileLock");
FileChannel channel = new RandomAccessFile(file, "rw").getChannel();
FileLock lock = null;
try {
lock = channel.tryLock(0, Long.MAX_VALUE, true);
if(lock != null) {
logger.log(level, writeThisToFile);
}
} catch (OverlappingFileLockException e) {
}
finally {
if(lock != null) {
lock.release();
}
channel.close();
}
} catch (IOException e) {}
}
EDIT #2: What it currently looks like.
Entrance point into my JAR:
public class StartingPoint {
public static void main(String[] args) {
MyLogger logger = new MyLogger("myFirstLogger");
logger.info("Info test message");
logger.warn("Warning test message");
logger.error("Error test message");
}
}
MyLogger class:
public class MyLogger {
protected static File file;
protected static Logger logger = Logger.getLogger("");
public MyLogger(String loggerName) {
setupLogger(loggerName);
}
private void setupLogger(String loggerName) {
String filePath = loggerName + "_%g" + ".log";
file = new File(filePath);
try {
FileHandler fileHandler = new FileHandler(filePath, 5242880, 5, true);
fileHandler.setFormatter(new java.util.logging.Formatter() {
#Override
public String format(LogRecord logRecord) {
if(logRecord.getLevel() == Level.INFO) {
return "[INFO " + createDateTimeLog() + "] " + logRecord.getMessage() + "\r\n";
} else if(logRecord.getLevel() == Level.WARNING) {
return "[WARN " + createDateTimeLog() + "] " + logRecord.getMessage() + "\r\n";
} else if(logRecord.getLevel() == Level.SEVERE) {
return "[ERROR " + createDateTimeLog() + "] " + logRecord.getMessage() + "\r\n";
} else {
return "[OTHER " + createDateTimeLog() + "] " + logRecord.getMessage() + "\r\n";
}
}
});
logger.addHandler(fileHandler);
logger.addHandler(new SharedFileHandler()); // <--- SharedFileHandler added
} catch (IOException e) {}
}
private void writeToFile(Level level, String writeThisToFile) {
logger.log(level, writeThisToFile);
}
private static String createDateTimeLog() {
String dateTime = "";
Date date = new Date();
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd H:mm:ss");
dateTime = simpleDateFormat.format(date);
return dateTime;
}
public void error(String message) {
writeToFile(Level.SEVERE, message);
}
public void warn(String message) {
writeToFile(Level.WARNING, message);
}
public void info(String message) {
writeToFile(Level.INFO, message);
}
}
And finally... SharedFileHandler:
public class SharedFileHandler extends Handler {
private final FileChannel mutex;
private final String pattern;
public SharedFileHandler() throws IOException {
this("loggerLockFile");
}
public SharedFileHandler(String pattern) throws IOException {
setFormatter(new SimpleFormatter());
this.pattern = pattern;
mutex = new RandomAccessFile(pattern, "rw").getChannel();
}
#Override
public void publish(LogRecord record) {
if (isLoggable(record)) {
record.getSourceMethodName(); //Infer caller.
try {
FileLock ticket = mutex.lock();
try {
doPublish(record);
} finally {
ticket.release();
}
} catch (IOException e) {}
catch (OverlappingFileLockException e) {}
catch (NullPointerException e) {}
}
}
private void doPublish(LogRecord record) throws IOException {
final FileHandler h = new FileHandler(pattern, 5242880, 5, true);
try {
h.setEncoding(getEncoding());
h.setErrorManager(getErrorManager());
h.setFilter(getFilter());
h.setFormatter(getFormatter());
h.setLevel(getLevel());
h.publish(record);
h.flush();
} finally {
h.close();
}
}
#Override
public void flush() {}
#Override
public synchronized void close() throws SecurityException {
super.setLevel(Level.OFF);
try {
mutex.close();
} catch (IOException ioe) {}
}
}

The FileHandler does everything it can to prevent two concurrently running JVMs from writing to the same log file. If this behavior was allowed the log file would be almost impossible to read and understand.
If you really want to write everything to one log file then you have to do one of the following:
Prevent concurrent JVM processes from starting by changing how it is launched.
Have your code detect if another JVM is running your code and exit before creating a FileHandler.
Have each JVM write to a distinct log file and create code to safely merge the files into one.
Create a proxy Handler that creates and closes a FileHandler for each log record. The proxy handler would use a predefined file name (different from the log file) and a FileLock to serialize access to the log file from different JVMs.
Use a dedicated process to write to the log file and have all the JVMs send log messages to that process.
Here is an untested example of a proxy handler:
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
import java.nio.file.Paths;
import java.util.logging.*;
import static java.nio.file.StandardOpenOption.*;
public class SharedFileHandler extends Handler {
private final FileChannel mutex;
private final String pattern;
public SharedFileHandler() throws IOException {
this("%hjava%g.log");
}
public SharedFileHandler(String pattern) throws IOException {
setFormatter(new SimpleFormatter());
this.pattern = pattern;
Path p = Paths.get(new File(".").getCanonicalPath(),
pattern.replace("%", "") + ".lck");
mutex = FileChannel.open(p, CREATE, WRITE, DELETE_ON_CLOSE);
}
#Override
public void publish(LogRecord record) {
if (isLoggable(record)) {
record.getSourceMethodName(); //Infer caller.
try {
FileLock ticket = mutex.lock();
try {
doPublish(ticket, record);
} finally {
ticket.release();
}
} catch (IOException | OverlappingFileLockException ex) {
reportError(null, ex, ErrorManager.WRITE_FAILURE);
}
}
}
private synchronized void doPublish(FileLock ticket, LogRecord record) throws IOException {
if (!ticket.isValid()) {
return;
}
final FileHandler h = new FileHandler(pattern, 5242880, 5, true);
try {
h.setEncoding(getEncoding());
h.setErrorManager(getErrorManager());
h.setFilter((Filter) null);
h.setFormatter(getFormatter());
h.setLevel(getLevel());
h.publish(record);
h.flush();
} finally {
h.close();
}
}
#Override
public void flush() {
}
#Override
public synchronized void close() throws SecurityException {
super.setLevel(Level.OFF);
try {
mutex.close();
} catch (IOException ioe) {
this.reportError(null, ioe, ErrorManager.CLOSE_FAILURE);
}
}
}
Here is a simple test case
public static void main(String[] args) throws Exception {
Random rnd = new Random();
logger.addHandler(new SharedFileHandler());
String id = ManagementFactory.getRuntimeMXBean().getName();
for (int i = 0; i < 600; i++) {
logger.log(Level.INFO, id);
Thread.sleep(rnd.nextInt(100));
}
}