We Keep Coding

Is there a better way to generate 5 million csv files quickly

I would like to create 5 million csv files, I have waiting for almost 3 hours, but the program is still running. Can somebody give me some advice, how to speed up the file generation.
After these 5 million files generation complete, I have to upload them to s3 bucket.
It would be better if someone know how to generate these files through AWS, thus, we can move files to s3 bucket directly and ignore network speed issue.(Just start to learning AWS, there are lots of knowledge need to know)
The following is my code.
public class ParallelCsvGenerate implements Runnable {
private static AtomicLong baseID = new AtomicLong(8160123456L);
private static ThreadLocalRandom random = ThreadLocalRandom.current();
private static ThreadLocalRandom random2 = ThreadLocalRandom.current();
private static String filePath = "C:\\5millionfiles\\";
private static List<String> headList = null;
private static String csvHeader = null;
public ParallelCsvGenerate() {
headList = generateHeadList();
csvHeader = String.join(",", headList);
}
#Override
public void run() {
for(int i = 0; i < 1000000; i++) {
generateCSV();
}s
}
private void generateCSV() {
StringBuilder builder = new StringBuilder();
builder.append(csvHeader).append(System.lineSeparator());
for (int i = 0; i < headList.size(); i++) {
if(i < headList.size() - 1) {
builder.append(i % 2 == 0 ? generateRandomInteger() : generateRandomStr()).append(",");
} else {
builder.append(i % 2 == 0 ? generateRandomInteger() : generateRandomStr());
}
}
String fileName = String.valueOf(baseID.addAndGet(1));
File csvFile = new File(filePath + fileName + ".csv");
FileWriter fileWriter = null;
try {
fileWriter = new FileWriter(csvFile);
fileWriter.write(builder.toString());
fileWriter.flush();
} catch (Exception e) {
System.err.println(e);
} finally {
try {
if(fileWriter != null) {
fileWriter.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
private static List<String> generateHeadList() {
List<String> headList = new ArrayList<>(20);
String baseFiledName = "Field";
for(int i = 1; i <=20; i++) {
headList.add(baseFiledName + i);
}
return headList;
}
/**
* generate a number in range of 0-50000
* #return
*/
private Integer generateRandomInteger() {
return random.nextInt(0,50000);
}
/**
* generate a string length is 5 - 8
* #return
*/
private String generateRandomStr() {
int strLength = random2.nextInt(5, 8);
String str="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
int length = str.length();
StringBuilder builder = new StringBuilder();
for (int i = 0; i < strLength; i++) {
builder.append(str.charAt(random.nextInt(length)));
}
return builder.toString();
}
Main
ParallelCsvGenerate generate = new ParallelCsvGenerate();
Thread a = new Thread(generate, "A");
Thread b = new Thread(generate, "B");
Thread c = new Thread(generate, "C");
Thread d = new Thread(generate, "D");
Thread e = new Thread(generate, "E");
a.run();
b.run();
c.run();
d.run();
e.run();
Thanks for your guys advice, just refactor the code, and generate 3.8million files using 2.8h, which is much better.
Refactor code:
public class ParallelCsvGenerate implements Callable<Integer> {
private static String filePath = "C:\\5millionfiles\\";
private static String[] header = new String[]{
"FIELD1","FIELD2","FIELD3","FIELD4","FIELD5",
"FIELD6","FIELD7","FIELD8","FIELD9","FIELD10",
"FIELD11","FIELD12","FIELD13","FIELD14","FIELD15",
"FIELD16","FIELD17","FIELD18","FIELD19","FIELD20",
};
private String fileName;
public ParallelCsvGenerate(String fileName) {
this.fileName = fileName;
}
#Override
public Integer call() throws Exception {
try {
generateCSV();
} catch (IOException e) {
e.printStackTrace();
}
return 0;
}
private void generateCSV() throws IOException {
CSVWriter writer = new CSVWriter(new FileWriter(filePath + fileName + ".csv"), CSVWriter.DEFAULT_SEPARATOR, CSVWriter.NO_QUOTE_CHARACTER);
String[] content = new String[]{
RandomGenerator.generateRandomInteger(),
RandomGenerator.generateRandomStr(),
RandomGenerator.generateRandomInteger(),
RandomGenerator.generateRandomStr(),
RandomGenerator.generateRandomInteger(),
RandomGenerator.generateRandomStr(),
RandomGenerator.generateRandomInteger(),
RandomGenerator.generateRandomStr(),
RandomGenerator.generateRandomInteger(),
RandomGenerator.generateRandomStr(),
RandomGenerator.generateRandomInteger(),
RandomGenerator.generateRandomStr(),
RandomGenerator.generateRandomInteger(),
RandomGenerator.generateRandomStr(),
RandomGenerator.generateRandomInteger(),
RandomGenerator.generateRandomStr(),
RandomGenerator.generateRandomInteger(),
RandomGenerator.generateRandomStr(),
RandomGenerator.generateRandomInteger(),
RandomGenerator.generateRandomStr()
};
writer.writeNext(header);
writer.writeNext(content);
writer.close();
}
}
Main
public static void main(String[] args) {
System.out.println("Start generate");
long start = System.currentTimeMillis();
ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(8, 8,
0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>());
List<ParallelCsvGenerate> taskList = new ArrayList<>(3800000);
for(int i = 0; i < 3800000; i++) {
taskList.add(new ParallelCsvGenerate(i+""));
}
try {
List<Future<Integer>> futures = threadPoolExecutor.invokeAll(taskList);
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("Success");
long end = System.currentTimeMillis();
System.out.println("Using time: " + (end-start));
}

You could write directly into the file (without allocating the whole file in one StringBuilder). (I think this is the biggest time+memory bottleneck here: builder.toString())
You could generate each file in parallel.
(little tweaks:) Omit the if's inside loop.
if(i < headList.size() - 1) is not needed, when you do a more clever loop + 1 extra iteration.
The i % 2 == 0 can be eliminated by a better iteration (i+=2) ..and more labor inside the loop (i -> int, i + 1 -> string)
If applicable prefer append(char) to append(String). (Better append(',') than append(",")!)
...

You can use Fork/Join framework (java 7 and above) to make your process in parallel and use multi core of your Cpu.
I'll take an example for you.
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinTask;
import java.util.concurrent.RecursiveTask;
import java.util.stream.LongStream;
public class ForkJoinAdd extends RecursiveTask<Long> {
private final long[] numbers;
private final int start;
private final int end;
public static final long threshold = 10_000;
public ForkJoinAdd(long[] numbers) {
this(numbers, 0, numbers.length);
}
private ForkJoinAdd(long[] numbers, int start, int end) {
this.numbers = numbers;
this.start = start;
this.end = end;
}
#Override
protected Long compute() {
int length = end - start;
if (length <= threshold) {
return add();
}
ForkJoinAdd firstTask = new ForkJoinAdd(numbers, start, start + length / 2);
firstTask.fork(); //start asynchronously
ForkJoinAdd secondTask = new ForkJoinAdd(numbers, start + length / 2, end);
Long secondTaskResult = secondTask.compute();
Long firstTaskResult = firstTask.join();
return firstTaskResult + secondTaskResult;
}
private long add() {
long result = 0;
for (int i = start; i < end; i++) {
result += numbers[i];
}
return result;
}
public static long startForkJoinSum(long n) {
long[] numbers = LongStream.rangeClosed(1, n).toArray();
ForkJoinTask<Long> task = new ForkJoinAdd(numbers);
return new ForkJoinPool().invoke(task);
}
}
use this example
And if you want to read more about it, Guide to the Fork/Join Framework in Java | Baeldung
and Fork/Join (The Java™ Tutorials
can help you to better understand and better design your app.
be lucky.

Remove the for(int i = 0; i < 1000000; i++) loop from run method (leave a single generateCSV() call.
Create 5 million ParallelCsvGenerate objects.
Submit them to a ThreadPoolExecutor
Converted main:
public static void main(String[] args) {
ThreadPoolExecutor ex = (ThreadPoolExecutor) Executors.newFixedThreadPool(8);
for(int i = 0; i < 5000000; i++) {
ParallelCsvGenerate generate = new ParallelCsvGenerate();
ex.submit(generate);
}
ex.shutdown();
}
It takes roughly 5 minutes to complete on my laptop (4 physical cores with hyperthreading, SSD drive).
EDIT:
I've replaced FileWriter with AsynchronousFileChannel using the following code:
Path file = Paths.get(filePath + fileName + ".csv");
try(AsynchronousFileChannel asyncFile = AsynchronousFileChannel.open(file,
StandardOpenOption.WRITE,
StandardOpenOption.CREATE)) {
asyncFile.write(ByteBuffer.wrap(builder.toString().getBytes()), 0);
} catch (IOException e) {
e.printStackTrace();
}
to achieve 30% speedup.
I believe that the main bottleneck is the hard drive and filesystem itself. Not much more can be achieved here.

How can I improve the performance of execution time? And Is their any better way to read this file?

I am trying to split a text file with multiple threads. The file is of 1 GB. I am reading the file by char. The Execution time is 24 min 54 seconds. Instead of reading a file by char is their any better way where I can reduce the execution time.
I'm having a hard time figuring out an approach that will reduce the execution time. Please do suggest me also, if there is any other better way to split file with multiple threads. I am very new to java.
Any help will be appreciated. :)
public static void main(String[] args) throws Exception {
RandomAccessFile raf = new RandomAccessFile("D:\\sample\\file.txt", "r");
long numSplits = 10;
long sourceSize = raf.length();
System.out.println("file length:" + sourceSize);
long bytesPerSplit = sourceSize / numSplits;
long remainingBytes = sourceSize % numSplits;
int maxReadBufferSize = 9 * 1024;
List<String> filePositionList = new ArrayList<String>();
long startPosition = 0;
long endPosition = bytesPerSplit;
for (int i = 0; i < numSplits; i++) {
raf.seek(endPosition);
String strData = raf.readLine();
if (strData != null) {
endPosition = endPosition + strData.length();
}
String str = startPosition + "|" + endPosition;
if (sourceSize > endPosition) {
startPosition = endPosition;
endPosition = startPosition + bytesPerSplit;
} else {
break;
}
filePositionList.add(str);
}
for (int i = 0; i < filePositionList.size(); i++) {
String str = filePositionList.get(i);
String[] strArr = str.split("\\|");
String strStartPosition = strArr[0];
String strEndPosition = strArr[1];
long startPositionFile = Long.parseLong(strStartPosition);
long endPositionFile = Long.parseLong(strEndPosition);
MultithreadedSplit objMultithreadedSplit = new MultithreadedSplit(startPositionFile, endPositionFile);
objMultithreadedSplit.start();
}
long endTime = System.currentTimeMillis();
System.out.println("It took " + (endTime - startTime) + " milliseconds");
}
}
public class MultithreadedSplit extends Thread {
public static String filePath = "D:\\tenlakh\\file.txt";
private int localCounter = 0;
private long start;
private long end;
public static String outPath;
List<String> result = new ArrayList<String>();
public MultithreadedSplit(long startPos, long endPos) {
start = startPos;
end = endPos;
}
#Override
public void run() {
try {
String threadName = Thread.currentThread().getName();
long currentTime = System.currentTimeMillis();
RandomAccessFile file = new RandomAccessFile("D:\\sample\\file.txt", "r");
String outFile = "out_" + threadName + ".txt";
System.out.println("Thread Reading started for start:" + start + ";End:" + end+";threadname:"+threadName);
FileOutputStream out2 = new FileOutputStream("D:\\sample\\" + outFile);
file.seek(start);
int nRecordCount = 0;
char c = (char) file.read();
StringBuilder objBuilder = new StringBuilder();
int nCounter = 1;
while (c != -1) {
objBuilder.append(c);
// System.out.println("char-->" + c);
if (c == '\n') {
nRecordCount++;
out2.write(objBuilder.toString().getBytes());
objBuilder.delete(0, objBuilder.length());
//System.out.println("--->" + nRecordCount);
// break;
}
c = (char) file.read();
nCounter++;
if (nCounter > end) {
break;
}
}
} catch (Exception ex) {
ex.printStackTrace();
}
}
}

The fastest way would be to map the file into memory segment by segment (mapping a large file as a whole may cause undesired side effects). It will skip few relatively expensive copy operations. The operating system will load file into RAM and JRE will expose it to your application as a view into an off-heap memory area in a form of a ByteBuffer. It would usually allow you to squeze last 2x/3x of the performance.
Memory-mapped way requires quite a bit of helper code (see the fragment in the bottom), it's not always the best tactical way. Instead, if your input is line-based and you just need reasonable performance (what you have now is probably not) then just do something like:
import java.nio.Files;
import java.nio.Paths;
...
File.lines(Paths.get("/path/to/the/file"), StandardCharsets.ISO_8859_1)
// .parallel() // parallel processing is still possible
.forEach(line -> { /* your code goes here */ });
For the contrast, a working example of the code for working with the file via memory mapping would look something like below. In case of fixed-size records (when segments can be selected precisely to match record boundaries) subsequent segments can be processed in parallel.
static ByteBuffer mapFileSegment(FileChannel fileChannel, long fileSize, long regionOffset, long segmentSize) throws IOException {
long regionSize = min(segmentSize, fileSize - regionOffset);
// small last region prevention
final long remainingSize = fileSize - (regionOffset + regionSize);
if (remainingSize < segmentSize / 2) {
regionSize += remainingSize;
}
return fileChannel.map(FileChannel.MapMode.READ_ONLY, regionOffset, regionSize);
}
...
final ToIntFunction<ByteBuffer> consumer = ...
try (FileChannel fileChannel = FileChannel.open(Paths.get("/path/to/file", StandardOpenOption.READ)) {
final long fileSize = fileChannel.size();
long regionOffset = 0;
while (regionOffset < fileSize) {
final ByteBuffer regionBuffer = mapFileSegment(fileChannel, fileSize, regionOffset, segmentSize);
while (regionBuffer.hasRemaining()) {
final int usedBytes = consumer.applyAsInt(regionBuffer);
if (usedBytes == 0)
break;
}
regionOffset += regionBuffer.position();
}
} catch (IOException ex) {
throw new UncheckedIOException(ex);
}

Java : URLConnection setRequestProperty range issue relying on multithreaded process?

I'm stuck trying to read by range a remote file.
According following trace sounds setRequestProperty is not performing its job (InputStream Length is nok to me except one .. the first one ???) ?
Task Runnable ID = 0 --> Requested Range Property = {Range=[bytes=0-93969]}
Task Runnable ID = 1 --> Resquested Range Property = {Range=[bytes=93970-187939]}
Task Runnable ID = 2 --> Requested Range Property = {Range=[bytes=187940-281909]}
Task Runnable ID = 3 --> Resquested Range Property = {Range=[bytes=281910-375883]}
Task Runnable ID = 0 --> InputStream Lenght = 93970 / StartByte = 0 / CurrentByte = 0 / EndByte = 93969
Task Runnable ID = 1 --> InputStream Length = 375883 / StartByte = 93970 / CurrentByte = 93970 / EndByte = 187939
Task Runnable ID = 3 --> InputStream Length = 375883 / StartByte = 281910 / CurrentByte = 281910 / EndByte = 375883
Task Runnable ID = 2 --> InputStream Length = 375883 / StartByte = 187940 / CurrentByte = 187940 / EndByte = 281909
My code is :
public class TaskRunnable implements Runnable {
private static final int BUFFER_SIZE = 4092;
private long startByte;
private long currentByte;
private long endByte;
private Task task;
private static int idCounter = 0;
private int id;
#SuppressWarnings("unused")
private TaskRunnable() {
}
public TaskRunnable(Task task, long startByte, long endByte) {
this.startByte = startByte;
this.endByte = endByte;
this.task = task;
this.currentByte = startByte;
this.id = idCounter++;
}
#Override
public void run() {
Thread.currentThread().setName("Download Runnable");
Authenticator authenticator = task.getManager().getAuthenticator();
if (authenticator != null) {
Authenticator.setDefault(authenticator);
}
File targetFile;
synchronized (this) {
targetFile = new File(task.getTargetFile().getAbsolutePath());
}
BufferedInputStream bufferedInputStream = null;
byte[] buf = new byte[BUFFER_SIZE];
URLConnection urlConnection = null;
try {
URL _url = new URL(task.getSourceFileUrl().toString());
Proxy proxy = task.getManager().getProxy();
if (proxy != null) {
urlConnection = _url.openConnection(proxy);
} else {
urlConnection = _url.openConnection();
}
urlConnection.setRequestProperty("Range", "bytes=" + currentByte + "-" + endByte);
System.out.println("Task Runnable ID = " + id + " --> Requested Range Property = " + urlConnection.getRequestProperties().toString());
bufferedInputStream = new BufferedInputStream(urlConnection.getInputStream());
int len = 0;
while (bufferedInputStream.read() != -1) {
len++;
}
System.out.println("Task Runnable ID = " + id + " --> InputStream Length = " + len + " / StartByte = " + startByte + " / CurrentByte = " + currentByte + " / EndByte = " + endByte);
bufferedInputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
Obviously it's my fault but not able to figure out what's wrong. Help more than welcome. Thks !
Note : all is running fine if using same code but single thread based.

I suspect a concurrent modification of the internal URLConnection state. Have you tried to invoke URLConnection.setUseCaches(false) ?
Update
Another global cache that affects a possible reusing of URLConnection objects is ResponseCache. You should disable it at initialization by executing
ResponseCache.setDefault(null);

Android internet speed test

I am developing a speed test app like OKLA app (http://www.speedtest.net/).
I've been trying to get bandwidth rate with the most common approach:
Get the time before downloading.
Download some file for some time X.
Get the time after downloading and the total size downloaded.
Calculate speed from TIME and BYTES RECEIVED.
Also, I execute this in two different threads at the same time because is required to saturate de connection to achieve good results.
This approach works very well on PC environment with this JAVA code:
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.FutureTask;
public class TestVelocidad {
static long totalBytesReceived = 0; //
static long startSample;
static long endSample ;
private static final long TIME_FOR_DOWNLOAD_MILISECONDS = (long) 10000.0;
private static final long MILI_TO_NANO = 1000000;
public static void main(String[] args) throws InterruptedException, ExecutionException {
try{
final ExecutorService service;
String downloadFileUrl100MB = "http://cachefly.cachefly.net/100mb.test";
startSample = System.nanoTime();
service = Executors.newFixedThreadPool(6);
FutureTask futureTask_1 = new FutureTask(new SpeedTestThread(downloadFileUrl100MB));
service.execute(futureTask_1);
FutureTask futureTask_2 = new FutureTask(new SpeedTestThread(downloadFileUrl100MB));
service.execute(futureTask_2);
service.shutdownNow();
long result1 = (Long) futureTask_1.get();
long result2 = (Long) futureTask_2.get();
endSample = System.nanoTime();
long timeSpent = (long) endSample-startSample;
long totalBytesReceived = result1 + result2;
System.out.println("Time of threads: " + timeSpent/1000000000.0 + " seconds " + "\nbytes received: " + (totalBytesReceived) );
double calculatedSpeed;
// long finalTimeSpent ;
// finalTimeSpent = (long) ((TIME_FOR_DOWNLOAD_MILISECONDS * MILI_TO_NANO - diff));
calculatedSpeed = SpeedInfo.calculate(timeSpent, totalBytesReceived).megabits;
System.out.println("Velocidad calculada: " + calculatedSpeed + " mbps" );
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
}
class SpeedTestThread implements Callable<Long> {
private String url = new String("");
private static final long TIME_FOR_DOWNLOAD_NANOSECONDS = (long) 10000000000.0;
private static final long MILI_TO_NANO = 1000000;
private long bytesThread;
public SpeedTestThread(String urlToDownload){
url = urlToDownload;
}
public void run() {
}
#Override
public Long call() throws Exception {
System.out.println("FileDownload " + " File to download: " + url );
InputStream stream = null;
long startCon = System.nanoTime();
URL urlToDownload = null;
try {
urlToDownload = new URL(url);
} catch (MalformedURLException e) {
e.printStackTrace();
}
URLConnection con = null;
try {
con = urlToDownload.openConnection();
} catch (IOException e) {
e.printStackTrace();
}
con.setUseCaches(false);
//Tiempo de acceso al archivo.
long connectionLatency = (System.nanoTime() - startCon)/MILI_TO_NANO;
System.out.println("Connection latency = " + connectionLatency + "");
con.setConnectTimeout(5000);
try {
stream = con.getInputStream();
} catch (IOException e) {
e.printStackTrace();
}
long startNano = System.nanoTime();
int currentByte = 0;
try {
while ((currentByte = stream.read()) != -1 ) {
bytesThread++;
if ((System.nanoTime() - startNano) > TIME_FOR_DOWNLOAD_NANOSECONDS){
System.out.println("Time");
break;
}
}
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("Thread bytes received: " + bytesThread);
return bytesThread;
}
}
class SpeedInfo {
public double kilobits = 0;
public double megabits = 0;
public double downspeed = 0;
private static final double BYTE_TO_KILOBIT = 0.008;
private static final double KILOBIT_TO_MEGABIT = 0.001;
/**
* 1 byte = 0.0078125 kilobits
* 1 kilobits = 0.0009765625 megabit
*
* #param downloadTime in miliseconds
* #param bytesIn number of bytes downloaded
* #return SpeedInfo containing current testVelocidadThread
*/
public static SpeedInfo calculate(final long downloadTime, final long bytesIn) {
SpeedInfo info = new SpeedInfo();
//from mil to sec
System.out.println("Bytes transferidos: " + bytesIn + "Tiempo de descarga: " + downloadTime/1000000000);
double time = downloadTime;
double byteIn1 = bytesIn;
double division = (double)(byteIn1 / time);
double bytespersecond = ((division) * 1000000000);
double kilobits = bytespersecond * BYTE_TO_KILOBIT;
double megabits = kilobits * KILOBIT_TO_MEGABIT;
info.downspeed = bytespersecond;
info.kilobits = kilobits;
info.megabits = megabits;
return info;
}
}
The problem is when I run this on a Android application, I had good results on phones with more processing and memory capacity, but poor results on phones with lower capacity.
Any good ideas to achieve good results on most android's phones?.

try to download the file with java nio rather than java io
java io transfer the file first to memory which make the performance poor on low end devices
while java nio using channels you can transfer the file to storage which will make the performance same on all devices approximately
use this code :
len = out.getChannel().transferFrom(readableByteChannel , seekPos , Long.MAX_VALUE);

Get memory and CPU usage

I would like to get the total physical memory, the CPU usage, and and the amount of memory being used. I have looked into Runtime.freeMemory(), but that isn't the free memory for the whole system.

I know I'm late with my answer, but I think this code is interesting.
This is an adaptation of "closed" code, and should be revised before aplying directly:
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.IOException;
import java.lang.Process;
import java.lang.Runtime;
import java.util.HashMap;
/**
* SystemStatusReader is a collection of methods to read system status (cpu and memory)
*
* #author Andreu Correa Casablanca
*/
public class SystemStatusReader
{
public static final int CONSERVATIVE = 0;
public static final int AVERAGE = 1;
public static final int OPTIMISTIC = 2;
/**
* cpuUsage gives us the percentage of cpu usage
*
* mpstat -P ALL out stream example:
*
* Linux 3.2.0-30-generic (castarco-laptop) 10/09/12 _x86_64_ (2 CPU) - To discard
* - To discard
* 00:16:30 CPU %usr %nice %sys %iowait %irq %soft %steal %guest %idle - To discard
* 00:16:30 all 17,62 0,03 3,55 0,84 0,00 0,03 0,00 0,00 77,93
* 00:16:30 0 17,36 0,05 3,61 0,83 0,00 0,05 0,00 0,00 78,12
* 00:16:30 1 17,88 0,02 3,49 0,86 0,00 0,01 0,00 0,00 77,74
*
* #param measureMode Indicates if we want optimistic, convervative or average measurements.
*/
public static Double cpuUsage (int measureMode) throws Exception {
BufferedReader mpstatReader = null;
String mpstatLine;
String[] mpstatChunkedLine;
Double selected_idle;
try {
Runtime runtime = Runtime.getRuntime();
Process mpstatProcess = runtime.exec("mpstat -P ALL");
mpstatReader = new BufferedReader(new InputStreamReader(mpstatProcess.getInputStream()));
// We discard the three first lines
mpstatReader.readLine();
mpstatReader.readLine();
mpstatReader.readLine();
mpstatLine = mpstatReader.readLine();
if (mpstatLine == null) {
throw new Exception("mpstat didn't work well");
} else if (measureMode == SystemStatusReader.AVERAGE) {
mpstatChunkedLine = mpstatLine.replaceAll(",", ".").split("\\s+");
selected_idle = Double.parseDouble(mpstatChunkedLine[10]);
} else {
selected_idle = (measureMode == SystemStatusReader.CONSERVATIVE)?200.:0.;
Double candidate_idle;
int i = 0;
while((mpstatLine = mpstatReader.readLine()) != null) {
mpstatChunkedLine = mpstatLine.replaceAll(",", ".").split("\\s+");
candidate_idle = Double.parseDouble(mpstatChunkedLine[10]);
if (measureMode == SystemStatusReader.CONSERVATIVE) {
selected_idle = (selected_idle < candidate_idle)?selected_idle:candidate_idle;
} else if (measureMode == SystemStatusReader.OPTIMISTIC) {
selected_idle = (selected_idle > candidate_idle)?selected_idle:candidate_idle;
}
++i;
}
if (i == 0) {
throw new Exception("mpstat didn't work well");
}
}
} catch (Exception e) {
throw e; // It's not desirable to handle the exception here
} finally {
if (mpstatReader != null) try {
mpstatReader.close();
} catch (IOException e) {
// Do nothing
}
}
return 100-selected_idle;
}
/**
* memoryUsage gives us data about memory usage (RAM and SWAP)
*/
public static HashMap<String, Integer> memoryUsage () throws Exception {
BufferedReader freeReader = null;
String freeLine;
String[] freeChunkedLine;
HashMap<String, Integer> usageData = new HashMap<String, Integer>();
try {
Runtime runtime = Runtime.getRuntime();
Process freeProcess = runtime.exec("free -k"); // We measure memory in kilobytes to obtain a greater granularity
freeReader = new BufferedReader(new InputStreamReader(freeProcess.getInputStream()));
// We discard the first line
freeReader.readLine();
freeLine = freeReader.readLine();
if (freeLine == null) {
throw new Exception("free didn't work well");
}
freeChunkedLine = freeLine.split("\\s+");
usageData.put("total", Integer.parseInt(freeChunkedLine[1]));
freeLine = freeReader.readLine();
if (freeLine == null) {
throw new Exception("free didn't work well");
}
freeChunkedLine = freeLine.split("\\s+");
usageData.put("used", Integer.parseInt(freeChunkedLine[2]));
freeLine = freeReader.readLine();
if (freeLine == null) {
throw new Exception("free didn't work well");
}
freeChunkedLine = freeLine.split("\\s+");
usageData.put("swap_total", Integer.parseInt(freeChunkedLine[1]));
usageData.put("swap_used", Integer.parseInt(freeChunkedLine[2]));
} catch (Exception e) {
throw e;
} finally {
if (freeReader != null) try {
freeReader.close();
} catch (IOException e) {
// Do nothing
}
}
return usageData;
}
}

You can use SIGAR (http://support.hyperic.com/display/SIGAR/Home). I believe this is cross platform (I've only tried it on Windows) and I know it works (because I've tried it).
Javadoc: http://www.hyperic.com/support/docs/sigar/
Binaries: http://support.hyperic.com/display/SIGAR/Home#Home-binaries

Memory and CPU.
CPU example:
static final ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
...
long start = threadBean.getCurrentThreadCpuTime();
for (int i = 0; i < 10000000; i++) {
...
}
long finish = threadBean.getCurrentThreadCpuTime();

On Linux you could open /proc/meminfo as a text file and parse the result.

Use JMX via 'jconsole', if this if for interactive use. It displays nice realtime graphs, and lots of other diagnostic information.