We Keep Coding

Android internet speed test

I am developing a speed test app like OKLA app (http://www.speedtest.net/).
I've been trying to get bandwidth rate with the most common approach:
Get the time before downloading.
Download some file for some time X.
Get the time after downloading and the total size downloaded.
Calculate speed from TIME and BYTES RECEIVED.
Also, I execute this in two different threads at the same time because is required to saturate de connection to achieve good results.
This approach works very well on PC environment with this JAVA code:
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.FutureTask;
public class TestVelocidad {
static long totalBytesReceived = 0; //
static long startSample;
static long endSample ;
private static final long TIME_FOR_DOWNLOAD_MILISECONDS = (long) 10000.0;
private static final long MILI_TO_NANO = 1000000;
public static void main(String[] args) throws InterruptedException, ExecutionException {
try{
final ExecutorService service;
String downloadFileUrl100MB = "http://cachefly.cachefly.net/100mb.test";
startSample = System.nanoTime();
service = Executors.newFixedThreadPool(6);
FutureTask futureTask_1 = new FutureTask(new SpeedTestThread(downloadFileUrl100MB));
service.execute(futureTask_1);
FutureTask futureTask_2 = new FutureTask(new SpeedTestThread(downloadFileUrl100MB));
service.execute(futureTask_2);
service.shutdownNow();
long result1 = (Long) futureTask_1.get();
long result2 = (Long) futureTask_2.get();
endSample = System.nanoTime();
long timeSpent = (long) endSample-startSample;
long totalBytesReceived = result1 + result2;
System.out.println("Time of threads: " + timeSpent/1000000000.0 + " seconds " + "\nbytes received: " + (totalBytesReceived) );
double calculatedSpeed;
// long finalTimeSpent ;
// finalTimeSpent = (long) ((TIME_FOR_DOWNLOAD_MILISECONDS * MILI_TO_NANO - diff));
calculatedSpeed = SpeedInfo.calculate(timeSpent, totalBytesReceived).megabits;
System.out.println("Velocidad calculada: " + calculatedSpeed + " mbps" );
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
}
class SpeedTestThread implements Callable<Long> {
private String url = new String("");
private static final long TIME_FOR_DOWNLOAD_NANOSECONDS = (long) 10000000000.0;
private static final long MILI_TO_NANO = 1000000;
private long bytesThread;
public SpeedTestThread(String urlToDownload){
url = urlToDownload;
}
public void run() {
}
#Override
public Long call() throws Exception {
System.out.println("FileDownload " + " File to download: " + url );
InputStream stream = null;
long startCon = System.nanoTime();
URL urlToDownload = null;
try {
urlToDownload = new URL(url);
} catch (MalformedURLException e) {
e.printStackTrace();
}
URLConnection con = null;
try {
con = urlToDownload.openConnection();
} catch (IOException e) {
e.printStackTrace();
}
con.setUseCaches(false);
//Tiempo de acceso al archivo.
long connectionLatency = (System.nanoTime() - startCon)/MILI_TO_NANO;
System.out.println("Connection latency = " + connectionLatency + "");
con.setConnectTimeout(5000);
try {
stream = con.getInputStream();
} catch (IOException e) {
e.printStackTrace();
}
long startNano = System.nanoTime();
int currentByte = 0;
try {
while ((currentByte = stream.read()) != -1 ) {
bytesThread++;
if ((System.nanoTime() - startNano) > TIME_FOR_DOWNLOAD_NANOSECONDS){
System.out.println("Time");
break;
}
}
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("Thread bytes received: " + bytesThread);
return bytesThread;
}
}
class SpeedInfo {
public double kilobits = 0;
public double megabits = 0;
public double downspeed = 0;
private static final double BYTE_TO_KILOBIT = 0.008;
private static final double KILOBIT_TO_MEGABIT = 0.001;
/**
* 1 byte = 0.0078125 kilobits
* 1 kilobits = 0.0009765625 megabit
*
* #param downloadTime in miliseconds
* #param bytesIn number of bytes downloaded
* #return SpeedInfo containing current testVelocidadThread
*/
public static SpeedInfo calculate(final long downloadTime, final long bytesIn) {
SpeedInfo info = new SpeedInfo();
//from mil to sec
System.out.println("Bytes transferidos: " + bytesIn + "Tiempo de descarga: " + downloadTime/1000000000);
double time = downloadTime;
double byteIn1 = bytesIn;
double division = (double)(byteIn1 / time);
double bytespersecond = ((division) * 1000000000);
double kilobits = bytespersecond * BYTE_TO_KILOBIT;
double megabits = kilobits * KILOBIT_TO_MEGABIT;
info.downspeed = bytespersecond;
info.kilobits = kilobits;
info.megabits = megabits;
return info;
}
}
The problem is when I run this on a Android application, I had good results on phones with more processing and memory capacity, but poor results on phones with lower capacity.
Any good ideas to achieve good results on most android's phones?.

try to download the file with java nio rather than java io
java io transfer the file first to memory which make the performance poor on low end devices
while java nio using channels you can transfer the file to storage which will make the performance same on all devices approximately
use this code :
len = out.getChannel().transferFrom(readableByteChannel , seekPos , Long.MAX_VALUE);

To print one confusion matrix instead of multiple matrices from each mapper

I am trying to print a confusion matrix of weka j48 algorithm and i am getting multiple matrices as output.
This is the class that runs the whole program. It is responsible for getting input from the user, setting up the mapper and reducer, organizing the weka input, etc.
public class WekDoop {
* The main method of this program.
* Precondition: arff file is uploaded into HDFS and the correct
* number of parameters were passed into the JAR file when it was run
*
* #param args
* #throws Exception
*/
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
// Make sure we have the correct number of arguments passed into the program
if (args.length != 4) {
System.err.println("Usage: WekDoop <# of splits> <classifier> <input file> <output file>");
System.exit(1);
}
// configure the job using the command line args
conf.setInt("Run-num.splits", Integer.parseInt(args[0]));
conf.setStrings("Run.classify", args[1]);
conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization," + "org.apache.hadoop.io.serializer.WritableSerialization");
// Configure the jobs main class, mapper and reducer
// TODO: Make the Job name print the name of the currently running classifier
Job job = new Job(conf, "WekDoop");
job.setJarByClass(WekDoop.class);
job.setMapperClass(WekaMap.class);
job.setReducerClass(WekaReducer.class);
// Start with 1
job.setNumReduceTasks(1);
// This section sets the values of the <K2, V2>
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(weka.classifiers.bayes.NaiveBayes.class);
job.setOutputValueClass(AggregateableEvaluation.class);
// Set the input and output directories based on command line args
FileInputFormat.addInputPath(job, new Path(args[2]));
FileOutputFormat.setOutputPath(job, new Path(args[3]));
// Set the input type of the environment
// (In this case we are overriding TextInputFormat)
job.setInputFormatClass(WekaInputFormat.class);
// wait until the job is complete to exit
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
Mapper Class
This class is a mapper for the weka classifiers It is given a chunk of data and it sets up a classifier to run on that data. There is a lot of other handling that occurs in the method as well.
public class WekaMap extends Mapper<Object, Text, Text, AggregateableEvaluation> {
private Instances randData = null;
private Classifier cls = null;
private AggregateableEvaluation eval = null;
private Classifier clsCopy = null;
// Run 10 mappers
private String numMaps = "10";
// TODO: Make sure this is not hard-coded -- preferably a command line arg
// Set the classifier
private String classname = "weka.classifiers.bayes.NaiveBayes";
private int seed = 20;
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
String line = value.toString();
System.out.println("CURRENT LINE: " + line);
//line = "/home/ubuntu/Workspace/hadoop-1.1.0/hadoop-data/spambase_processed.arff";
Configuration conf = new Configuration();
FileSystem fileSystem = FileSystem.get(conf);
Path path = new Path("/home/hduser/very_small_spam.arff");
// Make sure the file exists...
if (!fileSystem.exists(path)) {
System.out.println("File does not exists");
return;
}
JobID test = context.getJobID();
TaskAttemptID tid = context.getTaskAttemptID();
// Set up the weka configuration
Configuration wekaConfig = context.getConfiguration();
numMaps = wekaConfig.get("Run-num.splits");
classname = wekaConfig.get("Run.classify");
String[] splitter = tid.toString().split("_");
String jobNumber = "";
int n = 0;
if (splitter[4].length() > 0) {
jobNumber = splitter[4].substring(splitter[4].length() - 1);
n = Integer.parseInt(jobNumber);
}
FileSystem fs = FileSystem.get(context.getConfiguration());
System.out.println("PATH: " + path);
// Read in the data set
context.setStatus("Reading in the arff file...");
readArff(fs, path.toString());
context.setStatus("Done reading arff! Initializing aggregateable eval...");
try {
eval = new AggregateableEvaluation(randData);
}
catch (Exception e1) {
e1.printStackTrace();
}
// Split the data into two sets: Training set and a testing set
// this will allow us to use a little bit of data to train the classifier
// before running the classifier on the rest of the dataset
Instances trainInstance = randData.trainCV(Integer.parseInt(numMaps), n);
Instances testInstance = randData.testCV(Integer.parseInt(numMaps), n);
// Set parameters to be passed to the classifiers
String[] opts = new String[3];
if (classname.equals("weka.classifiers.lazy.IBk")) {
opts[0] = "";
opts[1] = "-K";
opts[2] = "1";
}
else if (classname.equals("weka.classifiers.trees.J48")) {
opts[0] = "";
opts[1] = "-C";
opts[2] = "0.25";
}
else if (classname.equals("weka.classifiers.bayes.NaiveBayes")) {
opts[0] = "";
opts[1] = "";
opts[2] = "";
}
else {
opts[0] = "";
opts[1] = "";
opts[2] = "";
}
// Start setting up the classifier and its various options
try {
cls = (Classifier) Utils.forName(Classifier.class, classname, opts);
}
catch (Exception e) {
e.printStackTrace();
}
// These are all used for timing different processes
long beforeAbstract = 0;
long beforeBuildClass = 0;
long afterBuildClass = 0;
long beforeEvalClass = 0;
long afterEvalClass = 0;
try {
// Create the classifier and record how long it takes to set up
context.setStatus("Creating the classifier...");
System.out.println(new Timestamp(System.currentTimeMillis()));
beforeAbstract = System.currentTimeMillis();
clsCopy = AbstractClassifier.makeCopy(cls);
beforeBuildClass = System.currentTimeMillis();
System.out.println(new Timestamp(System.currentTimeMillis()));
// Train the classifier on the training set and record how long this takes
context.setStatus("Training the classifier...");
clsCopy.buildClassifier(trainInstance);
afterBuildClass = System.currentTimeMillis();
System.out.println(new Timestamp(System.currentTimeMillis()));
beforeEvalClass = System.currentTimeMillis();
// Run the classifer on the rest of the data set and record its duration as well
context.setStatus("Evaluating the model...");
eval.evaluateModel(clsCopy, testInstance);
afterEvalClass = System.currentTimeMillis();
System.out.println(new Timestamp(System.currentTimeMillis()));
// We are done this iteration!
context.setStatus("Complete");
}
catch (Exception e) {
System.out.println("Debugging strarts here!");
e.printStackTrace();
}
// calculate the total times for each section
long abstractTime = beforeBuildClass - beforeAbstract;
long buildTime = afterBuildClass - beforeBuildClass;
long evalTime = afterEvalClass - beforeEvalClass;
// Print out the times
System.out.println("The value of creation time: " + abstractTime);
System.out.println("The value of Build time: " + buildTime);
System.out.println("The value of Eval time: " + evalTime);
context.write(new Text(line), eval);
}
/**
* This can be used to write out the results on HDFS, but it is not essential
* to the success of this project. If time allows, we can implement it.
*/
public void writeResult() {
}
/**
* This method reads in the arff file that is provided to the program.
* Nothing really special about the way the data is handled.
*
* #param fs
* #param filePath
* #throws IOException
* #throws InterruptedException
*/
public void readArff(FileSystem fs, String filePath) throws IOException, InterruptedException {
BufferedReader reader;
DataInputStream d;
ArffReader arff;
Instance inst;
Instances data;
try {
// Read in the data using a ton of wrappers
d = new DataInputStream(fs.open(new Path(filePath)));
reader = new BufferedReader(new InputStreamReader(d));
arff = new ArffReader(reader, 100000);
data = arff.getStructure();
data.setClassIndex(data.numAttributes() - 1);
// Add each line to the input stream
while ((inst = arff.readInstance(data)) != null) {
data.add(inst);
}
reader.close();
Random rand = new Random(seed);
randData = new Instances(data);
randData.randomize(rand);
// This is how weka handles the sampling of the data
// the stratify method splits up the data to cross validate it
if (randData.classAttribute().isNominal()) {
randData.stratify(Integer.parseInt(numMaps));
}
}
catch (IOException e) {
e.printStackTrace();
}
}
}
Reducer Class
This class is a reducer for the output from the weka classifiers It is given bunch of cross-validated data chunks from the mappers and its job is to aggregate the data into one solution.
public class WekaReducer extends Reducer<Text, AggregateableEvaluation, Text, IntWritable> {
Text result = new Text();
Evaluation evalAll = null;
IntWritable test = new IntWritable();
AggregateableEvaluation aggEval;
/**
* The reducer method takes all the stratified, cross-validated
* values from the mappers in a list and uses an aggregatable evaluation to consolidate
* them.
*/
public void reduce(Text key, Iterable<AggregateableEvaluation> values, Context context) throws IOException, InterruptedException {
int sum = 0;
// record how long it takes to run the aggregation
System.out.println(new Timestamp(System.currentTimeMillis()));
long beforeReduceTime = System.currentTimeMillis();
// loop through each of the values and "aggregate"
// which basically means to consolidate the values
for (AggregateableEvaluation val : values) {
System.out.println("IN THE REDUCER!");
// The first time through, give aggEval a value
if (sum == 0) {
try {
aggEval = val;
}
catch (Exception e) {
e.printStackTrace();
}
}
else {
// combine the values
aggEval.aggregate(val);
}
try {
// This is what is taken from the mapper to be aggregated
System.out.println("This is the map result");
System.out.println(aggEval.toMatrixString());
}
catch (Exception e) {
e.printStackTrace();
}
sum += 1;
}
// Here is where the typical weka matrix output is generated
try {
System.out.println("This is reduce matrix");
System.out.println(aggEval.toMatrixString());
}
catch (Exception e) {
e.printStackTrace();
}
// calculate the duration of the aggregation
context.write(key, new IntWritable(sum));
long afterReduceTime = System.currentTimeMillis();
long reduceTime = afterReduceTime - beforeReduceTime;
// display the output
System.out.println("The value of reduce time is: " + reduceTime);
System.out.println(new Timestamp(System.currentTimeMillis()));
}
}
And lastly InputFormatClass
Takes a JobContext and returns a list of data split into pieces Basically this is a way of handling large data sets. This method allows us to split a large data set into smaller chunks to pass across worker nodes (or in our case, just to make life a little easier and pass the chunks to a single node so that it is not overwhelmed by one large data set)
public class WekaInputFormat extends TextInputFormat {
public List<InputSplit> getSplits(JobContext job) throws IOException {
long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
long maxSize = getMaxSplitSize(job);
List<InputSplit> splits = new ArrayList<InputSplit>();
for (FileStatus file: listStatus(job)) {
Path path = file.getPath();
FileSystem fs = path.getFileSystem(job.getConfiguration());
//number of bytes in this file
long length = file.getLen();
BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length);
// make sure this is actually a valid file
if(length != 0) {
// set the number of splits to make. NOTE: the value can be changed to anything
int count = job.getConfiguration().getInt("Run-num.splits", 1);
for(int t = 0; t < count; t++) {
//split the file and add each chunk to the list
splits.add(new FileSplit(path, 0, length, blkLocations[0].getHosts()));
}
}
else {
// Create empty array for zero length files
splits.add(new FileSplit(path, 0, length, new String[0]));
}
}
return splits;
}
}

Hadoop stdout is always empty and bytes written is zero

I am trying to execute Weka on MapReduce and the stdout is always empty
This is the class that runs the whole program. It is responsible
for getting input from the user, setting up the mapper and reducer,
organizing the weka input, etc.
public class WekDoop {
* The main method of this program.
* Precondition: arff file is uploaded into HDFS and the correct
* number of parameters were passed into the JAR file when it was run
*
* #param args
* #throws Exception
*/
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
// Make sure we have the correct number of arguments passed into the program
if (args.length != 4) {
System.err.println("Usage: WekDoop <# of splits> <classifier> <input file> <output file>");
System.exit(1);
}
// configure the job using the command line args
conf.setInt("Run-num.splits", Integer.parseInt(args[0]));
conf.setStrings("Run.classify", args[1]);
conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization," + "org.apache.hadoop.io.serializer.WritableSerialization");
// Configure the jobs main class, mapper and reducer
// TODO: Make the Job name print the name of the currently running classifier
Job job = new Job(conf, "WekDoop");
job.setJarByClass(WekDoop.class);
job.setMapperClass(WekaMap.class);
job.setReducerClass(WekaReducer.class);
// Start with 1
job.setNumReduceTasks(1);
// This section sets the values of the <K2, V2>
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(weka.classifiers.bayes.NaiveBayes.class);
job.setOutputValueClass(AggregateableEvaluation.class);
// Set the input and output directories based on command line args
FileInputFormat.addInputPath(job, new Path(args[2]));
FileOutputFormat.setOutputPath(job, new Path(args[3]));
// Set the input type of the environment
// (In this case we are overriding TextInputFormat)
job.setInputFormatClass(WekaInputFormat.class);
// wait until the job is complete to exit
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
Mapper Class
This class is a mapper for the weka classifiers
It is given a chunk of data and it sets up a classifier to run on that data.
There is a lot of other handling that occurs in the method as well.
public class WekaMap extends Mapper<Object, Text, Text, AggregateableEvaluation> {
private Instances randData = null;
private Classifier cls = null;
private AggregateableEvaluation eval = null;
private Classifier clsCopy = null;
// Run 10 mappers
private String numMaps = "10";
// TODO: Make sure this is not hard-coded -- preferably a command line arg
// Set the classifier
private String classname = "weka.classifiers.bayes.NaiveBayes";
private int seed = 20;
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
String line = value.toString();
System.out.println("CURRENT LINE: " + line);
//line = "/home/ubuntu/Workspace/hadoop-1.1.0/hadoop-data/spambase_processed.arff";
Configuration conf = new Configuration();
FileSystem fileSystem = FileSystem.get(conf);
Path path = new Path("/home/hduser/very_small_spam.arff");
// Make sure the file exists...
if (!fileSystem.exists(path)) {
System.out.println("File does not exists");
return;
}
JobID test = context.getJobID();
TaskAttemptID tid = context.getTaskAttemptID();
// Set up the weka configuration
Configuration wekaConfig = context.getConfiguration();
numMaps = wekaConfig.get("Run-num.splits");
classname = wekaConfig.get("Run.classify");
String[] splitter = tid.toString().split("_");
String jobNumber = "";
int n = 0;
if (splitter[4].length() > 0) {
jobNumber = splitter[4].substring(splitter[4].length() - 1);
n = Integer.parseInt(jobNumber);
}
FileSystem fs = FileSystem.get(context.getConfiguration());
System.out.println("PATH: " + path);
// Read in the data set
context.setStatus("Reading in the arff file...");
readArff(fs, path.toString());
context.setStatus("Done reading arff! Initializing aggregateable eval...");
try {
eval = new AggregateableEvaluation(randData);
}
catch (Exception e1) {
e1.printStackTrace();
}
// Split the data into two sets: Training set and a testing set
// this will allow us to use a little bit of data to train the classifier
// before running the classifier on the rest of the dataset
Instances trainInstance = randData.trainCV(Integer.parseInt(numMaps), n);
Instances testInstance = randData.testCV(Integer.parseInt(numMaps), n);
// Set parameters to be passed to the classifiers
String[] opts = new String[3];
if (classname.equals("weka.classifiers.lazy.IBk")) {
opts[0] = "";
opts[1] = "-K";
opts[2] = "1";
}
else if (classname.equals("weka.classifiers.trees.J48")) {
opts[0] = "";
opts[1] = "-C";
opts[2] = "0.25";
}
else if (classname.equals("weka.classifiers.bayes.NaiveBayes")) {
opts[0] = "";
opts[1] = "";
opts[2] = "";
}
else {
opts[0] = "";
opts[1] = "";
opts[2] = "";
}
// Start setting up the classifier and its various options
try {
cls = (Classifier) Utils.forName(Classifier.class, classname, opts);
}
catch (Exception e) {
e.printStackTrace();
}
// These are all used for timing different processes
long beforeAbstract = 0;
long beforeBuildClass = 0;
long afterBuildClass = 0;
long beforeEvalClass = 0;
long afterEvalClass = 0;
try {
// Create the classifier and record how long it takes to set up
context.setStatus("Creating the classifier...");
System.out.println(new Timestamp(System.currentTimeMillis()));
beforeAbstract = System.currentTimeMillis();
clsCopy = AbstractClassifier.makeCopy(cls);
beforeBuildClass = System.currentTimeMillis();
System.out.println(new Timestamp(System.currentTimeMillis()));
// Train the classifier on the training set and record how long this takes
context.setStatus("Training the classifier...");
clsCopy.buildClassifier(trainInstance);
afterBuildClass = System.currentTimeMillis();
System.out.println(new Timestamp(System.currentTimeMillis()));
beforeEvalClass = System.currentTimeMillis();
// Run the classifer on the rest of the data set and record its duration as well
context.setStatus("Evaluating the model...");
eval.evaluateModel(clsCopy, testInstance);
afterEvalClass = System.currentTimeMillis();
System.out.println(new Timestamp(System.currentTimeMillis()));
// We are done this iteration!
context.setStatus("Complete");
}
catch (Exception e) {
System.out.println("Debugging strarts here!");
e.printStackTrace();
}
// calculate the total times for each section
long abstractTime = beforeBuildClass - beforeAbstract;
long buildTime = afterBuildClass - beforeBuildClass;
long evalTime = afterEvalClass - beforeEvalClass;
// Print out the times
System.out.println("The value of creation time: " + abstractTime);
System.out.println("The value of Build time: " + buildTime);
System.out.println("The value of Eval time: " + evalTime);
context.write(new Text(line), eval);
}
/**
* This can be used to write out the results on HDFS, but it is not essential
* to the success of this project. If time allows, we can implement it.
*/
public void writeResult() {
}
/**
* This method reads in the arff file that is provided to the program.
* Nothing really special about the way the data is handled.
*
* #param fs
* #param filePath
* #throws IOException
* #throws InterruptedException
*/
public void readArff(FileSystem fs, String filePath) throws IOException, InterruptedException {
BufferedReader reader;
DataInputStream d;
ArffReader arff;
Instance inst;
Instances data;
try {
// Read in the data using a ton of wrappers
d = new DataInputStream(fs.open(new Path(filePath)));
reader = new BufferedReader(new InputStreamReader(d));
arff = new ArffReader(reader, 100000);
data = arff.getStructure();
data.setClassIndex(data.numAttributes() - 1);
// Add each line to the input stream
while ((inst = arff.readInstance(data)) != null) {
data.add(inst);
}
reader.close();
Random rand = new Random(seed);
randData = new Instances(data);
randData.randomize(rand);
// This is how weka handles the sampling of the data
// the stratify method splits up the data to cross validate it
if (randData.classAttribute().isNominal()) {
randData.stratify(Integer.parseInt(numMaps));
}
}
catch (IOException e) {
e.printStackTrace();
}
}
}
Reducer Class
This class is a reducer for the output from the weka classifiers
It is given bunch of cross-validated data chunks from the mappers and its
job is to aggregate the data into one solution.
public class WekaReducer extends Reducer<Text, AggregateableEvaluation, Text, IntWritable> {
Text result = new Text();
Evaluation evalAll = null;
IntWritable test = new IntWritable();
AggregateableEvaluation aggEval;
/**
* The reducer method takes all the stratified, cross-validated
* values from the mappers in a list and uses an aggregatable evaluation to consolidate
* them.
*/
public void reduce(Text key, Iterable<AggregateableEvaluation> values, Context context) throws IOException, InterruptedException {
int sum = 0;
// record how long it takes to run the aggregation
System.out.println(new Timestamp(System.currentTimeMillis()));
long beforeReduceTime = System.currentTimeMillis();
// loop through each of the values and "aggregate"
// which basically means to consolidate the values
for (AggregateableEvaluation val : values) {
System.out.println("IN THE REDUCER!");
// The first time through, give aggEval a value
if (sum == 0) {
try {
aggEval = val;
}
catch (Exception e) {
e.printStackTrace();
}
}
else {
// combine the values
aggEval.aggregate(val);
}
try {
// This is what is taken from the mapper to be aggregated
System.out.println("This is the map result");
System.out.println(aggEval.toMatrixString());
}
catch (Exception e) {
e.printStackTrace();
}
sum += 1;
}
// Here is where the typical weka matrix output is generated
try {
System.out.println("This is reduce matrix");
System.out.println(aggEval.toMatrixString());
}
catch (Exception e) {
e.printStackTrace();
}
// calculate the duration of the aggregation
context.write(key, new IntWritable(sum));
long afterReduceTime = System.currentTimeMillis();
long reduceTime = afterReduceTime - beforeReduceTime;
// display the output
System.out.println("The value of reduce time is: " + reduceTime);
System.out.println(new Timestamp(System.currentTimeMillis()));
}
}
And lastly InputFormatClass
Takes a JobContext and returns a list of data split into pieces
Basically this is a way of handling large data sets. This method allows
us to split a large data set into smaller chunks to pass across worker nodes
(or in our case, just to make life a little easier and pass the chunks to a single
node so that it is not overwhelmed by one large data set)
public class WekaInputFormat extends TextInputFormat {
public List<InputSplit> getSplits(JobContext job) throws IOException {
long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
long maxSize = getMaxSplitSize(job);
List<InputSplit> splits = new ArrayList<InputSplit>();
for (FileStatus file: listStatus(job)) {
Path path = file.getPath();
FileSystem fs = path.getFileSystem(job.getConfiguration());
//number of bytes in this file
long length = file.getLen();
BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length);
// make sure this is actually a valid file
if(length != 0) {
// set the number of splits to make. NOTE: the value can be changed to anything
int count = job.getConfiguration().getInt("Run-num.splits", 1);
for(int t = 0; t < count; t++) {
//split the file and add each chunk to the list
splits.add(new FileSplit(path, 0, length, blkLocations[0].getHosts()));
}
}
else {
// Create empty array for zero length files
splits.add(new FileSplit(path, 0, length, new String[0]));
}
}
return splits;
}
}

For each of the mapper, reducer and overall job, there is an stderr file, an stdout file and a syslog file.
You are printing to stdout in the mapper and the reducer, so you should check the stdout file of the mapper and the reducer not that of the overall job.
Best of luck

Get CPU load per core using Java

I'm using this code to get CPU load from /proc/stat using Java code:
private static long PREV_IDLE; //CPU Idle time
private static long PREV_TOTAL; //CPU Total time
public static float getCPUProcOrig() throws Exception
{
BufferedReader cpuReader = null;
try
{
cpuReader = new BufferedReader(new InputStreamReader(new FileInputStream("/proc/stat")));
String cpuLine = cpuReader.readLine();
if (cpuLine == null)
{
throw new Exception("/proc/stat didn't work well");
}
else
{
String[] CPU = cpuLine.split("\\s+");
long IDLE = Long.parseLong(CPU[4]);//Get the idle CPU time.
long DIFF_IDLE = IDLE - PREV_IDLE;
long DIFF_TOTAL = TOTAL - PREV_TOTAL;
long DIFF_USAGE = DIFF_TOTAL == 0 ? 0 : (1000 * (DIFF_TOTAL - DIFF_IDLE) / DIFF_TOTAL + 5) / 10;
// System.out.println("CPU: " + DIFF_USAGE + "%");
PREV_TOTAL = TOTAL;
PREV_IDLE = IDLE;
return (float) DIFF_USAGE;
}
}
catch (Exception e)
{
throw e; // It's not desirable to handle the exception here
}
finally
{
if (cpuReader != null)
try
{
cpuReader.close();
}
catch (IOException e)
{
// Do nothing
}
}
}
Unfortunately this code works well but for average CPU load. I would like to list all cores load separately. I tried to extend the code:
private static long PREV_IDLE; //CPU Idle time
private static long PREV_TOTAL; //CPU Total time
private static final int CONSERVATIVE = 0;
private static final int AVERAGE = 1;
private static final int OPTIMISTIC = 2;
public HashMap<String, HashMap<String, Float>> getCPUProc() throws Exception
{
BufferedReader cpuReader = null;
HashMap<String, HashMap<String, Float>> usageData = new HashMap<>();
try
{
String line;
cpuReader = new BufferedReader(new InputStreamReader(new FileInputStream("/proc/stat")));
while ((line = cpuReader.readLine()) != null)
{
String[] CPU = line.split("\\s+");
if (CPU[0].startsWith("cpu"))
{
String cpuName = String.valueOf(CPU[0]);//Get the cpu number.
long IDLE = Long.parseLong(CPU[4]);//Get the idle CPU time.
long TOTAL = Long.parseLong(CPU[1]) + Long.parseLong(CPU[2]) + Long.parseLong(CPU[3]) + Long.parseLong(CPU[4]);
// System.out.println("IDLE : " + IDLE);
long DIFF_IDLE = IDLE - PREV_IDLE;
long DIFF_TOTAL = TOTAL - PREV_TOTAL;
long DIFF_USAGE = DIFF_TOTAL == 0 ? 0 : (1000 * (DIFF_TOTAL - DIFF_IDLE) / DIFF_TOTAL + 5) / 10;
// System.out.println("CPU: " + DIFF_USAGE + "%");
PREV_TOTAL = TOTAL;
PREV_IDLE = IDLE;
HashMap<String, Float> usageData2 = new HashMap<>();
usageData2.put("cpu", (float) DIFF_USAGE);
usageData.put(cpuName, usageData2);
}
// return (float) DIFF_USAGE;
}
}
catch (IOException | NumberFormatException e)
{
throw e; // It's not desirable to handle the exception here
}
finally
{
if (cpuReader != null)
try
{
cpuReader.close();
}
catch (IOException e)
{
// Do nothing
}
}
return usageData;
}
As you can see from the first code there are several static variables which are used to calculate the CPU load.
When I try to read all lines from /proc/stat these static variables are not used properly and data between the cores is messed up and the result is not accurate.
Can you help me to read the load properly? I'm out of ideas. How I can fix the code?

the problem is that lines:
long DIFF_IDLE = IDLE - PREV_IDLE;
long DIFF_TOTAL = TOTAL - PREV_TOTAL;
long DIFF_USAGE = DIFF_TOTAL == 0 ? 0 : (1000 * (DIFF_TOTAL - DIFF_IDLE) / DIFF_TOTAL + 5) / 10;
PREV_TOTAL = TOTAL;
PREV_IDLE = IDLE;
as you can see PREV_IDLE and PREV_TOTAL are shared between all cores; probably you want to keep them core-specific, so you should load that values before using them.
a nice idea would be instead of
PREV_TOTAL = TOTAL;
PREV_IDLE = IDLE;
save them into usageData2

How can I get the correct amount of internal and external space of device while factoring in the edge cases of Galaxy devices?

I'd like to get the amount on internal and external space in a device and after going to through a couple of posts on StackOverflow, I found that this is easy. I can get the amount of internal space using this:
StatFs sfsInternal = new StatFs(Environment.getRootDirectory().getAbsolutePath());
return Long.valueOf(sfsInternal.getBlockCount()) * Long.valueOf(sfsInternal.getBlockSize());
...and I can get the amount of external space using this:
StatFs sfsExternal = new StatFs(Environment.getExternalStorageDirectory().getAbsolutePath());
return Long.valueOf(sfsExternal.getBlockCount()) * Long.valueOf(sfsExternal.getBlockSize());
When I read about "internal" storage, I assumed that it would be the non-removable onboard storage on the device and "external" would the removable flash card storage but this hasn't been case entirely.
I found that Samsung devices e.e. Galaxy Note 2, show a large chunk of the internal storage as external. Here's an answer that discusses the same thing. https://stackoverflow.com/a/12087556/304151
How can I get the amount of internal storage (on-board and non-removable) and the amount of external storage (flash and removable) while factoring in the edge cases of Samsung's Galaxy devices. I'm yet to find an answer on StackOverflow that provides a complete working solution for this scenario. My code is for API level 17.
Thanks.

Here is the code to get available free space on different devices i have tested that code on samsung GALAXY Tab7 2.2 Froyo and Nexus 7 4.2.2 Jelly Beans
// calculate frespace on external storage
public static int getExternalStorageFreeSpace(String storagePath)
{
try
{
File file = new File(storagePath);
StatFs stat = new StatFs(file.getPath());
double sdAvailSize = (double) stat.getAvailableBlocks() * (double) stat.getBlockSize();
int valueinmb = (int) (sdAvailSize / 1024) / 1024;
return valueinmb;
}
catch (Exception e)
{
System.out.println("Message//////" + e.getMessage() + "Cause555555555555" + e.getCause());
}
return 0;
}
to diffrentiate between internal and external storages i have used this class and some logic
public class GetRemoveableDevices
{
private final static String TAG = "GetRemoveableDevice";
public GetRemoveableDevices()
{
}
public static String[] getDirectories()
{
Log.d(TAG, "getStorageDirectories");
File tempFile;
String[] directories = null;
String[] splits;
ArrayList<String> arrayList = new ArrayList<String>();
BufferedReader bufferedReader = null;
String lineRead;
try
{
arrayList.clear(); // redundant, but what the hey
bufferedReader = new BufferedReader(new FileReader("/proc/mounts"));
while ((lineRead = bufferedReader.readLine()) != null)
{
Log.d(TAG, "lineRead: " + lineRead);
splits = lineRead.split(" ");
// System external storage
if (splits[1].equals(Environment.getExternalStorageDirectory().getPath()))
{
arrayList.add(splits[1]);
Log.d(TAG, "gesd split 1: " + splits[1]);
continue;
}
// skip if not external storage device
if (!splits[0].contains("/dev/block/"))
{
continue;
}
// skip if mtdblock device
if (splits[0].contains("/dev/block/mtdblock"))
{
continue;
}
// skip if not in /mnt node
if (!splits[1].contains("/mnt"))
{
continue;
}
// skip these names
if (splits[1].contains("/secure"))
{
continue;
}
if (splits[1].contains("/mnt/asec"))
{
continue;
}
// Eliminate if not a directory or fully accessible
tempFile = new File(splits[1]);
if (!tempFile.exists())
{
continue;
}
if (!tempFile.isDirectory())
{
continue;
}
if (!tempFile.canRead())
{
continue;
}
if (!tempFile.canWrite())
{
continue;
}
// Met all the criteria, assume sdcard
arrayList.add(splits[1]);
}
}
catch (FileNotFoundException e)
{
}
catch (IOException e)
{
}
finally
{
if (bufferedReader != null)
{
try
{
bufferedReader.close();
}
catch (IOException e)
{
}
}
}
// Send list back to caller
if (arrayList.size() == 0)
{
arrayList.add("sdcard not found");
}
directories = new String[arrayList.size()];
for (int i = 0; i < arrayList.size(); i++)
{
directories[i] = arrayList.get(i);
}
return directories;
}
}
now i am showing you my logic
String[] dirs = GetRemoveableDevices.getDirectories();
ArrayList<String> directories=new ArrayList<String>();
for(String directory:dirs)
{
if(!directory.contains("."))
directories.add(directory);
}
String externalStorage = "";
String internalStorage = "";
if (directories.size()>= 2)
{
internalStorage = directories.get(0).toString();
externalStorage = directories.get(1).toString();
}
else if (directories.size() < 2)
{
internalStorage = directories.get(0).toString();
externalStorage = null;
}
hope it will be helpful

"Internal storage" is for privately held data. It's called internal because it's relative to the application itself. It's for sandboxing the application's data and keeping it private.
Environment.getRootDirectory() gets the phone's system folder. Which is not internal storage, but external storage.
External storage is for publicly shared data, external to the application.
Since mounting naming conventions vary greatly between phones, it can be difficult to differentiate from an SD card and normal onboard directories. But generally, the sd card is mounted to the directory /sdcard/.

I found some code that does this on this blog post.
package com.sapien.music.importer.util;
import java.io.File;
#SuppressLint("NewApi")
public class StorageOptions {
public static String[] labels;
public static String[] paths;
public static int count = 0;
private static Context sContext;
private static ArrayList<String> sVold = new ArrayList<String>();
public static void determineStorageOptions(Context context) {
sContext = context.getApplicationContext();
readVoldFile();
testAndCleanList();
setProperties();
}
private static void readVoldFile() {
/*
* Scan the /system/etc/vold.fstab file and look for lines like this:
* dev_mount sdcard /mnt/sdcard 1
* /devices/platform/s3c-sdhci.0/mmc_host/mmc0
*
* When one is found, split it into its elements and then pull out the
* path to the that mount point and add it to the arraylist
*
* some devices are missing the vold file entirely so we add a path here
* to make sure the list always includes the path to the first sdcard,
* whether real or emulated.
*/
sVold.add("/mnt/sdcard");
try {
Scanner scanner = new Scanner(new File("/system/etc/vold.fstab"));
while (scanner.hasNext()) {
String line = scanner.nextLine();
if (line.startsWith("dev_mount")) {
String[] lineElements = line.split(" ");
String element = lineElements[2];
if (element.contains(":"))
element = element.substring(0, element.indexOf(":"));
if (element.contains("usb"))
continue;
// don't add the default vold path
// it's already in the list.
if (!sVold.contains(element))
sVold.add(element);
}
}
} catch (Exception e) {
// swallow - don't care
e.printStackTrace();
}
}
private static void testAndCleanList() {
/*
* Now that we have a cleaned list of mount paths, test each one to make
* sure it's a valid and available path. If it is not, remove it from
* the list.
*/
for (int i = 0; i < sVold.size(); i++) {
String voldPath = sVold.get(i);
File path = new File(voldPath);
if (!path.exists() || !path.isDirectory() || !path.canWrite())
sVold.remove(i--);
}
}
private static void setProperties() {
/*
* At this point all the paths in the list should be valid. Build the
* public properties.
*/
ArrayList<String> labelList = new ArrayList<String>();
int j = 0;
if (sVold.size() > 0) {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.GINGERBREAD)
labelList.add("Auto");
else if (Build.VERSION.SDK_INT < Build.VERSION_CODES.HONEYCOMB) {
if (Environment.isExternalStorageRemovable()) {
labelList.add(sContext
.getString(R.string.text_external_storage) + " 1");
j = 1;
} else
labelList.add(sContext
.getString(R.string.text_internal_storage));
} else {
if (!Environment.isExternalStorageRemovable()
|| Environment.isExternalStorageEmulated())
labelList.add(sContext
.getString(R.string.text_internal_storage));
else {
labelList.add(sContext
.getString(R.string.text_external_storage) + " 1");
j = 1;
}
}
if (sVold.size() > 1) {
for (int i = 1; i < sVold.size(); i++) {
labelList.add(sContext
.getString(R.string.text_external_storage)
+ " " + (i + j));
}
}
}
labels = new String[labelList.size()];
labelList.toArray(labels);
paths = new String[sVold.size()];
sVold.toArray(paths);
count = Math.min(labels.length, paths.length);
/*
* don't need these anymore, clear the lists to reduce memory use and to
* prepare them for the next time they're needed.
*/
sVold.clear();
}

try this out:
private boolean is_sdCardSaveToUse(){
/**default disk cache size in bytes*/
final int DEFAULT_DISK_CACHE_SIZE = 1024 * 1024 * 10; //10 MB
/**get sdCard state*/
String sdCardState = Environment.getExternalStorageState();
/**check if the sdCard is mounted*/
/**check if we can write to sdCard*/if (Environment.MEDIA_MOUNTED.equals(sdCardState)) {
if (Environment.MEDIA_MOUNTED_READ_ONLY.equals(sdCardState)) {
Log.d("sdCard", "mounted readOnly");
} else {
Log.d("sdCard", "mounted readWrite");
/**get free usable space in bytes */
long freeUsableSpace = Environment.getExternalStorageDirectory().getUsableSpace();
int temp = Math.round(((float) freeUsableSpace / 1024) / 1024); //convert from bytes to MB.
Log.d("usableSpace= ", Integer.toString(temp) + " MB");
if (freeUsableSpace > DEFAULT_DISK_CACHE_SIZE){
return true;
} else {
Log.d("sdCard","not enough space");
return false;
}
}
} else{
Log.d("sdCard","not mounted");
return false;
}
return false;
}