We Keep Coding

Unexpected state occurred for AsyncDataSetIterator: runnable died or no data available

I'm trying to build a neural network using deep learning4j framework, I get the following error :
java.lang.IllegalStateException: Unexpected state occurred for AsyncDataSetIterator: runnable died or no data available" this exception
Here is my code
package com.neuralnetwork;
import com.sliit.preprocessing.NormalizeDataset;
import com.sliit.ruleengine.RuleEngine;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.datavec.api.records.reader.SequenceRecordReader;
import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader;
import org.deeplearning4j.eval.Evaluation;
import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.Updater;
import org.deeplearning4j.nn.conf.layers.GravesLSTM;
import org.deeplearning4j.nn.conf.layers.RnnOutputLayer;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.deeplearning4j.util.ModelSerializer;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.api.DataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction;
import weka.core.Instances;
import weka.core.converters.CSVLoader;
import weka.core.converters.CSVSaver;
import weka.filters.Filter;
import weka.filters.supervised.instance.StratifiedRemoveFolds;
import java.io.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
/**
* Deep Belif Neural Network to detect frauds.
*/
public class FraudDetectorDeepBeilefNN {
private static final Log log = LogFactory.getLog(FraudDetectorDeepBeilefNN.class);
public int outputNum = 4;
private int iterations = 5;
private int seed = 1234;
private MultiLayerNetwork model = null;
public int HIDDEN_LAYER_COUNT = 8;
public int numHiddenNodes = 21;
public int inputs = 41;
private String uploadDirectory = "D:/Data";
private ArrayList < Map < String, Double >> roc;
public FraudDetectorDeepBeilefNN() {
}
public void buildModel() {
System.out.println("Build model....");
iterations = outputNum + 1;
NeuralNetConfiguration.Builder builder = new NeuralNetConfiguration.Builder();
builder.iterations(iterations);
builder.learningRate(0.001);
// builder.momentum(0.01);
builder.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
builder.seed(seed);
builder.biasInit(1);
builder.regularization(true).l2(1e-5);
builder.updater(Updater.RMSPROP);
builder.weightInit(WeightInit.XAVIER);
NeuralNetConfiguration.ListBuilder list = builder.list();
for (int i = 0; i < HIDDEN_LAYER_COUNT; i++) {
GravesLSTM.Builder hiddenLayerBuilder = new GravesLSTM.Builder();
hiddenLayerBuilder.nIn(i == 0 ? inputs : numHiddenNodes);
hiddenLayerBuilder.nOut(numHiddenNodes);
hiddenLayerBuilder.activation("tanh");
list.layer(i, hiddenLayerBuilder.build());
}
RnnOutputLayer.Builder outputLayer = new RnnOutputLayer.Builder(LossFunction.MCXENT);
outputLayer.activation("softmax");
outputLayer.nIn(numHiddenNodes);
outputLayer.nOut(outputNum);
list.layer(HIDDEN_LAYER_COUNT, outputLayer.build());
list.pretrain(false);
list.backprop(true);
MultiLayerConfiguration configuration = list.build();
model = new MultiLayerNetwork(configuration);
model.init();
//model.setListeners(new ScoreIterationListener(1));
}
public String trainModel(String modelName, String filePath, int outputs, int inputsTot) throws NeuralException {
try {
System.out.println("Neural Network Training start");
loadSaveNN(modelName, false);
if (model == null) {
buildModel();
}
System.out.println("modal" + model);
System.out.println("file path " + filePath);
File fileGeneral = new File(filePath);
CSVLoader loader = new CSVLoader();
loader.setSource(fileGeneral);
Instances instances = loader.getDataSet();
instances.setClassIndex(instances.numAttributes() - 1);
StratifiedRemoveFolds stratified = new StratifiedRemoveFolds();
String[] options = new String[6];
options[0] = "-N";
options[1] = Integer.toString(5);
options[2] = "-F";
options[3] = Integer.toString(1);
options[4] = "-S";
options[5] = Integer.toString(1);
stratified.setOptions(options);
stratified.setInputFormat(instances);
stratified.setInvertSelection(false);
Instances testInstances = Filter.useFilter(instances, stratified);
stratified.setInvertSelection(true);
Instances trainInstances = Filter.useFilter(instances, stratified);
String directory = fileGeneral.getParent();
CSVSaver saver = new CSVSaver();
File trainFile = new File(directory + "/" + "normtrainadded.csv");
File testFile = new File(directory + "/" + "normtestadded.csv");
if (trainFile.exists()) {
trainFile.delete();
}
trainFile.createNewFile();
if (testFile.exists()) {
testFile.delete();
}
testFile.createNewFile();
saver.setFile(trainFile);
saver.setInstances(trainInstances);
saver.writeBatch();
saver = new CSVSaver();
saver.setFile(testFile);
saver.setInstances(testInstances);
saver.writeBatch();
SequenceRecordReader recordReader = new CSVSequenceRecordReader(0, ",");
recordReader.initialize(new org.datavec.api.split.FileSplit(trainFile));
SequenceRecordReader testReader = new CSVSequenceRecordReader(0, ",");
testReader.initialize(new org.datavec.api.split.FileSplit(testFile));
DataSetIterator iterator = new org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator(recordReader, 2, outputs, inputsTot, false);
DataSetIterator testIterator = new org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator(testReader, 2, outputs, inputsTot, false);
roc = new ArrayList < Map < String, Double >> ();
String statMsg = "";
Evaluation evaluation;
for (int i = 0; i < 100; i++) {
if (i % 2 == 0) {
model.fit(iterator);
evaluation = model.evaluate(testIterator);
} else {
model.fit(testIterator);
evaluation = model.evaluate(iterator);
}
Map < String, Double > map = new HashMap < String, Double > ();
Map < Integer, Integer > falsePositives = evaluation.falsePositives();
Map < Integer, Integer > trueNegatives = evaluation.trueNegatives();
Map < Integer, Integer > truePositives = evaluation.truePositives();
Map < Integer, Integer > falseNegatives = evaluation.falseNegatives();
double fpr = falsePositives.get(1) / (falsePositives.get(1) + trueNegatives.get(1));
double tpr = truePositives.get(1) / (truePositives.get(1) + falseNegatives.get(1));
map.put("FPR", fpr);
map.put("TPR", tpr);
roc.add(map);
statMsg = evaluation.stats();
iterator.reset();
testIterator.reset();
}
loadSaveNN(modelName, true);
System.out.println("ROC " + roc);
return statMsg;
} catch (Exception e) {
e.printStackTrace();
System.out.println("Error ocuured while building neural netowrk :" + e.getMessage());
throw new NeuralException(e.getLocalizedMessage(), e);
}
}
public boolean generateModel(String modelName) {
boolean status = false;
try {
loadSaveNN(modelName, true);
status = true;
} catch (Exception e) {
System.out.println("Error occurred:" + e.getLocalizedMessage());
}
return status;
}
private void loadSaveNN(String name, boolean save) {
File directory = new File(uploadDirectory);
File[] allNN = directory.listFiles();
boolean status = false;
try {
if (model == null && save) {
buildModel();
}
if (allNN != null && allNN.length > 0) {
for (File NN: allNN) {
String fnme = FilenameUtils.removeExtension(NN.getName());
if (name.equals(fnme)) {
status = true;
if (save) {
ModelSerializer.writeModel(model, NN, true);
System.out.println("Model Saved With Weights Successfully");
} else {
model = ModelSerializer.restoreMultiLayerNetwork(NN);
}
break;
}
}
}
if (!status && save) {
//File tempFIle = File.createTempFile(name,".zip",directory);
File tempFile = new File(directory.getAbsolutePath() + "/" + name + ".zip");
if (!tempFile.exists()) {
tempFile.createNewFile();
}
ModelSerializer.writeModel(model, tempFile, true);
}
} catch (IOException e) {
System.out.println("Error occurred:" + e.getMessage());
}
}
public String testModel(String modelName, String[] rawData, Map < Integer, String > map, int inputs, int outputs, String ruleModelSavePath) throws Exception {
String status = "";
String fpath = uploadDirectory;
FileWriter fwriter = new FileWriter(uploadDirectory + "original/insertdata.csv", true);
fwriter.write("\n");
fwriter.write(rawData[0]);
fwriter.close();
if (model == null) {
loadSaveNN(modelName, false);
}
NormalizeDataset norm = new NormalizeDataset(uploadDirectory + "original/insertdata.csv");
norm.updateStringValues(map);
norm.whiteningData();
norm.normalizeDataset();
BufferedReader bufferedReader = new BufferedReader(new FileReader(new File(uploadDirectory + "originalnorminsertdata.csv")));
String output = "";
String prevOutput = "";
while ((output = bufferedReader.readLine()) != null) {
prevOutput = output;
}
bufferedReader.close();
File readFile = new File(uploadDirectory + "normtest.csv");
if (readFile.exists()) {
readFile.delete();
}
readFile.createNewFile();
PrintWriter writer = new PrintWriter(readFile);
writer.println(prevOutput);
writer.flush();
writer.close();
SequenceRecordReader recordReader = new CSVSequenceRecordReader(0, ",");
recordReader.initialize(new org.datavec.api.split.FileSplit(new File(uploadDirectory + "normtest.csv")));
DataSetIterator iterator = new org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator(recordReader, 2, outputs, inputs, false);
INDArray outputArr = null;
while (iterator.hasNext()) {
DataSet ds = iterator.next();
INDArray provided = ds.getFeatureMatrix();
outputArr = model.rnnTimeStep(provided);
}
//INDArray finalOutput = Nd4j.argMax(outputArr,1);
double result = Double.parseDouble(Nd4j.argMax(outputArr, 1).toString());
if (result == 0.0) {
status = "Normal Transaction";
} else {
status = "Fraud Transaction, ";
bufferedReader = new BufferedReader(new FileReader(new File(uploadDirectory + "original/insertdata.csv")));
String heading = "";
heading = bufferedReader.readLine();
bufferedReader.close();
File ruleFile = new File(uploadDirectory + "normrules.csv");
if (ruleFile.exists()) {
ruleFile.delete();
}
ruleFile.createNewFile();
PrintWriter writeNew = new PrintWriter(ruleFile);
writeNew.println(heading);
writeNew.println(rawData[0]);
writeNew.flush();
writeNew.close();
RuleEngine engine = new RuleEngine(fpath + "original/insertdata.csv");
engine.geneateModel(ruleModelSavePath, false);
String finalStatus = status + "Attack Type:" + engine.predictionResult(uploadDirectory + "normrules.csv");
status = finalStatus;
}
return status;
}
public void setUploadDirectory(String uploadDirectory) {
this.uploadDirectory = uploadDirectory;
}
public static void main(String[] args) {
FraudDetectorDeepBeilefNN neural_network = new FraudDetectorDeepBeilefNN();
System.out.println("start=======================");
try {
neural_network.inputs = 4;
neural_network.numHiddenNodes = 3;
neural_network.HIDDEN_LAYER_COUNT = 2;
neural_network.outputNum = 2;
neural_network.buildModel();
String output = neural_network.trainModel("nn", "D:/Data/a.csv", 2, 4);
System.out.println(output);
} catch (Exception e) {
e.printStackTrace();
}
}
public ArrayList < Map < String, Double >> getRoc() {
return roc;
}
}
Here is the 1st few lines of my dataset :
length,caplen,hlen,version,output
60,60,6,0,normal
243,243,8,0,normal
60,60,6,0,neptune
60,60,6,0,neptune

Unable to print console output in same format onto text document

I have my java file set up to calculate a phone bill and print out to the console in this format.
Invoice
--------------------------
Account Amount Due
10011 $12.25
10033 $11.70
--------------------------
Total $23.95
but when I run the program I only get this on the text file
Account Amount_Due
10011 $12.25
10033 $11.7
Can someone help me edit my filecreating code in correct format?
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.StringTokenizer;
import java.util.Vector;
public class PhoneBill {
Vector data;
Vector processed = new Vector();
Vector markProcessed = new Vector();
public void readFile(String inFileStr)
{
String str = "";
data = new Vector<LineItem>();
FileReader fReader;
InputStream inFileStream;
try{
fReader = new FileReader(inFileStr);
BufferedReader br=new BufferedReader(fReader);
String line;
while ((line=br.readLine())!= null){
if (line.indexOf("_") != -1)
continue;
else
if (!line.isEmpty()){
data.add(new LineItem(line.trim()));
}
}
br.close();
}
catch (Exception e){
}
}
public void processCharges()
{
String out = "Account Amount_Due\n";
System.out.println ("Invoice");
System.out.println ("--------------------------");
System.out.println ("Account " + "Amount Due ");
double total = 0.0;
double lCharges =0;
boolean done = false;
DecimalFormat numFormatter = new DecimalFormat("$##.##");
for (int j = 0; j < data.size(); j++ ){
LineItem li = (LineItem)data.get(j);
String accNum = li.getAccountNum();
if (j > 0){
done = checkProcessed(accNum);}
else
processed.add(accNum);
if (!done){
lCharges = 0;
for (int i = 0; i < data.size(); i++){
String acc = ((LineItem)data.get(i)).getAccountNum();
if (acc.equals(accNum) && !done)
lCharges += processItemCharges(accNum);
done = checkProcessed(accNum);
}
lCharges+=10.0;
System.out.format ("%s" + " $%.2f%n",accNum, lCharges);
out += accNum+" ";
out += numFormatter.format(lCharges)+"\n";
processed.add(accNum);
total += lCharges;
}
}
System.out.println ("--------------------------");
System.out.format ("%s" + " $%.2f%n","Total", total);
writeToFile("invoice.txt", out);
}
private void writeToFile(String filename,String outStr)
{
try{
File file = new File(filename);
// if file doesnt exists, then create it
if (!file.exists()) {
file.createNewFile();
}
FileWriter fw = new FileWriter(file.getAbsoluteFile());
BufferedWriter bw = new BufferedWriter(fw);
bw.write(outStr);
bw.close();
} catch (IOException ioe){
System.out.println(ioe.getMessage());
}
}
private boolean checkProcessed(String accNum){
if (processed.contains(accNum))
return true;
else
return false;
}
private double processItemCharges(String accNum)
{
double charges = 0.0;
for (int i = 0; i < data.size(); i++)
{
if(((LineItem)data.get(i)).getAccountNum().equals(accNum))
charges += ((LineItem)data.get(i)).getCharges();
}
return charges;
}
public static void main(String[] args)
{
PhoneBill pB = new PhoneBill();
pB.readFile("input_data.txt");
pB.processCharges();
}
class LineItem{
String accNum ;
String timeOfCall;
double mins;
double amountDue;
boolean counted = false;
public LineItem(String accStr)
{
processAccount(accStr);
}
private void processAccount(String accStr){
StringTokenizer st = new StringTokenizer(accStr);
accNum = (String)st.nextElement();
timeOfCall = (String) st.nextElement();
mins = Double.parseDouble((String) st.nextElement());
if (timeOfCall.compareTo("08:00")>0 && timeOfCall.compareTo("22:00")<0)
amountDue = mins*0.10;
else
amountDue = mins*0.05;
}
public String getAccountNum()
{
return accNum;
}
public double getCharges()
{
return amountDue;
}
}
}

Study this. It uses a bit more advanced Java but understanding it will be well worth your while.
package test;
import java.io.*;
import java.util.*;
public class PhoneBill {
private static String BILL_FORMAT = "%-10s $%,6.2f\n";
private static boolean DEBUG=true;
Map<String, List<LineItem>> accounts = new HashMap<String,List<LineItem>>();
public void readFile(String inFileStr) {
FileReader fReader=null;
try {
fReader = new FileReader(inFileStr);
BufferedReader br = new BufferedReader(fReader);
String line;
while ((line = br.readLine()) != null) {
if (line.indexOf("_") != -1)
continue;
else if (!line.isEmpty()) {
LineItem li = new LineItem(line.trim());
List<LineItem> list = accounts.get(li.accNum);
if(list==null){
list = new ArrayList<LineItem>();
accounts.put(li.accNum, list);
}
list.add(li);
}
}
br.close();
} catch (Exception e) {
/* Don't just swallow Exceptions. */
e.printStackTrace();
} finally {
if(fReader!=null){
try{
fReader.close();
} catch(Exception e){
e.printStackTrace();
}
}
}
}
public void processCharges() {
StringBuffer out = new StringBuffer(100)
.append("Invoice\n")
.append("--------------------------\n")
.append("Account Amount Due \n");
double total = 0.0;
double lCharges = 0;
for (String accNum:accounts.keySet()) {
List<LineItem> account = accounts.get(accNum);
lCharges = 10;
for(LineItem li:account){
lCharges += li.getCharges();
}
total += lCharges;
out.append(String.format(BILL_FORMAT, accNum, lCharges));
}
out.append("--------------------------\n");
out.append(String.format(BILL_FORMAT, "Total", total));
writeToFile("invoice.txt", out.toString());
}
private void writeToFile(String filename, String outStr) {
if(DEBUG){
System.out.printf("========%swriteToFile:%s=========\n", '=', filename);
System.out.println(outStr);
System.out.printf("========%swriteToFile:%s=========\n", '/', filename);
}
try {
File file = new File(filename);
// If file doesn't exist, then create it.
if (!file.exists()) {
file.createNewFile();
}
FileWriter fw = new FileWriter(file.getAbsoluteFile());
BufferedWriter bw = new BufferedWriter(fw);
bw.write(outStr);
bw.close();
} catch (IOException ioe) {
System.out.println(ioe.getMessage());
}
}
public static void main(String[] args) {
PhoneBill pB = new PhoneBill();
pB.readFile("input_data.txt");
pB.processCharges();
}
static class LineItem {
String accNum;
double timeOfCall;
double mins;
double amountDue;
boolean counted = false;
private static final double EIGHT_AM = convertTime("08:00");
private static final double TEN_PM = convertTime("22:00");
public LineItem(String accStr) {
processAccount(accStr);
}
private void processAccount(String accStr) {
StringTokenizer st = new StringTokenizer(accStr);
accNum = st.nextToken();
timeOfCall = convertTime(st.nextToken());
mins = Double.parseDouble(st.nextToken());
if (timeOfCall > EIGHT_AM && timeOfCall < TEN_PM)
amountDue = mins * 0.10;
else
amountDue = mins * 0.05;
}
public String getAccountNum() {
return accNum;
}
public double getCharges() {
return amountDue;
}
private static double convertTime(String in){
/* Will blow up if `in` isn't given in h:m. */
String[] h_m = in.split(":");
return Double.parseDouble(h_m[0])*60+Double.parseDouble(h_m[1]);
}
}
}

Printing to the console (i.e. System.out.println) is not the same as concatenating to your out variable (i.e. out+=).
So when you call
writeToFile("invoice.txt", out);
You are only writing what is in the string 'out' to the file. If you look back at your code, you'll see that all of your missing lines are only ever printed to the console, but not concatenated to the 'out' variable. Correct that and you shouldn't have a problem.

Download file from URL to FOLDER

java.io.BufferedInputStream in = new java.io.BufferedInputStream(new
java.net.URL(args[1].toString()).openStream());
java.io.FileOutputStream fos = new java.io.FileOutputStream("WorldEdit/schematics/"+args[2].toString());
java.io.BufferedOutputStream bout = new BufferedOutputStream(fos,1024);
byte data[] = new byte[1024];
while(in.read(data,0,1024)>=0)
{
bout.write(data);
}
bout.close();
in.close();
}
I want this to download a file from a url, and put it into a folder. This program will be in a folder named "plugins" where I want to put the downloaded file is in "plugins/WorldEdit/schematic".
Doesn't seem to work. Any suggestions?

in.read returns a number indicating how many bytes were actually read. You are ignoring that number and assuming 1024 bytes are read on each call.
If you are using Java 7, you can save a URL to a file this way:
try (InputStream in = new URL(args[1].toString()).openStream()) {
Files.copy(in, Paths.get("WorldEdit", "schematics", args[2].toString()));
}
In Java 6 (and 5 and 4), you can use channels:
FileChannel fileChannel = fos.getChannel();
ReadableByteChannel urlChannel = Channels.newChannel(
new URL(args[1].toString()).openStream());
fileChannel.transferFrom(urlChannel, 0, Long.MAX_VALUE);

You can refer the following program and edit according to your needs.
In this program url is fetching from a table in an html file using Jsoup to parse the html file and downloading into different folder..path of the folder I took from the table(table has 3 column filename,path,download link) .
Go to downloadFile(String urlString) method,there you can find the answer for your question
import java.io.File;
import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
/**
*
* #author nudanesh
*/
public class URLDownload {
private Document doc;
String url = "", folder, file;
private final File sourceFile;
int i = 1;
int r = 1, c = 1;
int anchorCol = 3;
Library lib;
URLDownload() {
lib = new Library();
sourceFile = new File("Download.html");
try {
doc = Jsoup.parse(sourceFile, "UTF-8");
} catch (IOException ex) {
Logger.getLogger(URLDownload.class.getName()).log(Level.SEVERE, null, ex);
}
//Elements links = doc.select("a[href]");
Elements rows = doc.select("tr");
System.out.println("Size=" + rows.size());
for (Element row : rows) {
Elements cols = row.getElementsByTag("td");
c = 1;
for (Element col : cols) {
System.out.println("Row"+r);
if (c == 1) {
file = col.text();//System.out.println("File in main"+file);
} else if (c == 2) {
folder = col.text();//System.out.println("Folder in main"+folder);
} else {
try {
url = col.getElementsByTag("a").attr("href");
} catch (Exception e) {
System.out.print("-");
}
}
c++;
}
if (!url.equals("")) {
lib.setLocation(file,folder);
lib.downloadFile(url);
}
url = "";
i++;
r++;
}
}
public static void main(String arg[]) {
new URLDownload();
}
}
and following is the Library class file
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.nio.file.Files;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
/**
*
* #author nudanesh
*/
public class Library {
boolean downloaded = false;
Thread t;
int waitTime = 0;
String baseLoc = "";
int size = 1024, ByteWritten = 0;
URL url;
URLConnection uCon = null;
String folderLoc = "", file = "firstFile.csv";
File loc;
private OutputStream outStream;
private InputStream is=null;
private byte[] buf;
private int ByteRead;
private int FolderInUrl = 4;
private boolean rootFolder = true;
private File resultFile;
private FileOutputStream fileResult;
private XSSFWorkbook workbookResult;
private XSSFSheet sheetResult;
private int updateExcelRowNum = -1;
private int updateExcelColNum = -1;
String date;
private int waitLimit = 900000;
Library() {
/*System.out.print(Calendar.getInstance().toString());
Date d=new Date();
String date=d.toString();
System.out.println(date);*/
//t = new Thread(this);
// t.start();
date = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss").format(Calendar.getInstance().getTime());
System.out.print(date);
baseLoc = date + "/";
WriteDataToExcel();
baseLoc += "Business Reports/";
createRowExcel(updateExcelRowNum);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Report Name");
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Path");
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Status");
updateExcel();
}
public void setLocation(String a, String b) {
file = a;
file += ".csv";
folderLoc = baseLoc + getFolderPath(b);
// System.out.println("File Name: "+file);
// System.out.println("Folder loc: "+folderLoc);
}
public String getFolderPath(String b) {
String path = "";
try {
System.out.println("path" + b);
path = b;
// path = java.net.URLDecoder.decode(b, "UTF-8");
String p[] = path.split("/");
path = "";
for (int i = FolderInUrl; i < p.length - 1; i++) {
rootFolder = false;
p[i] = removeSpacesAtEnd(p[i]);
path = path + p[i] + "/";
}
} catch (Exception ex) {
Logger.getLogger(Library.class.getName()).log(Level.SEVERE, null, ex);
}
return path;
}
public void downloadFile(String urlString) {
// System.out.println("Started");
try {
url = new URL(urlString);
} catch (MalformedURLException ex) {
Logger.getLogger(Library.class.getName()).log(Level.SEVERE, null, ex);
}
try {
loc = new File(folderLoc);
if (!loc.exists()) {
loc.mkdirs();
}
outStream = new BufferedOutputStream(new FileOutputStream(folderLoc + file));
uCon = url.openConnection();
uCon.setReadTimeout(waitLimit);
is = uCon.getInputStream();
downloaded=true;
buf = new byte[size];
while ((ByteRead = is.read(buf)) != -1) {
System.out.println("while executing" + ByteRead);
outStream.write(buf, 0, ByteRead);
ByteWritten += ByteRead;
}
//System.out.println("Downloaded" + ByteWritten);
resetCounters();
createRowExcel(updateExcelRowNum);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, file);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, folderLoc);
if (ByteWritten < 1000) {
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Downloaded ");
} else {
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Downloaded ");
}
updateExcel();
} catch (Exception e) {
System.out.println("error catch" + e);
resetCounters();
createRowExcel(updateExcelRowNum);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, file);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, folderLoc);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Rejected the Download after waiting " + (waitLimit / 60000) + " minutes");
updateExcel();
waitTime = 0;
} finally {
try {
System.out.println("Error in streams");
if(downloaded)
is.close();
outStream.close();
downloaded= false;
} catch (IOException e) {
e.printStackTrace();
}
}
}
public void moveToFolder(String reportName, String path) {
try {
File repo = new File(folderLoc + "/" + reportName + ".csv");
path = folderLoc + "/" + path;
File pathFolder = new File(path);
if (!pathFolder.exists()) {
pathFolder.mkdirs();
}
pathFolder = new File(path + reportName + ".csv");
System.out.println("Path=" + pathFolder.getAbsolutePath() + "\nReport path=" + repo.getAbsolutePath());
System.out.println("Source" + repo.getAbsolutePath());
//System.out.println("Status" + repo.renameTo(new File(pathFolder.getAbsolutePath())));
System.out.println("Status" + Files.move(repo.toPath(), new File(pathFolder.getAbsolutePath()).toPath(), REPLACE_EXISTING));
//Files.
} catch (Exception e) {
System.out.println("error while moving" + e);
}
}
public String changeSpecialCharacters(String report) {
report = report.replaceAll(":", "_");
return report;
}
public String removeSpacesAtEnd(String inputPath) {
for (int i = inputPath.length() - 1; i >= 0; i--) {
if (inputPath.charAt(i) != ' ') {
break;
} else {
System.out.println("Before string is" + inputPath);
inputPath = inputPath.substring(0, i);
System.out.println("AFter string is" + inputPath);
}
}
return inputPath;
}
public void WriteDataToExcel() {
try {
// file = new FileInputStream(new File("config.xlsx"));
// File resultFolder = new File("Results");
// if (resultFolder.exists()) {
// deleteDirectory(resultFolder);
// }
// resultFolder.mkdirs();
if (!new File(baseLoc).exists()) {
new File(baseLoc).mkdirs();
}
resultFile = new File(baseLoc + "Reports info " + date + ".xlsx");
System.out.println("Path" + resultFile.getAbsolutePath());
resultFile.createNewFile();
// rFilePath = resultFile.getAbsolutePath();
fileResult = new FileOutputStream(resultFile);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
//Get the workbook instance for XLS file
// System.out.println("file success");
XSSFWorkbook workbook = null;
try {
workbookResult = new XSSFWorkbook();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("Opening the browser");
//Get first sheet from the workbook
sheetResult = workbookResult.createSheet();
//sheetResult.set
//Get iterator to all the rows in current sheet
//Get iterator to all cells of current row
//ar.add(folderLocation);
// ar.add(firefoxProfileLocation);
}
public void updateExcel() {
try {
//fileResult.close();
fileResult = new FileOutputStream(resultFile);
workbookResult.write(fileResult);
fileResult.close();
} catch (Exception e) {
System.out.println(e);
}
}
public void createRowExcel(int num) {
updateExcelRowNum++;
num = updateExcelRowNum;
sheetResult.createRow(num);
}
public void updateRowColExcel(int rnum, int cnum, String value) {
updateExcelColNum++;
cnum = updateExcelColNum;
sheetResult.getRow(rnum).createCell(cnum);
XSSFCell cell = sheetResult.getRow(rnum).getCell(cnum);
cell.setCellValue(value);
}
public void updateColumn(int rnum, int cnum, String value) {
XSSFCell cell = sheetResult.getRow(rnum).getCell(cnum);
cell.setCellValue(value);
}
public void resetCounters() {
updateExcelColNum = -1;
}
/* #Override
public void run() {
while (true) {
if (true) {
waitTime += 1000;
System.out.println(waitTime);
if (waitTime > waitLimit) {
try {
is.close();
outStream.close();
//downloaded=false;
// cancelDownload=true;
} catch (Exception ex) {
Logger.getLogger(Library.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
try {
Thread.sleep(1000);
} catch (Exception e) {
}
}
}*/
}

program to show a progress bar while transferring all files from one server to another in java

i have written a java code to transfer files from one server to another using the concept of socket programming. now in the same code i also want to check for the network connection availability before each file is transferred. i also want that the files transferred should be transferred according to their numerical sequence. And while the files are being transferred the transfer progress of each file i.e the progress percentage bar should also be visible on the screen.
i will really appreciate if someone can help me with the code. i am posting the code i have written for file transfer.
ClientMain.java
import java.io.*;
import java.net.Socket;
import java.net.SocketException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
public class ClientMain {
private DirectoryTxr transmitter = null;
Socket clientSocket = null;
private boolean connectedStatus = false;
private String ipAddress;
String srcPath = null;
String dstPath = "";
public ClientMain() {
}
public void setIpAddress(String ip) {
this.ipAddress = ip;
}
public void setSrcPath(String path) {
this.srcPath = path;
}
public void setDstPath(String path) {
this.dstPath = path;
}
private void createConnection() {
Runnable connectRunnable = new Runnable() {
public void run() {
while (!connectedStatus) {
try {
clientSocket = new Socket(ipAddress, 3339);
connectedStatus = true;
transmitter = new DirectoryTxr(clientSocket, srcPath, dstPath);
} catch (IOException io) {
io.printStackTrace();
}
}
}
};
Thread connectionThread = new Thread(connectRunnable);
connectionThread.start();
}
public static void main(String[] args) {
ClientMain main = new ClientMain();
main.setIpAddress("10.6.3.92");
main.setSrcPath("E:/temp/movies/");
main.setDstPath("D:/tcp/movies");
main.createConnection();
}
}
(DirectoryTxr.java)
import java.io.*;
import java.net.Socket;
import java.net.SocketException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
public class DirectoryTxr {
Socket clientSocket = null;
String srcDir = null;
String dstDir = null;
byte[] readBuffer = new byte[1024];
private InputStream inStream = null;
private OutputStream outStream = null;
int state = 0;
final int permissionReqState = 1;
final int initialState = 0;
final int dirHeaderSendState = 2;
final int fileHeaderSendState = 3;
final int fileSendState = 4;
final int fileFinishedState = 5;
private boolean isLive = false;
private int numFiles = 0;
private int filePointer = 0;
String request = "May I send?";
String respServer = "Yes,You can";
String dirResponse = "Directory created...Please send files";
String fileHeaderRecvd = "File header received ...Send File";
String fileReceived = "File Received";
String dirFailedResponse = "Failed";
File[] opFileList = null;
public DirectoryTxr(Socket clientSocket, String srcDir, String dstDir) {
try {
this.clientSocket = clientSocket;
inStream = clientSocket.getInputStream();
outStream = clientSocket.getOutputStream();
isLive = true;
this.srcDir = srcDir;
this.dstDir = dstDir;
state = initialState;
readResponse(); //starting read thread
sendMessage(request);
state = permissionReqState;
} catch (IOException io) {
io.printStackTrace();
}
}
private void sendMessage(String message) {
try {
sendBytes(request.getBytes("UTF-8"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
/**
* Thread to read response from server
*/
private void readResponse() {
Runnable readRunnable = new Runnable() {
public void run() {
while (isLive) {
try {
int num = inStream.read(readBuffer);
if (num > 0) {
byte[] tempArray = new byte[num];
System.arraycopy(readBuffer, 0, tempArray, 0, num);
processBytes(tempArray);
}
} catch (SocketException se) {
System.exit(0);
} catch (IOException io) {
io.printStackTrace();
isLive = false;
}
}
}
};
Thread readThread = new Thread(readRunnable);
readThread.start();
}
private void sendDirectoryHeader() {
File file = new File(srcDir);
if (file.isDirectory()) {
try {
String[] childFiles = file.list();
numFiles = childFiles.length;
String dirHeader = "$" + dstDir + "#" + numFiles + "&";
sendBytes(dirHeader.getBytes("UTF-8"));
} catch (UnsupportedEncodingException en) {
en.printStackTrace();
}
} else {
System.out.println(srcDir + " is not a valid directory");
}
}
private void sendFile(String dirName) {
File file = new File(dirName);
if (!file.isDirectory()) {
try {
int len = (int) file.length();
int buffSize = len / 8;
//to avoid the heap limitation
RandomAccessFile raf = new RandomAccessFile(file, "rw");
FileChannel channel = raf.getChannel();
int numRead = 0;
while (numRead >= 0) {
ByteBuffer buf = ByteBuffer.allocate(1024 * 100000);
numRead = channel.read(buf);
if (numRead > 0) {
byte[] array = new byte[numRead];
System.arraycopy(buf.array(), 0, array, 0, numRead);
sendBytes(array);
}
}
System.out.println("Finished");
} catch (IOException io) {
io.printStackTrace();
}
}
}
private void sendHeader(String fileName) {
try {
File file = new File(fileName);
if (file.isDirectory())
return;//avoiding child directories to avoid confusion
//if want we can sent them recursively
//with proper state transitions
String header = "&" + fileName + "#" + file.length() + "*";
sendHeader(header);
sendBytes(header.getBytes("UTF-8"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
private void sendBytes(byte[] dataBytes) {
synchronized (clientSocket) {
if (outStream != null) {
try {
outStream.write(dataBytes);
outStream.flush();
} catch (IOException io) {
io.printStackTrace();
}
}
}
}
private void processBytes(byte[] data) {
try {
String parsedMessage = new String(data, "UTF-8");
System.out.println(parsedMessage);
setResponse(parsedMessage);
} catch (UnsupportedEncodingException u) {
u.printStackTrace();
}
}
private void setResponse(String message) {
if (message.trim().equalsIgnoreCase(respServer) && state == permissionReqState) {
state = dirHeaderSendState;
sendDirectoryHeader();
} else if (message.trim().equalsIgnoreCase(dirResponse) && state == dirHeaderSendState) {
state = fileHeaderSendState;
if (LocateDirectory()) {
createAndSendHeader();
} else {
System.out.println("Vacant or invalid directory");
}
} else if (message.trim().equalsIgnoreCase(fileHeaderRecvd) && state == fileHeaderSendState) {
state = fileSendState;
sendFile(opFileList[filePointer].toString());
state = fileFinishedState;
filePointer++;
} else if (message.trim().equalsIgnoreCase(fileReceived) && state == fileFinishedState) {
if (filePointer < numFiles) {
createAndSendHeader();
}
System.out.println("Successfully sent");
} else if (message.trim().equalsIgnoreCase(dirFailedResponse)) {
System.out.println("Going to exit....Error ");
// System.exit(0);
} else if (message.trim().equalsIgnoreCase("Thanks")) {
System.out.println("All files were copied");
}
}
private void closeSocket() {
try {
clientSocket.close();
} catch (IOException e) {
e.printStackTrace();
}
}
private boolean LocateDirectory() {
boolean status = false;
File file = new File(srcDir);
if (file.isDirectory()) {
opFileList = file.listFiles();
numFiles = opFileList.length;
if (numFiles <= 0) {
System.out.println("No files found");
} else {
status = true;
}
}
return status;
}
private void createAndSendHeader() {
File opFile = opFileList[filePointer];
String header = "&" + opFile.getName() + "#" + opFile.length() + "*";
try {
state = fileHeaderSendState;
sendBytes(header.getBytes("UTF-8"));
} catch (UnsupportedEncodingException e) {
}
}
private void sendListFiles() {
createAndSendHeader();
}
}
(ServerMain.java)
public class ServerMain {
public ServerMain() {
}
public static void main(String[] args) {
DirectoryRcr dirRcr = new DirectoryRcr();
}
}
(DirectoryRcr.java)
import java.io.*;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketException;
public class DirectoryRcr {
String request = "May I send?";
String respServer = "Yes,You can";
String dirResponse = "Directory created...Please send files";
String dirFailedResponse = "Failed";
String fileHeaderRecvd = "File header received ...Send File";
String fileReceived = "File Received";
Socket socket = null;
OutputStream ioStream = null;
InputStream inStream = null;
boolean isLive = false;
int state = 0;
final int initialState = 0;
final int dirHeaderWait = 1;
final int dirWaitState = 2;
final int fileHeaderWaitState = 3;
final int fileContentWaitState = 4;
final int fileReceiveState = 5;
final int fileReceivedState = 6;
final int finalState = 7;
byte[] readBuffer = new byte[1024 * 100000];
long fileSize = 0;
String dir = "";
FileOutputStream foStream = null;
int fileCount = 0;
File dstFile = null;
public DirectoryRcr() {
acceptConnection();
}
private void acceptConnection() {
try {
ServerSocket server = new ServerSocket(3339);
socket = server.accept();
isLive = true;
ioStream = socket.getOutputStream();
inStream = socket.getInputStream();
state = initialState;
startReadThread();
} catch (IOException io) {
io.printStackTrace();
}
}
private void startReadThread() {
Thread readRunnable = new Thread() {
public void run() {
while (isLive) {
try {
int num = inStream.read(readBuffer);
if (num > 0) {
byte[] tempArray = new byte[num];
System.arraycopy(readBuffer, 0, tempArray, 0, num);
processBytes(tempArray);
}
sleep(100);
} catch (SocketException s) {
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException i) {
i.printStackTrace();
}
}
}
};
Thread readThread = new Thread(readRunnable);
readThread.start();
}
private void processBytes(byte[] buff) throws InterruptedException {
if (state == fileReceiveState || state == fileContentWaitState) {
//write to file
if (state == fileContentWaitState)
state = fileReceiveState;
fileSize = fileSize - buff.length;
writeToFile(buff);
if (fileSize == 0) {
state = fileReceivedState;
try {
foStream.close();
} catch (IOException io) {
io.printStackTrace();
}
System.out.println("Received " + dstFile.getName());
sendResponse(fileReceived);
fileCount--;
if (fileCount != 0) {
state = fileHeaderWaitState;
} else {
System.out.println("Finished");
state = finalState;
sendResponse("Thanks");
Thread.sleep(2000);
System.exit(0);
}
System.out.println("Received");
}
} else {
parseToUTF(buff);
}
}
private void parseToUTF(byte[] data) {
try {
String parsedMessage = new String(data, "UTF-8");
System.out.println(parsedMessage);
setResponse(parsedMessage);
} catch (UnsupportedEncodingException u) {
u.printStackTrace();
}
}
private void setResponse(String message) {
if (message.trim().equalsIgnoreCase(request) && state == initialState) {
sendResponse(respServer);
state = dirHeaderWait;
} else if (state == dirHeaderWait) {
if (createDirectory(message)) {
sendResponse(dirResponse);
state = fileHeaderWaitState;
} else {
sendResponse(dirFailedResponse);
System.out.println("Error occurred...Going to exit");
System.exit(0);
}
} else if (state == fileHeaderWaitState) {
createFile(message);
state = fileContentWaitState;
sendResponse(fileHeaderRecvd);
} else if (message.trim().equalsIgnoreCase(dirFailedResponse)) {
System.out.println("Error occurred ....");
System.exit(0);
}
}
private void sendResponse(String resp) {
try {
sendBytes(resp.getBytes("UTF-8"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
private boolean createDirectory(String dirName) {
boolean status = false;
dir = dirName.substring(dirName.indexOf("$") + 1, dirName.indexOf("#"));
fileCount = Integer.parseInt(dirName.substring(dirName.indexOf("#") + 1, dirName.indexOf("&")));
if (new File(dir).mkdir()) {
status = true;
System.out.println("Successfully created directory " + dirName);
} else if (new File(dir).mkdirs()) {
status = true;
System.out.println("Directories were created " + dirName);
} else if (new File(dir).exists()) {
status = true;
System.out.println("Directory exists" + dirName);
} else {
System.out.println("Could not create directory " + dirName);
status = false;
}
return status;
}
private void createFile(String fileName) {
String file = fileName.substring(fileName.indexOf("&") + 1, fileName.indexOf("#"));
String lengthFile = fileName.substring(fileName.indexOf("#") + 1, fileName.indexOf("*"));
fileSize = Integer.parseInt(lengthFile);
dstFile = new File(dir + "/" + file);
try {
foStream = new FileOutputStream(dstFile);
System.out.println("Starting to receive " + dstFile.getName());
} catch (FileNotFoundException fn) {
fn.printStackTrace();
}
}
private void writeToFile(byte[] buff) {
try {
foStream.write(buff);
} catch (IOException io) {
io.printStackTrace();
}
}
private void sendBytes(byte[] dataBytes) {
synchronized (socket) {
if (ioStream != null) {
try {
ioStream.write(dataBytes);
} catch (IOException io) {
io.printStackTrace();
}
}
}
}
}
ClientMain.java and DirectoryTxr.java are the two classes under client application. ServerMain.java and DirectoryRcr.java are the two classes under Server application.
run the ClientMain.java and ServerMain.java

You can sort an array using Arrays.sort(...)
ie
String[] childFiles = file.list();
Arrays.sort(childFiles);
As I understand it, this will sort the files in natural order, based on the file name.
You can modify this by passing a custom Comparator...
Arrays.sort(childFiles, new Comparator<File>() {...}); // Fill out with your requirements...
Progress monitoring will come down to your requirements. Do you want to see only the overall progress of the number of files, the individual files, a combination of both?
What I've done in the past is pre-iterated the file list, calculated the total number of bytes to be copied and used a ProgressMonitor to display the overall bytes been copied...
Otherwise you will need to supply your own dialog. In either case, you will need use SwingUtilities.invokeLater to update them correctly.
Check out Concurrency in Swing for more details.
Network connectivity is subjective. You could simply send a special "message" to there server and wait for a response, but this only suggests that the message was sent and a recipt was received. It could just as easily fail when you start transfering the file again...

Consecutive jobs in hadoop not running

I have a set of consecutive jobs written for hadoop. Only the first method runs and it ends with a successful run. The second method (job) doesn't start at all. Here is my code:
Just the distanceCalculator method runs, and it doesn't proceed further.
import java.io.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.omg.CosNaming.IstringHelper;
import weka.classifiers.functions.LibSVM;
import weka.core.Attribute;
import weka.core.FastVector;
import weka.core.Instance;
import weka.core.Instances;
import weka.filters.unsupervised.attribute.Add;
import com.wipro.decision.SVMDecision;
import com.wipro.instance.InstanceData;
//include path of test file
public class UniqueEntityClustering {
static Configuration conf = null;
static Instances inpInst = null;
static FastVector fvWekaAttributes = null;
static String pathOfTestFile = "/user/root/TestData.txt";
static String pathOfRefFile = "";
static String pathOfClustFile = "";
static String job3RefFile = "";
static int iterations = 0;
/* Mapper
* The mapper that will read the input file and generate the distance file
*/
public static class ClustererMapper
extends Mapper<Object, Text, Text, Text>{
private FileSystem fs;
private ArrayList<String> curClustArr = new ArrayList<String>();
//static int iterations;
// static String pathOfRefFile = "";
public void setup(Context context)
throws IOException, InterruptedException {
FileSystem fs = FileSystem.get(new Configuration());
String fileName = "";
if(iterations == 0)
fileName = "/user/root/AllClusterDistances/part-r-00000";
else
fileName =
"/user/root/AllClusterDistances"+iterations+"/part-r-00000";
Path svmpath = new Path(fileName);
InputStream ins = fs.open(svmpath);
BufferedReader reader = new BufferedReader(new
InputStreamReader(ins));
String line=null;
String[] clustPairIDs = null;
String[] temp = null;
if((line=reader.readLine())!=null)
clustPairIDs = line.split("=");
temp = clustPairIDs[0].split("/");
curClustArr.add(0, temp[0]);
curClustArr.add(1, temp[1]);
}
public void map(Object key, Text value, Context context) throws
IOException, InterruptedException {
String text = value.toString();
String[] values = null;
int flag = 0;
if(iterations < 1)
values = text.split("\\t");
else
values = text.split("=");
if(!curClustArr.contains(values[0]))
context.write(new Text(values[0]),new Text("0"));
else
{
if(flag==0)
{
flag=1;
String wKey = curClustArr.get(0) + "+" +
curClustArr.get(1);
context.write(new Text(wKey), value);
}
}
}
}
public static class ClustererReducer
extends Reducer<Text, Text, Text, Text> {
public void reduce(Text key, Iterable<Text> values,
Context context
) throws IOException, InterruptedException {
for (Text val : values) {
context.write(key, val);
}
}
}
public static class DistMapper
extends Mapper<Object, Text, DoubleWritable, Text>{
private HashMap<String,Double> RefMap = new HashMap<String,Double>();
public void setup(Context context) throws IOException, InterruptedException
{
String line = null;
String[] temp = null;
FileSystem fs = FileSystem.get(new Configuration());
Path svmpath = new Path(job3RefFile);
InputStream ins = fs.open(svmpath);
BufferedReader reader = new BufferedReader(new
InputStreamReader(ins));
while((line=reader.readLine())!=null)
{
temp = line.split("=");
RefMap.put(temp[0], Double.valueOf(temp[1]));
}
}
public void map(Object key, Text value, Context context) throws
IOException, InterruptedException {
String text = value.toString();
String[] values = text.split("=");
String curMapEle = values[0];
String[] curMapEleArr = curMapEle.split("+");
String line = null;
String curFileEle = null;
String[] curFileEleArr = null;
String temp = null;
Double min = null;
Double dist = null;
FileSystem fs = FileSystem.get(new Configuration());
Path svmpath = new Path(pathOfClustFile);
InputStream ins = fs.open(svmpath);
BufferedReader reader = new BufferedReader(new
InputStreamReader(ins));
while((line=reader.readLine())!=null)
{
curFileEle = line.split("=")[0];
curFileEleArr = curFileEle.split("+");
if(!curFileEle.equals(curMapEle))
{
for(int i=0; i<curMapEleArr.length; i++)
for(int j=0; j<curFileEleArr.length; j++)
{
temp =
curMapEleArr[i]+"/"+curFileEleArr[j];
dist = RefMap.get(temp);
if(i==0 && j==0)
min=dist;
else
{
if(dist<min)
min=dist;
}
}
temp = curMapEle + "/" + curFileEle;
context.write(new DoubleWritable(min), new
Text(temp));
}
}
}
}
public static class DistReducer
extends Reducer<DoubleWritable, Text, Text, DoubleWritable> {
public void reduce(Text key, Iterable<Text> values, Context context) throws
IOException, InterruptedException {
for (Text val : values) {
context.write(val, new
DoubleWritable(Double.valueOf(key.toString())));
}
}
}
public static class DistanceMapper
extends Mapper<Object, Text, DoubleWritable, Text>{
private FileSystem fs;
static LibSVM svm = null;
public void setup(Context context)
{
try {
fs = FileSystem.get(context.getConfiguration());
} catch (IOException e) {
e.printStackTrace();
}
Attribute Attribute1 = new Attribute("nameDist");
Attribute Attribute2 = new Attribute("locDist");
Attribute Attribute3 = new Attribute("workDist");
FastVector fvNominalVal = new FastVector(2);
fvNominalVal.addElement("0");
fvNominalVal.addElement("1");
Attribute ClassAttribute = new Attribute("class", fvNominalVal);
fvWekaAttributes = new FastVector(4);
fvWekaAttributes.addElement(Attribute1);
fvWekaAttributes.addElement(Attribute2);
fvWekaAttributes.addElement(Attribute3);
fvWekaAttributes.addElement(ClassAttribute);
inpInst = new Instances("Rel", fvWekaAttributes, 1);
inpInst.setClassIndex(3);
FileSystem fs;
ObjectInputStream ob = null;
try {
fs = FileSystem.get(new Configuration());
Path svmpath = new Path("/user/root/j48.model");
InputStream ins = fs.open(svmpath);
ob = new ObjectInputStream(ins);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// try {
try {
svm = (LibSVM)ob.readObject();
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
/* } catch (ClassNotFoundException e) {
e.printStackTrace();
}*/
}
public void map(Object key, Text value, Context context
) throws IOException, InterruptedException {
String text = value.toString();
String[] values = text.split("\t");
int mclusterID = Integer.valueOf(values[0]);
String mfirstName = values[1];
String mmiddleName = values[2];
String mlastName = values[3];
String mlocation = values[4];
String[] mworkArray = new String[values.length - 5];
for(int i=5; i<values.length; i++) {
mworkArray[i-5] = values[i];
}
InstanceData mapInstance = new InstanceData(mfirstName,
mmiddleName, mlastName, mlocation, mworkArray);
FileSystem fs = FileSystem.get(new Configuration());
Path path = new Path(pathOfTestFile); //fstatus.getPath();
FSDataInputStream fis = fs.open(path);
BufferedReader reader = new BufferedReader(new
InputStreamReader(fis));
String line = null;
while ((line = reader.readLine()) != null) {
String[] ele = line.split("\t");
int fclusterID = Integer.valueOf(ele[0]);
if(fclusterID == mclusterID)
continue;
String ffirstName = ele[1];
String fmiddleName = ele[2];
String flastName = ele[3];
String flocation = ele[4];
String[] fworkArray = new String[ele.length - 5];
for(int i=5; i<ele.length; i++) {
fworkArray[i-5] = ele[i];
}
InstanceData fileInstance = new InstanceData(ffirstName,
fmiddleName, flastName, flocation, fworkArray);
double nameDist =
NameDistance.getDistanceBetweenNames(mapInstance, fileInstance);
double locationDist =
LocationDistance.getDistanceBetweenLocations(mapInstance, fileInstance);
double workDistance =
WorkDistance.getDistanceBetweenWorkArr(mapInstance, fileInstance);
try {
fs = FileSystem.get(context.getConfiguration());
} catch (IOException e) {
e.printStackTrace();
}
Attribute Attribute1 = new Attribute("nameDist");
Attribute Attribute2 = new Attribute("locDist");
Attribute Attribute3 = new Attribute("workDist");
FastVector fvNominalVal = new FastVector(2);
fvNominalVal.addElement("0");
fvNominalVal.addElement("1");
Attribute ClassAttribute = new Attribute("class", fvNominalVal);
fvWekaAttributes = new FastVector(4);
fvWekaAttributes.addElement(Attribute1);
fvWekaAttributes.addElement(Attribute2);
fvWekaAttributes.addElement(Attribute3);
fvWekaAttributes.addElement(ClassAttribute);
inpInst = new Instances("Rel", fvWekaAttributes, 1);
inpInst.setClassIndex(3);
Instance inst = new Instance(4);
inst.setValue((Attribute)fvWekaAttributes.elementAt(0),
nameDist);
inst.setValue((Attribute)fvWekaAttributes.elementAt(1),
locationDist);
inst.setValue((Attribute)fvWekaAttributes.elementAt(2),
workDistance);
inst.setValue((Attribute)fvWekaAttributes.elementAt(3),
"0");
inpInst.add(inst);
//try {
try {
svm.classifyInstance(inpInst.instance(0));
} catch (Exception e) {
e.printStackTrace();
}
double prob = SVMDecision.decision_value;
DoubleWritable outputKey = new DoubleWritable();
outputKey.set(prob);
Text outputValue = new
Text(String.valueOf(mclusterID)+"/"+String.valueOf(fclusterID));
context.write(outputKey, outputValue);
/* } catch (Exception
e) {
e.printStackTrace();
}*/
}
//}
}
}
public static class DistanceReducer
extends Reducer<DoubleWritable, Text, Text, DoubleWritable> {
public void reduce(DoubleWritable key, Iterable<Text> values,
Context context
) throws IOException, InterruptedException {
for (Text val : values) {
context.write(val, key);
}
}
}
public static void distanceCalculator(Path inputDir, Path outputDir) throws
Exception{
conf = new Configuration();
Job job = new Job(conf, "distanceCalculator");
job.setJarByClass(UniqueEntityClustering.class);
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(DistanceMapper.class);
job.setReducerClass(DistanceReducer.class);
job.setMapOutputKeyClass(DoubleWritable.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(DoubleWritable.class);
FileInputFormat.addInputPath(job, inputDir);
FileOutputFormat.setOutputPath(job, outputDir);
System.exit(job.waitForCompletion(true) ? 0: 1);
}
public static void findCluster(Path inputPath, Path outputPath, int ite) throws
IOException, ClassNotFoundException, InterruptedException
{
conf = new Configuration();
Job job = new Job(conf, "Clusterer"+ite);
job.setJarByClass(UniqueEntityClustering.class);
job.setInputFormatClass(TextInputFormat.class);
//ClustererMapper.iterations = ite;
//ClustererMapper.pathOfRefFile = fileName;
job.setMapperClass(ClustererMapper.class);
job.setReducerClass(ClustererReducer.class);
FileInputFormat.addInputPath(job, inputPath);
FileOutputFormat.setOutputPath(job, outputPath);
System.exit(job.waitForCompletion(true) ? 0: 1);
}
public static void distjob(Path inputPath, Path outputPath, int ite) throws
IOException, ClassNotFoundException, InterruptedException
{
conf = new Configuration();
Job job = new Job(conf, "Dist"+ite);
job.setJarByClass(UniqueEntityClustering.class);
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(DistMapper.class);
job.setReducerClass(DistReducer.class);
FileInputFormat.addInputPath(job, inputPath);
FileOutputFormat.setOutputPath(job, outputPath);
System.exit(job.waitForCompletion(true) ? 0: 1);
}
public static void main(String[] args) throws Exception {
conf = new Configuration();
//String[] otherArgs = new GenericOptionsParser(conf,
args).getRemainingArgs();
if (args.length != 2) {
System.err.println("Usage: wordcount <in> <out>");
System.exit(2);
}
boolean isClusteringTerminated = false;
String job2OutDir = null;
String job2InpDir = null;
String job3InpDir = null;
String job3OutDir = null;
String job1OutDir = null;
String job1InpDir = null;
String job3OutDirBase = null;
String job2OutDirBase = "/user/root/clusterer";
job3OutDirBase = "/user/root/AllClusterDistances";
job1InpDir = "/user/root/TestData";
job1OutDir = "/user/root/AllClusterDistances";
//int iterations = 0;
//String pathOfRefFile = "";
//job2InpDir = "/user/root/OutputDist"+String.valueOf(iterations);
//Start the jobs
distanceCalculator(new Path(job1InpDir), new Path(job1OutDir));
//while(isClusteringTerminated == false) {
//parameters for job 2
if(iterations == 0) {
pathOfRefFile = job1OutDir + "/" + "part-r-00000";
job2InpDir = job1InpDir;
}
else {
pathOfRefFile = job3OutDir+ "/" + "part-r-00000";
job2InpDir = job2OutDirBase + String.valueOf(iterations -
1);
}
job2OutDir = job2OutDirBase + String.valueOf(iterations);
findCluster(new Path(job2InpDir), new Path(job2OutDir),
iterations); //, pathOfRefFile);
//parameters for job 3
job3RefFile = job1OutDir + "/" + "part-r-00000";
job3InpDir = job2OutDir;
job3OutDir = job3OutDirBase + String.valueOf(iterations);
distjob(new Path(job3InpDir), new Path(job3OutDir), iterations);
iterations++;
isClusteringTerminated = determineTermination(new
Path(job3OutDir));
//}
}
public static boolean determineTermination(Path dirPath) {
FileSystem fs = null;
try {
fs = FileSystem.get(new Configuration());
} catch (IOException e) {
e.printStackTrace();
}
FileStatus[] fstatuses = null;
try {
fstatuses = fs.listStatus(dirPath);
} catch (IOException e) {
e.printStackTrace();
}
for (FileStatus fstatus : fstatuses) {
Path path = fstatus.getPath();
if (! path.getName().startsWith("part-r")) {
continue;
}
FSDataInputStream fis = null;
try {
fis = fs.open(path);
} catch (IOException e) {
e.printStackTrace();
}
BufferedReader reader = new BufferedReader(new
InputStreamReader(fis));
String line = null;
try {
line = reader.readLine();
} catch (IOException e) {
e.printStackTrace();
}
String[] values = line.split("=");
double val = Double.valueOf(values[1]);
if(val < 0.0)
return true;
}
return false;
}
}