I have a set of consecutive jobs written for hadoop. Only the first method runs and it ends with a successful run. The second method (job) doesn't start at all. Here is my code:
Just the distanceCalculator method runs, and it doesn't proceed further.
import java.io.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.omg.CosNaming.IstringHelper;
import weka.classifiers.functions.LibSVM;
import weka.core.Attribute;
import weka.core.FastVector;
import weka.core.Instance;
import weka.core.Instances;
import weka.filters.unsupervised.attribute.Add;
import com.wipro.decision.SVMDecision;
import com.wipro.instance.InstanceData;
//include path of test file
public class UniqueEntityClustering {
static Configuration conf = null;
static Instances inpInst = null;
static FastVector fvWekaAttributes = null;
static String pathOfTestFile = "/user/root/TestData.txt";
static String pathOfRefFile = "";
static String pathOfClustFile = "";
static String job3RefFile = "";
static int iterations = 0;
/* Mapper
* The mapper that will read the input file and generate the distance file
*/
public static class ClustererMapper
extends Mapper<Object, Text, Text, Text>{
private FileSystem fs;
private ArrayList<String> curClustArr = new ArrayList<String>();
//static int iterations;
// static String pathOfRefFile = "";
public void setup(Context context)
throws IOException, InterruptedException {
FileSystem fs = FileSystem.get(new Configuration());
String fileName = "";
if(iterations == 0)
fileName = "/user/root/AllClusterDistances/part-r-00000";
else
fileName =
"/user/root/AllClusterDistances"+iterations+"/part-r-00000";
Path svmpath = new Path(fileName);
InputStream ins = fs.open(svmpath);
BufferedReader reader = new BufferedReader(new
InputStreamReader(ins));
String line=null;
String[] clustPairIDs = null;
String[] temp = null;
if((line=reader.readLine())!=null)
clustPairIDs = line.split("=");
temp = clustPairIDs[0].split("/");
curClustArr.add(0, temp[0]);
curClustArr.add(1, temp[1]);
}
public void map(Object key, Text value, Context context) throws
IOException, InterruptedException {
String text = value.toString();
String[] values = null;
int flag = 0;
if(iterations < 1)
values = text.split("\\t");
else
values = text.split("=");
if(!curClustArr.contains(values[0]))
context.write(new Text(values[0]),new Text("0"));
else
{
if(flag==0)
{
flag=1;
String wKey = curClustArr.get(0) + "+" +
curClustArr.get(1);
context.write(new Text(wKey), value);
}
}
}
}
public static class ClustererReducer
extends Reducer<Text, Text, Text, Text> {
public void reduce(Text key, Iterable<Text> values,
Context context
) throws IOException, InterruptedException {
for (Text val : values) {
context.write(key, val);
}
}
}
public static class DistMapper
extends Mapper<Object, Text, DoubleWritable, Text>{
private HashMap<String,Double> RefMap = new HashMap<String,Double>();
public void setup(Context context) throws IOException, InterruptedException
{
String line = null;
String[] temp = null;
FileSystem fs = FileSystem.get(new Configuration());
Path svmpath = new Path(job3RefFile);
InputStream ins = fs.open(svmpath);
BufferedReader reader = new BufferedReader(new
InputStreamReader(ins));
while((line=reader.readLine())!=null)
{
temp = line.split("=");
RefMap.put(temp[0], Double.valueOf(temp[1]));
}
}
public void map(Object key, Text value, Context context) throws
IOException, InterruptedException {
String text = value.toString();
String[] values = text.split("=");
String curMapEle = values[0];
String[] curMapEleArr = curMapEle.split("+");
String line = null;
String curFileEle = null;
String[] curFileEleArr = null;
String temp = null;
Double min = null;
Double dist = null;
FileSystem fs = FileSystem.get(new Configuration());
Path svmpath = new Path(pathOfClustFile);
InputStream ins = fs.open(svmpath);
BufferedReader reader = new BufferedReader(new
InputStreamReader(ins));
while((line=reader.readLine())!=null)
{
curFileEle = line.split("=")[0];
curFileEleArr = curFileEle.split("+");
if(!curFileEle.equals(curMapEle))
{
for(int i=0; i<curMapEleArr.length; i++)
for(int j=0; j<curFileEleArr.length; j++)
{
temp =
curMapEleArr[i]+"/"+curFileEleArr[j];
dist = RefMap.get(temp);
if(i==0 && j==0)
min=dist;
else
{
if(dist<min)
min=dist;
}
}
temp = curMapEle + "/" + curFileEle;
context.write(new DoubleWritable(min), new
Text(temp));
}
}
}
}
public static class DistReducer
extends Reducer<DoubleWritable, Text, Text, DoubleWritable> {
public void reduce(Text key, Iterable<Text> values, Context context) throws
IOException, InterruptedException {
for (Text val : values) {
context.write(val, new
DoubleWritable(Double.valueOf(key.toString())));
}
}
}
public static class DistanceMapper
extends Mapper<Object, Text, DoubleWritable, Text>{
private FileSystem fs;
static LibSVM svm = null;
public void setup(Context context)
{
try {
fs = FileSystem.get(context.getConfiguration());
} catch (IOException e) {
e.printStackTrace();
}
Attribute Attribute1 = new Attribute("nameDist");
Attribute Attribute2 = new Attribute("locDist");
Attribute Attribute3 = new Attribute("workDist");
FastVector fvNominalVal = new FastVector(2);
fvNominalVal.addElement("0");
fvNominalVal.addElement("1");
Attribute ClassAttribute = new Attribute("class", fvNominalVal);
fvWekaAttributes = new FastVector(4);
fvWekaAttributes.addElement(Attribute1);
fvWekaAttributes.addElement(Attribute2);
fvWekaAttributes.addElement(Attribute3);
fvWekaAttributes.addElement(ClassAttribute);
inpInst = new Instances("Rel", fvWekaAttributes, 1);
inpInst.setClassIndex(3);
FileSystem fs;
ObjectInputStream ob = null;
try {
fs = FileSystem.get(new Configuration());
Path svmpath = new Path("/user/root/j48.model");
InputStream ins = fs.open(svmpath);
ob = new ObjectInputStream(ins);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// try {
try {
svm = (LibSVM)ob.readObject();
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
/* } catch (ClassNotFoundException e) {
e.printStackTrace();
}*/
}
public void map(Object key, Text value, Context context
) throws IOException, InterruptedException {
String text = value.toString();
String[] values = text.split("\t");
int mclusterID = Integer.valueOf(values[0]);
String mfirstName = values[1];
String mmiddleName = values[2];
String mlastName = values[3];
String mlocation = values[4];
String[] mworkArray = new String[values.length - 5];
for(int i=5; i<values.length; i++) {
mworkArray[i-5] = values[i];
}
InstanceData mapInstance = new InstanceData(mfirstName,
mmiddleName, mlastName, mlocation, mworkArray);
FileSystem fs = FileSystem.get(new Configuration());
Path path = new Path(pathOfTestFile); //fstatus.getPath();
FSDataInputStream fis = fs.open(path);
BufferedReader reader = new BufferedReader(new
InputStreamReader(fis));
String line = null;
while ((line = reader.readLine()) != null) {
String[] ele = line.split("\t");
int fclusterID = Integer.valueOf(ele[0]);
if(fclusterID == mclusterID)
continue;
String ffirstName = ele[1];
String fmiddleName = ele[2];
String flastName = ele[3];
String flocation = ele[4];
String[] fworkArray = new String[ele.length - 5];
for(int i=5; i<ele.length; i++) {
fworkArray[i-5] = ele[i];
}
InstanceData fileInstance = new InstanceData(ffirstName,
fmiddleName, flastName, flocation, fworkArray);
double nameDist =
NameDistance.getDistanceBetweenNames(mapInstance, fileInstance);
double locationDist =
LocationDistance.getDistanceBetweenLocations(mapInstance, fileInstance);
double workDistance =
WorkDistance.getDistanceBetweenWorkArr(mapInstance, fileInstance);
try {
fs = FileSystem.get(context.getConfiguration());
} catch (IOException e) {
e.printStackTrace();
}
Attribute Attribute1 = new Attribute("nameDist");
Attribute Attribute2 = new Attribute("locDist");
Attribute Attribute3 = new Attribute("workDist");
FastVector fvNominalVal = new FastVector(2);
fvNominalVal.addElement("0");
fvNominalVal.addElement("1");
Attribute ClassAttribute = new Attribute("class", fvNominalVal);
fvWekaAttributes = new FastVector(4);
fvWekaAttributes.addElement(Attribute1);
fvWekaAttributes.addElement(Attribute2);
fvWekaAttributes.addElement(Attribute3);
fvWekaAttributes.addElement(ClassAttribute);
inpInst = new Instances("Rel", fvWekaAttributes, 1);
inpInst.setClassIndex(3);
Instance inst = new Instance(4);
inst.setValue((Attribute)fvWekaAttributes.elementAt(0),
nameDist);
inst.setValue((Attribute)fvWekaAttributes.elementAt(1),
locationDist);
inst.setValue((Attribute)fvWekaAttributes.elementAt(2),
workDistance);
inst.setValue((Attribute)fvWekaAttributes.elementAt(3),
"0");
inpInst.add(inst);
//try {
try {
svm.classifyInstance(inpInst.instance(0));
} catch (Exception e) {
e.printStackTrace();
}
double prob = SVMDecision.decision_value;
DoubleWritable outputKey = new DoubleWritable();
outputKey.set(prob);
Text outputValue = new
Text(String.valueOf(mclusterID)+"/"+String.valueOf(fclusterID));
context.write(outputKey, outputValue);
/* } catch (Exception
e) {
e.printStackTrace();
}*/
}
//}
}
}
public static class DistanceReducer
extends Reducer<DoubleWritable, Text, Text, DoubleWritable> {
public void reduce(DoubleWritable key, Iterable<Text> values,
Context context
) throws IOException, InterruptedException {
for (Text val : values) {
context.write(val, key);
}
}
}
public static void distanceCalculator(Path inputDir, Path outputDir) throws
Exception{
conf = new Configuration();
Job job = new Job(conf, "distanceCalculator");
job.setJarByClass(UniqueEntityClustering.class);
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(DistanceMapper.class);
job.setReducerClass(DistanceReducer.class);
job.setMapOutputKeyClass(DoubleWritable.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(DoubleWritable.class);
FileInputFormat.addInputPath(job, inputDir);
FileOutputFormat.setOutputPath(job, outputDir);
System.exit(job.waitForCompletion(true) ? 0: 1);
}
public static void findCluster(Path inputPath, Path outputPath, int ite) throws
IOException, ClassNotFoundException, InterruptedException
{
conf = new Configuration();
Job job = new Job(conf, "Clusterer"+ite);
job.setJarByClass(UniqueEntityClustering.class);
job.setInputFormatClass(TextInputFormat.class);
//ClustererMapper.iterations = ite;
//ClustererMapper.pathOfRefFile = fileName;
job.setMapperClass(ClustererMapper.class);
job.setReducerClass(ClustererReducer.class);
FileInputFormat.addInputPath(job, inputPath);
FileOutputFormat.setOutputPath(job, outputPath);
System.exit(job.waitForCompletion(true) ? 0: 1);
}
public static void distjob(Path inputPath, Path outputPath, int ite) throws
IOException, ClassNotFoundException, InterruptedException
{
conf = new Configuration();
Job job = new Job(conf, "Dist"+ite);
job.setJarByClass(UniqueEntityClustering.class);
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(DistMapper.class);
job.setReducerClass(DistReducer.class);
FileInputFormat.addInputPath(job, inputPath);
FileOutputFormat.setOutputPath(job, outputPath);
System.exit(job.waitForCompletion(true) ? 0: 1);
}
public static void main(String[] args) throws Exception {
conf = new Configuration();
//String[] otherArgs = new GenericOptionsParser(conf,
args).getRemainingArgs();
if (args.length != 2) {
System.err.println("Usage: wordcount <in> <out>");
System.exit(2);
}
boolean isClusteringTerminated = false;
String job2OutDir = null;
String job2InpDir = null;
String job3InpDir = null;
String job3OutDir = null;
String job1OutDir = null;
String job1InpDir = null;
String job3OutDirBase = null;
String job2OutDirBase = "/user/root/clusterer";
job3OutDirBase = "/user/root/AllClusterDistances";
job1InpDir = "/user/root/TestData";
job1OutDir = "/user/root/AllClusterDistances";
//int iterations = 0;
//String pathOfRefFile = "";
//job2InpDir = "/user/root/OutputDist"+String.valueOf(iterations);
//Start the jobs
distanceCalculator(new Path(job1InpDir), new Path(job1OutDir));
//while(isClusteringTerminated == false) {
//parameters for job 2
if(iterations == 0) {
pathOfRefFile = job1OutDir + "/" + "part-r-00000";
job2InpDir = job1InpDir;
}
else {
pathOfRefFile = job3OutDir+ "/" + "part-r-00000";
job2InpDir = job2OutDirBase + String.valueOf(iterations -
1);
}
job2OutDir = job2OutDirBase + String.valueOf(iterations);
findCluster(new Path(job2InpDir), new Path(job2OutDir),
iterations); //, pathOfRefFile);
//parameters for job 3
job3RefFile = job1OutDir + "/" + "part-r-00000";
job3InpDir = job2OutDir;
job3OutDir = job3OutDirBase + String.valueOf(iterations);
distjob(new Path(job3InpDir), new Path(job3OutDir), iterations);
iterations++;
isClusteringTerminated = determineTermination(new
Path(job3OutDir));
//}
}
public static boolean determineTermination(Path dirPath) {
FileSystem fs = null;
try {
fs = FileSystem.get(new Configuration());
} catch (IOException e) {
e.printStackTrace();
}
FileStatus[] fstatuses = null;
try {
fstatuses = fs.listStatus(dirPath);
} catch (IOException e) {
e.printStackTrace();
}
for (FileStatus fstatus : fstatuses) {
Path path = fstatus.getPath();
if (! path.getName().startsWith("part-r")) {
continue;
}
FSDataInputStream fis = null;
try {
fis = fs.open(path);
} catch (IOException e) {
e.printStackTrace();
}
BufferedReader reader = new BufferedReader(new
InputStreamReader(fis));
String line = null;
try {
line = reader.readLine();
} catch (IOException e) {
e.printStackTrace();
}
String[] values = line.split("=");
double val = Double.valueOf(values[1]);
if(val < 0.0)
return true;
}
return false;
}
}
Related
the below codes are working fine(individually) i just want to pass letter[i] value from FindDrive class to Ziputils input file location such that i can zip pendrive data automatically.
FindDrive Class
package com.prosper;
import java.io.File;
public class FindDrive
{
/**
* Application Entry Point
*/
public static void main(String[] args)
{
String[] letters = new String[]{ "A", "B", "C", "D", "E", "F", "G", "H", "I"};
File[] drives = new File[letters.length];
boolean[] isDrive = new boolean[letters.length];
// init the file objects and the initial drive state
for ( int i = 0; i < letters.length; ++i )
{
drives[i] = new File(letters[i]+":/");
isDrive[i] = drives[i].canRead();
}
System.out.println("FindDrive: waiting for devices...");
// loop indefinitely
while(true)
{
// check each drive
for ( int i = 0; i < letters.length; ++i )
{
boolean pluggedIn = drives[i].canRead();
// if the state has changed output a message
if ( pluggedIn != isDrive[i] )
{
if ( pluggedIn ){
System.out.println("Drive "+letters[i]+" has been plugged in");
}
else
System.out.println("Drive "+letters[i]+" has been unplugged");
isDrive[i] = pluggedIn;
}
}
// wait before looping
try { Thread.sleep(100); }
catch (InterruptedException e) { /* do nothing */ }
}
}
}
The ZipUtils Class
package com.prosper;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
public class ZipUtils {
private List <String> fileList;
private static final String OUTPUT_ZIP_FILE = "E:\\appu\\Folder.zip";
private static final String SOURCE_FOLDER = "E:\\appu\\"; //SourceFolder
public ZipUtils() {
fileList = new ArrayList < String > ();
}
public static void main(String[] args) {
ZipUtils appZip = new ZipUtils();
appZip.generateFileList(new File(SOURCE_FOLDER));
appZip.zipIt(OUTPUT_ZIP_FILE);
}
public void zipIt(String zipFile) {
byte[] buffer = new byte[1024];
String source = new File(SOURCE_FOLDER).getName();
FileOutputStream fos = null;
ZipOutputStream zos = null;
try {
fos = new FileOutputStream(zipFile);
zos = new ZipOutputStream(fos);
System.out.println("Output to Zip : " + zipFile);
FileInputStream in = null;
for (String file: this.fileList) {
System.out.println("File Added : " + file);
ZipEntry ze = new ZipEntry(source + File.separator + file);
zos.putNextEntry(ze);
try {
in = new FileInputStream(SOURCE_FOLDER + File.separator + file);
int len;
while ((len = in .read(buffer)) > 0) {
zos.write(buffer, 0, len);
}
} finally {
in.close();
}
}
zos.closeEntry();
System.out.println("Folder successfully compressed");
} catch (IOException ex) {
ex.printStackTrace();
} finally {
try {
zos.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
public void generateFileList(File node) {
// add file only
if (node.isFile()) {
fileList.add(generateZipEntry(node.toString()));
}
if (node.isDirectory()) {
String[] subNote = node.list();
for (String filename: subNote) {
generateFileList(new File(node, filename));
}
}
}
private String generateZipEntry(String file) {
return file.substring(SOURCE_FOLDER.length() + 1, file.length());
}
}
To pass data from one class to another, you can do the following:
1) Create an object of the class in your FindDrive class:
ZipUtils utils = new ZipUtils();
2) Pass the value to the method of that class:
utils.zipIt(letter[i])
I have inserted a few statements, which create a new object of ZipUtils class and call the method, however I do suggest you looking into Dependency Injection as well, if you want to improve your code quality here
package com.prosper;
import java.io.File;
public class FindDrive {
/**
* Application Entry Point
*/
public static void main(String[] args) {
String[] letters = new String[] {"A", "B", "C", "D", "E", "F", "G",
"H", "I"};
ZipUtils utils;
File[] drives = new File[letters.length];
boolean[] isDrive = new boolean[letters.length];
// init the file objects and the initial drive state
for (int i = 0; i < letters.length; ++i) {
drives[i] = new File(letters[i] + ":/");
isDrive[i] = drives[i].canRead();
}
System.out.println("FindDrive: waiting for devices...");
// loop indefinitely
while (true) {
// check each drive
for (int i = 0; i < letters.length; ++i) {
boolean pluggedIn = drives[i].canRead();
utils = new ZipUtils();
utils.changeDirectory(letters[i]);
// if the state has changed output a message
if (pluggedIn != isDrive[i]) {
if (pluggedIn) {
System.out.println("Drive " + letters[i] + " has been plugged in");
} else {
System.out.println("Drive " + letters[i] + " has been unplugged");
}
isDrive[i] = pluggedIn;
}
}
// wait before looping
try {
Thread.sleep(100);
} catch (InterruptedException e) { /* do nothing */ }
}
}
}
To make changes to SOURCE_FOLDER after declaration, you need to make sure its not a constant i.e it is not final.
Add a method in ZipUtils:
private static String source_folder = "E:\\appu\\"; //SourceFolder
public void changeDirectory(String zip) {
source_folder = zip + ":\\";
}
So I'm doing this app for a coursework project and I am getting this error when I try to communicate with server:
java.io.StreamCorruptedException: invalid stream header: 43686F6F
at java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:806)
at java.io.ObjectInputStream.<init>(ObjectInputStream.java:299)
at Client.ChatClient.<init>(ChatClient.java:31)
at Client.ChatClient.main(ChatClient.java:102)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144)
Here is my Server Code:
public class ChatServer implements Observer {
private ServerSocket serverSocket = null;
private ArrayList<ClientHandler> clients = null;
private Executor service = Executors.newFixedThreadPool(10);
public ChatServer( int port ) {
try {
serverSocket = new ServerSocket( port );
}
catch (IOException e) {
e.printStackTrace();
}
clients = new ArrayList<ClientHandler>();
Thread t = new Thread( new ServerLoop() );
t.start();
}
public void tellEveryone( String message ) {
Iterator it = clients.iterator();
while ( it.hasNext() ) {
System.out.println( 1 );
ClientHandler client = (ClientHandler) it.next();
client.send(message);
}
System.out.println( message );
}
#Override
public void update(Observable o, Object arg) {
System.out.println("from server - updateMethod : " + arg );
}
public class ServerLoop implements Runnable {
public void run() {
try {
while (true) {
Socket clientSocket = serverSocket.accept();
ClientHandler c = new ClientHandler(clientSocket);
clients.add(c);
c.addObserver(ChatServer.this);
service.execute(c);
tellEveryone("new client");
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
public static void main( String[] args ) {
ChatServer s = new ChatServer( 5000 );
}
}
This is my Client Handler (main part):
public class ClientHandler extends Observable implements Runnable {
private Scanner reader = null;
private PrintWriter writer = null;
private String path = "../";
private Socket client;
private BufferedOutputStream bufferedOutputStream;
private boolean isConnected = false;
private ObjectOutputStream outputStream = null;
private String sourceDirectory = "/home/csunix/sc16hsm/IdeaProjects/ServerClient/src/Server/folder1";
private String destinationDirectory = "/home/csunix/sc16hsm/IdeaProjects/ServerClient/src/Client/Downloads/";
private int fileCount = 0;
private FileEvent fileEvent = null;
public ClientHandler( Socket client ) {
try {
this.client=client;
reader = new Scanner( client.getInputStream() );
writer = new PrintWriter( client.getOutputStream(), true );
bufferedOutputStream = new BufferedOutputStream(client.getOutputStream());
}
catch (IOException e) {
e.printStackTrace();
}
}
public void locateFiles() {
File srcDir = new File(sourceDirectory);
if (!srcDir.isDirectory()) {
System.out.println("Source directory is not valid ..Exiting the client");
System.exit(0);
}
File[] files = srcDir.listFiles();
fileCount = files.length;
if (fileCount == 0) {
System.out.println("Empty directory ..Exiting the client");
System.exit(0);
}
for (int i = 0; i < fileCount; i++) {
System.out.println("Sending " + files[i].getAbsolutePath());
sendFile(files[i].getAbsolutePath(), fileCount - i - 1);
System.out.println(files[i].getAbsolutePath());
}
}
public void sendFile(String fileName, int index) {
fileEvent = new FileEvent();
fileEvent.setDestinationDirectory(destinationDirectory);
fileEvent.setSourceDirectory(sourceDirectory);
File file = new File(fileName);
fileEvent.setFilename(file.getName());
fileEvent.setRemainder(index);
DataInputStream diStream = null;
try {
diStream = new DataInputStream(new FileInputStream(file));
long len = (int) file.length();
byte[] fileBytes = new byte[(int) len];
int read = 0; int numRead = 0;
while (read < fileBytes.length && (numRead = diStream.read(fileBytes, read, fileBytes.length - read)) >= 0) {
read = read + numRead;
}
fileEvent.setFileData(fileBytes);
fileEvent.setStatus("Success");
} catch (Exception e) {
e.printStackTrace();
fileEvent.setStatus("Error");
}
try {
outputStream.writeObject(fileEvent);
} catch (IOException e) {
e.printStackTrace();
}
}
public void listFolders(){
File file = new File("../");
String[] directories = file.list(new FilenameFilter() {
#Override
public boolean accept(File current, String name) {
return new File(current, name).isDirectory();
}
});
send(Arrays.toString(directories));
}
public void listAll(String fileName){
File folder = new File(path + "/" + fileName);
path = folder.getPath();
System.out.println(path);
File[] listOfFiles = folder.listFiles();
for (int i = 0; i < listOfFiles.length; i++) {
if (listOfFiles[i].isFile()) {
writer.println("File: " + listOfFiles[i].getName());
} else if (listOfFiles[i].isDirectory()) {
writer.println("Directory: " + listOfFiles[i].getName());
}
}
}
public void send( String message ) {
writer.println( message );
}
public void run() {
String message;
writer.println("Choose a file");
listAll("");
while ((message = reader.nextLine()) != null) {
writer.println("Choose a file");
listAll(message);
setChanged();
notifyObservers( message );
if(message.equals("download")) {
System.out.println("MESSAGE FROM CLIENT: " + message);
writer.println("ready");
locateFiles();
}
}
}
}
This is my Client code:
public class ChatClient {
private Scanner socketIn = null;
private PrintWriter socketOut = null;
private Scanner keyboardIn = null;
private Socket socket = null;
private ObjectInputStream inputStream = null;
private FileEvent fileEvent;
private File dstFile = null;
private FileOutputStream fileOutputStream = null;
public ChatClient( String host, int port ) {
try {
Socket socket = new Socket( host, port );
socketIn = new Scanner( socket.getInputStream() );
socketOut = new PrintWriter( socket.getOutputStream(), true );
keyboardIn = new Scanner( System.in );
inputStream = new ObjectInputStream(socket.getInputStream());
}
catch( IOException e ) {
e.printStackTrace();
}
}
public void downloadFiles() {
while (socket.isConnected()) {
try {
fileEvent = (FileEvent) inputStream.readObject();
if (fileEvent.getStatus().equalsIgnoreCase("Error")) {
System.out.println("Error occurred ..with file" + fileEvent.getFilename() + "at sending end ..");
}
String outputFile = fileEvent.getDestinationDirectory() + fileEvent.getFilename();
if (!new File(fileEvent.getDestinationDirectory()).exists()) {
new File(fileEvent.getDestinationDirectory()).mkdirs();
}
dstFile = new File(outputFile);
fileOutputStream = new FileOutputStream(dstFile);
fileOutputStream.write(fileEvent.getFileData());
fileOutputStream.flush();
fileOutputStream.close();
System.out.println("Output file : " + outputFile + " is successfully saved ");
if (fileEvent.getRemainder() == 0) {
System.out.println("Whole directory is copied...So system is going to exit");
System.exit(0);
}
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
}
}
private void talktoServer() {
String message;
Thread readerThread = new Thread( new IncomingReader() );
readerThread.start();
while ((message = keyboardIn.nextLine()) != null) {
System.out.println("client typed: " + message);
socketOut.println( message );
}
}
private class IncomingReader implements Runnable {
public void run() {
String message;
while ((message = socketIn.nextLine()) != null) {
System.out.println("client read: " + message);
if(message.equals("ready")) {
downloadFiles();
}
}
}
}
public static void main( String[] args ) {
ChatClient c = new ChatClient( "127.0.0.1", 5000 );
c.talktoServer();
}
}
Any help at all would be greatly appreciated!
I'm trying to build a neural network using deep learning4j framework, I get the following error :
java.lang.IllegalStateException: Unexpected state occurred for AsyncDataSetIterator: runnable died or no data available" this exception
Here is my code
package com.neuralnetwork;
import com.sliit.preprocessing.NormalizeDataset;
import com.sliit.ruleengine.RuleEngine;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.datavec.api.records.reader.SequenceRecordReader;
import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader;
import org.deeplearning4j.eval.Evaluation;
import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.Updater;
import org.deeplearning4j.nn.conf.layers.GravesLSTM;
import org.deeplearning4j.nn.conf.layers.RnnOutputLayer;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.deeplearning4j.util.ModelSerializer;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.api.DataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction;
import weka.core.Instances;
import weka.core.converters.CSVLoader;
import weka.core.converters.CSVSaver;
import weka.filters.Filter;
import weka.filters.supervised.instance.StratifiedRemoveFolds;
import java.io.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
/**
* Deep Belif Neural Network to detect frauds.
*/
public class FraudDetectorDeepBeilefNN {
private static final Log log = LogFactory.getLog(FraudDetectorDeepBeilefNN.class);
public int outputNum = 4;
private int iterations = 5;
private int seed = 1234;
private MultiLayerNetwork model = null;
public int HIDDEN_LAYER_COUNT = 8;
public int numHiddenNodes = 21;
public int inputs = 41;
private String uploadDirectory = "D:/Data";
private ArrayList < Map < String, Double >> roc;
public FraudDetectorDeepBeilefNN() {
}
public void buildModel() {
System.out.println("Build model....");
iterations = outputNum + 1;
NeuralNetConfiguration.Builder builder = new NeuralNetConfiguration.Builder();
builder.iterations(iterations);
builder.learningRate(0.001);
// builder.momentum(0.01);
builder.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
builder.seed(seed);
builder.biasInit(1);
builder.regularization(true).l2(1e-5);
builder.updater(Updater.RMSPROP);
builder.weightInit(WeightInit.XAVIER);
NeuralNetConfiguration.ListBuilder list = builder.list();
for (int i = 0; i < HIDDEN_LAYER_COUNT; i++) {
GravesLSTM.Builder hiddenLayerBuilder = new GravesLSTM.Builder();
hiddenLayerBuilder.nIn(i == 0 ? inputs : numHiddenNodes);
hiddenLayerBuilder.nOut(numHiddenNodes);
hiddenLayerBuilder.activation("tanh");
list.layer(i, hiddenLayerBuilder.build());
}
RnnOutputLayer.Builder outputLayer = new RnnOutputLayer.Builder(LossFunction.MCXENT);
outputLayer.activation("softmax");
outputLayer.nIn(numHiddenNodes);
outputLayer.nOut(outputNum);
list.layer(HIDDEN_LAYER_COUNT, outputLayer.build());
list.pretrain(false);
list.backprop(true);
MultiLayerConfiguration configuration = list.build();
model = new MultiLayerNetwork(configuration);
model.init();
//model.setListeners(new ScoreIterationListener(1));
}
public String trainModel(String modelName, String filePath, int outputs, int inputsTot) throws NeuralException {
try {
System.out.println("Neural Network Training start");
loadSaveNN(modelName, false);
if (model == null) {
buildModel();
}
System.out.println("modal" + model);
System.out.println("file path " + filePath);
File fileGeneral = new File(filePath);
CSVLoader loader = new CSVLoader();
loader.setSource(fileGeneral);
Instances instances = loader.getDataSet();
instances.setClassIndex(instances.numAttributes() - 1);
StratifiedRemoveFolds stratified = new StratifiedRemoveFolds();
String[] options = new String[6];
options[0] = "-N";
options[1] = Integer.toString(5);
options[2] = "-F";
options[3] = Integer.toString(1);
options[4] = "-S";
options[5] = Integer.toString(1);
stratified.setOptions(options);
stratified.setInputFormat(instances);
stratified.setInvertSelection(false);
Instances testInstances = Filter.useFilter(instances, stratified);
stratified.setInvertSelection(true);
Instances trainInstances = Filter.useFilter(instances, stratified);
String directory = fileGeneral.getParent();
CSVSaver saver = new CSVSaver();
File trainFile = new File(directory + "/" + "normtrainadded.csv");
File testFile = new File(directory + "/" + "normtestadded.csv");
if (trainFile.exists()) {
trainFile.delete();
}
trainFile.createNewFile();
if (testFile.exists()) {
testFile.delete();
}
testFile.createNewFile();
saver.setFile(trainFile);
saver.setInstances(trainInstances);
saver.writeBatch();
saver = new CSVSaver();
saver.setFile(testFile);
saver.setInstances(testInstances);
saver.writeBatch();
SequenceRecordReader recordReader = new CSVSequenceRecordReader(0, ",");
recordReader.initialize(new org.datavec.api.split.FileSplit(trainFile));
SequenceRecordReader testReader = new CSVSequenceRecordReader(0, ",");
testReader.initialize(new org.datavec.api.split.FileSplit(testFile));
DataSetIterator iterator = new org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator(recordReader, 2, outputs, inputsTot, false);
DataSetIterator testIterator = new org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator(testReader, 2, outputs, inputsTot, false);
roc = new ArrayList < Map < String, Double >> ();
String statMsg = "";
Evaluation evaluation;
for (int i = 0; i < 100; i++) {
if (i % 2 == 0) {
model.fit(iterator);
evaluation = model.evaluate(testIterator);
} else {
model.fit(testIterator);
evaluation = model.evaluate(iterator);
}
Map < String, Double > map = new HashMap < String, Double > ();
Map < Integer, Integer > falsePositives = evaluation.falsePositives();
Map < Integer, Integer > trueNegatives = evaluation.trueNegatives();
Map < Integer, Integer > truePositives = evaluation.truePositives();
Map < Integer, Integer > falseNegatives = evaluation.falseNegatives();
double fpr = falsePositives.get(1) / (falsePositives.get(1) + trueNegatives.get(1));
double tpr = truePositives.get(1) / (truePositives.get(1) + falseNegatives.get(1));
map.put("FPR", fpr);
map.put("TPR", tpr);
roc.add(map);
statMsg = evaluation.stats();
iterator.reset();
testIterator.reset();
}
loadSaveNN(modelName, true);
System.out.println("ROC " + roc);
return statMsg;
} catch (Exception e) {
e.printStackTrace();
System.out.println("Error ocuured while building neural netowrk :" + e.getMessage());
throw new NeuralException(e.getLocalizedMessage(), e);
}
}
public boolean generateModel(String modelName) {
boolean status = false;
try {
loadSaveNN(modelName, true);
status = true;
} catch (Exception e) {
System.out.println("Error occurred:" + e.getLocalizedMessage());
}
return status;
}
private void loadSaveNN(String name, boolean save) {
File directory = new File(uploadDirectory);
File[] allNN = directory.listFiles();
boolean status = false;
try {
if (model == null && save) {
buildModel();
}
if (allNN != null && allNN.length > 0) {
for (File NN: allNN) {
String fnme = FilenameUtils.removeExtension(NN.getName());
if (name.equals(fnme)) {
status = true;
if (save) {
ModelSerializer.writeModel(model, NN, true);
System.out.println("Model Saved With Weights Successfully");
} else {
model = ModelSerializer.restoreMultiLayerNetwork(NN);
}
break;
}
}
}
if (!status && save) {
//File tempFIle = File.createTempFile(name,".zip",directory);
File tempFile = new File(directory.getAbsolutePath() + "/" + name + ".zip");
if (!tempFile.exists()) {
tempFile.createNewFile();
}
ModelSerializer.writeModel(model, tempFile, true);
}
} catch (IOException e) {
System.out.println("Error occurred:" + e.getMessage());
}
}
public String testModel(String modelName, String[] rawData, Map < Integer, String > map, int inputs, int outputs, String ruleModelSavePath) throws Exception {
String status = "";
String fpath = uploadDirectory;
FileWriter fwriter = new FileWriter(uploadDirectory + "original/insertdata.csv", true);
fwriter.write("\n");
fwriter.write(rawData[0]);
fwriter.close();
if (model == null) {
loadSaveNN(modelName, false);
}
NormalizeDataset norm = new NormalizeDataset(uploadDirectory + "original/insertdata.csv");
norm.updateStringValues(map);
norm.whiteningData();
norm.normalizeDataset();
BufferedReader bufferedReader = new BufferedReader(new FileReader(new File(uploadDirectory + "originalnorminsertdata.csv")));
String output = "";
String prevOutput = "";
while ((output = bufferedReader.readLine()) != null) {
prevOutput = output;
}
bufferedReader.close();
File readFile = new File(uploadDirectory + "normtest.csv");
if (readFile.exists()) {
readFile.delete();
}
readFile.createNewFile();
PrintWriter writer = new PrintWriter(readFile);
writer.println(prevOutput);
writer.flush();
writer.close();
SequenceRecordReader recordReader = new CSVSequenceRecordReader(0, ",");
recordReader.initialize(new org.datavec.api.split.FileSplit(new File(uploadDirectory + "normtest.csv")));
DataSetIterator iterator = new org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator(recordReader, 2, outputs, inputs, false);
INDArray outputArr = null;
while (iterator.hasNext()) {
DataSet ds = iterator.next();
INDArray provided = ds.getFeatureMatrix();
outputArr = model.rnnTimeStep(provided);
}
//INDArray finalOutput = Nd4j.argMax(outputArr,1);
double result = Double.parseDouble(Nd4j.argMax(outputArr, 1).toString());
if (result == 0.0) {
status = "Normal Transaction";
} else {
status = "Fraud Transaction, ";
bufferedReader = new BufferedReader(new FileReader(new File(uploadDirectory + "original/insertdata.csv")));
String heading = "";
heading = bufferedReader.readLine();
bufferedReader.close();
File ruleFile = new File(uploadDirectory + "normrules.csv");
if (ruleFile.exists()) {
ruleFile.delete();
}
ruleFile.createNewFile();
PrintWriter writeNew = new PrintWriter(ruleFile);
writeNew.println(heading);
writeNew.println(rawData[0]);
writeNew.flush();
writeNew.close();
RuleEngine engine = new RuleEngine(fpath + "original/insertdata.csv");
engine.geneateModel(ruleModelSavePath, false);
String finalStatus = status + "Attack Type:" + engine.predictionResult(uploadDirectory + "normrules.csv");
status = finalStatus;
}
return status;
}
public void setUploadDirectory(String uploadDirectory) {
this.uploadDirectory = uploadDirectory;
}
public static void main(String[] args) {
FraudDetectorDeepBeilefNN neural_network = new FraudDetectorDeepBeilefNN();
System.out.println("start=======================");
try {
neural_network.inputs = 4;
neural_network.numHiddenNodes = 3;
neural_network.HIDDEN_LAYER_COUNT = 2;
neural_network.outputNum = 2;
neural_network.buildModel();
String output = neural_network.trainModel("nn", "D:/Data/a.csv", 2, 4);
System.out.println(output);
} catch (Exception e) {
e.printStackTrace();
}
}
public ArrayList < Map < String, Double >> getRoc() {
return roc;
}
}
Here is the 1st few lines of my dataset :
length,caplen,hlen,version,output
60,60,6,0,normal
243,243,8,0,normal
60,60,6,0,neptune
60,60,6,0,neptune
I'm using Encog and I ran the ocr sample. It works fine. However, I want to pass an image file (png, jpg,...) as a parameter. This image contains the text to to be recognized. Then, the system should return a string with the "same" text.
Has someone already did something similar? How should I start?
Thanks!
Step 1: In GUI create file input and take file from user
JFileChooser fc;
JButton b, b1;
JTextField tf;
FileInputStream in;
Socket s;
DataOutputStream dout;
DataInputStream din;
int i;
public void actionPerformed(ActionEvent e) {
try {
if (e.getSource() == b) {
int x = fc.showOpenDialog(null);
if (x == JFileChooser.APPROVE_OPTION) {
fileToBeSent = fc.getSelectedFile();
tf.setText(f1.getAbsolutePath());
b1.setEnabled(true);
} else {
fileToBeSent = null;
tf.setText(null;);
b1.setEnabled(false);
}
}
if (e.getSource() == b1) {
send();
}
} catch (Exception ex) {
}
}
public void copy() throws IOException {
File f1 = fc.getSelectedFile();
tf.setText(f1.getAbsolutePath());
in = new FileInputStream(f1.getAbsolutePath());
while ((i = in.read()) != -1) {
System.out.print(i);
}
}
public void send() throws IOException {
dout.write(i);
dout.flush();
}
Step 2: Down sample it
private void processNetwork() throws IOException {
System.out.println("Downsampling images...");
for (final ImagePair pair : this.imageList) {
final MLData ideal = new BasicMLData(this.outputCount);
final int idx = pair.getIdentity();
for (int i = 0; i < this.outputCount; i++) {
if (i == idx) {
ideal.setData(i, 1);
} else {
ideal.setData(i, -1);
}
}
final Image img = ImageIO.read(fc.getFile());
final ImageMLData data = new ImageMLData(img);
this.training.add(data, ideal);
}
final String strHidden1 = getArg("hidden1");
final String strHidden2 = getArg("hidden2");
this.training.downsample(this.downsampleHeight, this.downsampleWidth);
final int hidden1 = Integer.parseInt(strHidden1);
final int hidden2 = Integer.parseInt(strHidden2);
this.network = EncogUtility.simpleFeedForward(this.training
.getInputSize(), hidden1, hidden2,
this.training.getIdealSize(), true);
System.out.println("Created network: " + this.network.toString());
}
Step 3: Begin Training with training set
private void processTrain() throws IOException {
final String strMode = getArg("mode");
final String strMinutes = getArg("minutes");
final String strStrategyError = getArg("strategyerror");
final String strStrategyCycles = getArg("strategycycles");
System.out.println("Training Beginning... Output patterns="
+ this.outputCount);
final double strategyError = Double.parseDouble(strStrategyError);
final int strategyCycles = Integer.parseInt(strStrategyCycles);
final ResilientPropagation train = new ResilientPropagation(this.network, this.training);
train.addStrategy(new ResetStrategy(strategyError, strategyCycles));
if (strMode.equalsIgnoreCase("gui")) {
TrainingDialog.trainDialog(train, this.network, this.training);
} else {
final int minutes = Integer.parseInt(strMinutes);
EncogUtility.trainConsole(train, this.network, this.training,
minutes);
}
System.out.println("Training Stopped...");
}
Step 4: Give down sampled file to neural network
public void processWhatIs() throws IOException {
final String filename = getArg("image");
final File file = new File(filename);
final Image img = ImageIO.read(file);
final ImageMLData input = new ImageMLData(img);
input.downsample(this.downsample, false, this.downsampleHeight,
this.downsampleWidth, 1, -1);
final int winner = this.network.winner(input);
System.out.println("What is: " + filename + ", it seems to be: "
+ this.neuron2identity.get(winner));
}
Step 5: Check Result
When I finish to do my home task, I get error "could not find or load main class".
I spent near 20 minutes in fully disunderstanding. Reason was "Kaspersky Anti-Virus", which was blocking my MyClass.class.
Here is the code, anyone knows, what's here Kaspersky can indentified as virus?
import javax.swing.*;
import java.awt.*;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.*;
public class CryptoUtil extends JFrame{
private String keyWord;
private JFileChooser inputFileChooser = new JFileChooser();
private JButton openFileChooser = new JButton("Выбрать файл");
private JTextField inputKeyWord = new JTextField();
private JButton operate = new JButton("Выполнить");
private File inputFile;
private File encode;
private File decode;
private String inputData;
public CryptoUtil(){
initFrame();
}
public void initFrame(){
this.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
this.setTitle("Шифратор/дешифратор XOR");
this.setLayout(new GridLayout(3, 1));
this.setSize(300, 100);
operate.addActionListener(new ActionListener() {
#Override
public void actionPerformed(ActionEvent e) {
try {
operate();
Desktop.getDesktop().open(new File(encode.getParent()));
} catch (IOException e1) {
e1.printStackTrace();
}
}
});
openFileChooser.addActionListener(new ActionListener() {
#Override
public void actionPerformed(ActionEvent e) {
inputFileChooser.showOpenDialog(CryptoUtil.this);
}
});
this.add(openFileChooser);
this.add(inputKeyWord);
this.add(operate);
}
public void operate() throws IOException {
keyWord = inputKeyWord.getText();
inputFile = inputFileChooser.getSelectedFile();
encode = new File(inputFile.getParent() + "//encode.txt");
decode = new File(inputFile.getParent() + "//decode.txt");
BufferedReader br = new BufferedReader(new FileReader(inputFile));
StringBuilder sb = new StringBuilder();
String line = br.readLine();
while (line != null) {
sb.append(line);
sb.append(System.lineSeparator());
line = br.readLine();
}
inputData = sb.toString();
br.close();
PrintWriter writerEn = new PrintWriter(encode, "UTF-8");
writerEn.print(encrypt(inputData, keyWord));
writerEn.close();
PrintWriter writerDe = new PrintWriter(decode, "UTF-8");
writerDe.print(decrypt(encrypt(inputData, keyWord), keyWord));
writerDe.close();
}
public byte[] encrypt(String text, String keyWord){
byte[] arr = text.getBytes();
byte[] keyArr = keyWord.getBytes();
byte[] result = new byte[arr.length];
for(int i = 0; i < arr.length; i++){
result[i] = (byte) (arr[i] ^ keyArr[i % keyArr.length]);
}
return result;
}
public String decrypt(byte[] text, String keyWord){
byte[] result = new byte[text.length];
byte[] keyArr = keyWord.getBytes();
for(int i = 0; i < text.length;i++){
result[i] = (byte) (text[i] ^ keyArr[i% keyArr.length]);
}
return new String(result);
}
public static void main(String[] args) {
new CryptoUtil().setVisible(true);
}
}
You can add your file as trusted in kaspersky antivirus, same happened with me also, necessarily it won't be a virus.