We Keep Coding

Learning curve from a weka experiment java

I am trying to get a learning curve for an automated weka experiment. I currently have the following java code.
public static void EvaluateModel(AbstractClassifier cl, String datapath, String outfile) throws Exception {
Experiment exp = new Experiment();
ClassifierSplitEvaluator se = new ClassifierSplitEvaluator();
se.setClassifier(cl);
Classifier sec = ((ClassifierSplitEvaluator) se).getClassifier();
CrossValidationResultProducer cvrp = new CrossValidationResultProducer();
cvrp.setNumFolds(10);
cvrp.setSplitEvaluator(se);
PropertyNode[] propertyPath = new PropertyNode[2];
try {
propertyPath[0] = new PropertyNode(
se,
new PropertyDescriptor("splitEvaluator",
CrossValidationResultProducer.class),
CrossValidationResultProducer.class);
propertyPath[1] = new PropertyNode(sec,
new PropertyDescriptor("classifier", se.getClass()),
se.getClass());
} catch (IntrospectionException e) {
e.printStackTrace();
}
exp.setResultProducer(cvrp);
exp.setPropertyPath(propertyPath);
exp.setPropertyArray(new Classifier[]{cl});
DefaultListModel model = new DefaultListModel();
model.addElement(new File(datapath));
exp.setDatasets(model);
InstancesResultListener irl = new InstancesResultListener();
irl.setOutputFile(new File(outfile));
exp.setResultListener(irl);
System.out.println("Initializing...");
exp.initialize();
System.out.println("Running...");
exp.runExperiment();
System.out.println("Finishing...");
exp.postProcess();
System.out.println("Evaluating...");
PairedTTester tester = new PairedCorrectedTTester();
FileReader reader = new FileReader(irl.getOutputFile());
Instances result = new Instances(reader);
tester.setInstances(result);
tester.setSortColumn(-1);
tester.setRunColumn(result.attribute("Key_Run").index());
tester.setFoldColumn(result.attribute("Key_Fold").index());
tester.setDatasetKeyColumns(
new Range(
""
+ (result.attribute("Key_Dataset").index() + 1)));
tester.setResultsetKeyColumns(
new Range(
""
+ (result.attribute("Key_Scheme").index() + 1)
+ ","
+ (result.attribute("Key_Scheme_options").index() + 1)
+ ","
+ (result.attribute("Key_Scheme_version_ID").index() + 1)));
tester.setResultMatrix(new ResultMatrixPlainText());
tester.setDisplayedResultsets(null);
tester.setSignificanceLevel(0.05);
tester.setShowStdDevs(true);
// fill result matrix (but discarding the output)
tester.multiResultsetFull(0, result.attribute("Percent_correct").index());
// output results for reach dataset
System.out.println("\nResult:");
ResultMatrix matrix = tester.getResultMatrix();
for (int i = 0; i < matrix.getColCount(); i++) {
System.out.println(matrix.getColName(i));
System.out.println(" Perc. correct: " + matrix.getMean(i, 0));
System.out.println(" StdDev: " + matrix.getStdDev(i, 0));
}
}
What I would like to do is either save or display the learning curve in this method. I cannot find info for how to do this programmatically.

How does regex query work on lucene?

I am trying to implement luecene search engine in my application.
I am using lucene 5.4.1
I have successfully implemented wildequeries and normal queries of lucene.
But my main focus is to search specific text in a text file with regex patterns.
Index Writer code:
public IndexWriter generateIndex(String docsPath) throws IOException {
String indexPath = System.getProperty("java.io.tmpdir") +File.separator+"indexDirectory";
if (indexPath == null) {
throw new IOException("System property 'java.io.tmpdir' does not specify a tmp dir");
}
File tmpDir = new File(indexPath);
if (!tmpDir.exists()) {
boolean created = tmpDir.mkdirs();
if (!created) {
throw new IOException("Unable to create tmp dir " + tmpDir);
}
}
boolean create = true;
final Path docDir = Paths.get(docsPath);
if (!Files.isReadable(docDir)) {
System.out.println("Document directory '" + docDir.toAbsolutePath()
+ "' does not exist or is not readable, please check the path");
System.exit(1);
}
Date start = new Date();
try {
System.out.println("Indexing to directory '" + indexPath + "'...");
Directory dir = FSDirectory.open(Paths.get(indexPath));
Analyzer analyzer = new StandardAnalyzer();
IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
if (create) {
iwc.setOpenMode(OpenMode.CREATE);
} else {
iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
}
IndexWriter writer = new IndexWriter(dir, iwc);
indexDocs(writer, docDir);
setIndexWriter(writer);
Date end = new Date();
System.out.println(end.getTime() - start.getTime() + " total milliseconds");
writer.close();
} catch (IOException e) {
System.out.println(" caught a " + e.getClass() + "\n with message: " + e.getMessage());
}
return getIndexWriter();
}
static void indexDocs(final IndexWriter writer, Path path) throws IOException {
if (Files.isDirectory(path)) {
Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
#Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
try {
indexDoc(writer, file, attrs.lastModifiedTime().toMillis());
} catch (IOException ignore) {
// don't index files that can't be read.
}
return FileVisitResult.CONTINUE;
}
});
} else {
indexDoc(writer, path, Files.getLastModifiedTime(path).toMillis());
}
}
static void indexDoc(IndexWriter writer, Path file, long lastModified) throws IOException {
try (InputStream stream = Files.newInputStream(file)) {
Document doc = new Document();
Field pathField = new StringField("path", file.toString(), Field.Store.NO);
doc.add(pathField);
doc.add(new LongField("modified", lastModified, Field.Store.NO));
doc.add(new TextField("contents",
new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))));
if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
System.out.println("adding " + file);
writer.addDocument(doc);
} else {
System.out.println("updating " + file);
writer.updateDocument(new Term("path", file.toString()), doc);
}
}
}
Index Searching Code:
public IndexReader searchExecutor(String index, String queryString, RegexCapabilities capability) throws Exception {
String field = "contents";
String queries = null;
boolean raw = false;
int hitsPerPage = Integer.MAX_VALUE;
IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(index)));
IndexSearcher searcher = new IndexSearcher(reader);
Analyzer analyzer = new StandardAnalyzer();
BufferedReader in = null;
Query q = new RegexpQuery(new Term("text", queryString));
q = q.rewrite(reader);
RegexQuery query = new RegexQuery(new Term("\\s*(FIND|find)"));
if (capability != null)
query.setRegexImplementation(capability);
System.out.println("Searching for: " + query.toString(field));
searcher.search(query, null, 1000);
doSearch(in, searcher, query, hitsPerPage, raw, queries == null && queryString == null);
//reader.close();
return reader;
}
public static void doSearch(BufferedReader in, IndexSearcher searcher, Query query, int hitsPerPage, boolean raw,
boolean interactive)
throws IOException {
TopDocs results = searcher.search(query, 5 * hitsPerPage);
ScoreDoc[] hits = results.scoreDocs;
//generateIndex.deleteDocuments(query);
//generateIndex.getDirectory();
// TermsEnum.totalTermFreq();
int numTotalHits = results.totalHits;
System.out.println(numTotalHits + " total matching documents");
int start = 0;
int end = Math.min(numTotalHits, hitsPerPage);
for (int i = start; i < end; i++) {
Document doc = searcher.doc(hits[i].doc);
String path = doc.get("path");
File file = new File(path);
if (path != null) {
System.out.println((i + 1) + ". " + path);
String title = doc.get("title");
if (title != null) {
System.out.println(" Title: " + doc.get("title"));
}
} else {
System.out.println((i + 1) + ". " + "No path for this document");
}
}
}
Please help.

Your question is about search with regular expressions in lucene.
You are using RegexQuery which is deprecated so try RegexpQuery
Your regEx-example starts with \s* but you do not use KeywordTokenizer. Most other tokenizer will remove (aka "split at") whitespace
Your regEx-example is not purely lower case. But standard analyzer contains LowerCaseFilter. Be aware: your regEx will go directly against the tokens of your index (not against the original text)
--> read Supported RegExp syntax and syntax in ES and TestRegexpRandom (test class) and play with https://github.com/DmitryKey/luke on your index.

Creating a jsp search form to run a java Search program

The background info here is that I have a working Indexer and Search (in java) that indexes and searches a file directory for the filenames and then copies the files to a "Results" Directory.
What I need/ don't have much experience in is writing jsp files. I need the jsp file to have a search bar for the text and then a search button. When text is entered in the bar, and the button is clicked, I need it to run my search program with the entered text as an arg.
I have added the IndexFiles and the SearchFiles classes for reference.
Please explain with a good example if you can help out!
public class SearchFiles {
static File searchDirectory = new File(
"C:\\Users\\flood.j.2\\Desktop\\IndexSearch\\Results");
static String v = new String();
static String path = null;
String title = null;
File addedFile = null;
OutputStream out = null;
String dirName = "C:\\Users\\flood.j.2\\Desktop\\IndexSearch\\Results";
public static void main(String[] args) throws Exception {
String usage = "Usage:\tjava org.apache.lucene.demo.SearchFiles [-index dir] [-field f] [-repeat n] [-queries file] [-query string]";
if (args.length > 0
&& ("-h".equals(args[0]) || "-help".equals(args[0]))) {
System.out.println(usage);
System.exit(0);
}
for (int j = 5; j < args.length; j++) {
v += args[j] + " ";
}
String index = "index";
String field = "contents";
String queries = null;
boolean raw = false;
String queryString = null;
int hits = 100;
for (int i = 0; i < args.length; i++) {
if ("-index".equals(args[i])) {
index = args[i + 1];
i++;
} else if ("-field".equals(args[i])) {
field = args[i + 1];
i++;
} else if ("-queries".equals(args[i])) {
queries = args[i + 1];
i++;
} else if ("-query".equals(args[i])) {
queryString = v;
i++;
}
}
IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(
index)));
IndexSearcher searcher = new IndexSearcher(reader);
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_40);
BufferedReader in = null;
if (queries != null) {
in = new BufferedReader(new InputStreamReader(new FileInputStream(
queries), "UTF-8"));
} else {
in = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
}
QueryParser parser = new QueryParser(Version.LUCENE_40, field, analyzer);
for (int m = 0; m < 2; m++) {
if (queries == null && queryString == null) {
System.out.println("Enter query: ");
}
String line = queryString != null ? queryString : in.readLine();
if (line == null || line.length() == -1) {
break;
}
line = line.trim();
if (line.length() == 0) {
break;
}
Query query = parser.parse(line);
System.out.println("Searching for: " + query.toString(field));
doPagingSearch(in, searcher, query, hits, raw, queries == null
&& queryString == null);
if (queryString == null) {
break;
}
}
reader.close();
}
public static void doPagingSearch(BufferedReader in,
IndexSearcher searcher, Query query, int hitsPerPage, boolean raw,
boolean interactive) throws IOException {
// Collect enough docs to show 500 pages
TopDocs results = searcher.search(query, 5 * hitsPerPage);
ScoreDoc[] hits = results.scoreDocs;
int numTotalHits = results.totalHits;
System.out.println(numTotalHits + " total matching documents");
int start = 0;
int end = Math.min(numTotalHits, hitsPerPage);
FileUtils.deleteDirectory(searchDirectory);
while (true) {
for (int i = start; i < end; i++) {
Document doc = searcher.doc(hits[i].doc);
path = doc.get("path");
if (path != null) {
System.out.println((i + 1) + ". " + path);
File addFile = new File(path);
try {
FileUtils.copyFileToDirectory(addFile, searchDirectory);
} catch (IOException e) {
e.printStackTrace();
}
}
}
if (!interactive || end == 0) {
break;
}
System.exit(0);
}
}
}
public class IndexFiles {
private IndexFiles() {
}
public static void main(String[] args) {
String usage = "java org.apache.lucene.demo.IndexFiles"
+ " [-index INDEX_PATH] [-docs DOCS_PATH] [-update]\n\n"
+ "This indexes the documents in DOCS_PATH, creating a Lucene index"
+ "in INDEX_PATH that can be searched with SearchFiles";
String indexPath = null;
String docsPath = null;
boolean create = true;
for (int i = 0; i < args.length; i++) {
if ("-index".equals(args[i])) {
indexPath = args[i + 1];
i++;
} else if ("-docs".equals(args[i])) {
docsPath = args[i + 1];
i++;
} else if ("-update".equals(args[i])) {
create = false;
}
}
if (docsPath == null) {
System.err.println("Usage: " + usage);
System.exit(1);
}
final File docDir = new File(docsPath);
if (!docDir.exists() || !docDir.canRead()) {
System.out
.println("Document directory '"
+ docDir.getAbsolutePath()
+ "' does not exist or is not readable, please check the path");
System.exit(1);
}
Date start = new Date();
try {
System.out.println("Indexing to directory '" + indexPath + "'...");
Directory dir = FSDirectory.open(new File(indexPath));
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_40);
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_40,
analyzer);
if (create) {
iwc.setOpenMode(OpenMode.CREATE);
} else {
iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
}
IndexWriter writer = new IndexWriter(dir, iwc);
indexDocs(writer, docDir);
writer.close();
Date end = new Date();
System.out.println(end.getTime() - start.getTime()
+ " total milliseconds");
} catch (IOException e) {
System.out.println(" caught a " + e.getClass()
+ "\n with message: " + e.getMessage());
}
}
static void indexDocs(IndexWriter writer, File file) throws IOException {
if (file.canRead()) {
if (file.isDirectory()) {
String[] files = file.list();
if (files != null) {
for (int i = 0; i < files.length; i++) {
indexDocs(writer, new File(file, files[i]));
}
}
} else {
FileInputStream fis;
try {
fis = new FileInputStream(file);
} catch (FileNotFoundException fnfe) {
return;
}
try {
Document doc = new Document();
Field pathField = new StringField("path",
file.getAbsolutePath(), Field.Store.YES);
doc.add(pathField);
doc.add(new LongField("modified", file.lastModified(),
Field.Store.NO));
doc.add(new TextField("title", file.getName(), null));
System.out.println(pathField);
if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
System.out.println("adding " + file);
writer.addDocument(doc);
} else {
System.out.println("updating " + file);
writer.updateDocument(new Term("path", file.getPath()),
doc);
}
} finally {
fis.close();
}
}
}
}
}

First, you should definitely do this in a servlet rather than a JSP. Putting lots of logic in JSP is bad practice. (See the servlets info page).
Second, it would probably be better on performance to make a cronjob (Linux) or Task (Windows) to run the search program every hour and store the results in a database and just have your servlet pull from there rather than allow the user to initiate the search program.

PhraseQuery Not working Lucene 4.5.0

I tried to work with PhraseQuery but could not get hits from search. I am using Lucene 4.5.0.
My Indexing code
private IndexWriter writer;
public LuceneIndexSF(final String indexDir) throws IOException {
Analyzer analyzer = new KeywordAnalyzer();
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_45,
analyzer);
Directory directory = FSDirectory.open(new File(indexDir));
writer = new IndexWriter(directory, config);
}
private Document getDocument(File f, String line, int lineNum)
throws IOException {
Document doc = new Document();
Field field = null;
if (line != null && line.split(DELIMITER).length >= 5) {
String[] lineValues = line.split(DELIMITER);
field = new Field("name", line.split("\t")[1],
TextField.TYPE_STORED);
doc.add(field);
if (lineValues[2] != null && !lineValues[2].trim().isEmpty()) {
field = new Field("ref", lineValues[2], TextField.TYPE_STORED);
doc.add(field);
}
field = new Field("type", lineValues[3], TextField.TYPE_STORED);
doc.add(field);
field = new LongField("code", Long.parseLong(lineValues[4]),
LongField.TYPE_STORED);
doc.add(field);
if (lineValues.length == 7 && lineValues[5] != null
&& !lineValues[5].trim().isEmpty()) {
field = new Field("alias1", lineValues[5],
TextField.TYPE_STORED);
doc.add(field);
}
if (lineValues.length == 7 && lineValues[6] != null
&& !lineValues[6].trim().isEmpty()) {
field = new Field("alias2", lineValues[6],
TextField.TYPE_STORED);
doc.add(field);
}
}
field = new IntField("linenum", lineNum, IntField.TYPE_STORED);
doc.add(field);
return doc;
}
.... and other code where i add document in writer using writer.addDocument(doc);
My Searching Code
private static void search(String indexDir, String quer) throws IOException,
ParseException {
IndexReader inxRead = DirectoryReader.open(FSDirectory.open(new File(
indexDir)));
IndexSearcher is = new IndexSearcher(inxRead);
String[] termArr = quer.split(" ");
PhraseQuery phraseQuery= new PhraseQuery();
for(int inx = 0; inx < termArr.length; inx++){
phraseQuery.add(new Term("name", termArr[inx]));
}
phraseQuery.setSlop(4);
long start = System.currentTimeMillis();
TopDocs hits = is.search(phraseQuery, 1000);
long end = System.currentTimeMillis();
System.err.println("Parser> Found " + hits.totalHits
+ " document(s) (in " + (end - start)
+ " milliseconds) that matched query '" + multiQuery + "':");
for (ScoreDoc scoreDoc : hits.scoreDocs) {
Document doc = is.doc(scoreDoc.doc);
System.out.println("Parser> " + scoreDoc.score + " :: "
+ doc.get("type") + " - " + doc.get("code") + " - "
+ doc.get("name") + ", " + doc.get("linenum"));
}
inxRead.close();
}
Please tell me if i am doing any thing wrong.
Edit
also tried with Standard Analyzer still not results
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_45);
Solution
According to Arun's answer PhraseQuery to work properly requires Analyzer which Tokenize each word in Document's Field for my case i used LowerCaseFilter with making all queries lower case so that it can work without case sensitivity. And used EdgeNGramTokenFilter which for auto completion purposes.
public LuceneIndexSF(final String indexDir) throws IOException {
Analyzer analyzer = new Analyzer() {
#Override
protected TokenStreamComponents createComponents(String fieldName,
java.io.Reader reader) {
Tokenizer source = new StandardTokenizer(Version.LUCENE_45,
reader);
TokenStream result = new StandardFilter(Version.LUCENE_45,
source);
result = new LowerCaseFilter(Version.LUCENE_45, result);
result = new EdgeNGramTokenFilter(Version.LUCENE_45, result, 1,
20);
return new TokenStreamComponents(source, result);
}
};
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_45,
analyzer);
Directory directory = FSDirectory.open(new File(indexDir));
writer = new IndexWriter(directory, config);
}
My final search method
private static void search(String indexDir, String quer) throws IOException,
ParseException {
IndexReader inxRead = DirectoryReader.open(FSDirectory.open(new File(
indexDir)));
IndexSearcher is = new IndexSearcher(inxRead);
String[] termArr = quer.split(" ");
PhraseQuery query1 = new PhraseQuery();
PhraseQuery query2 = new PhraseQuery();
PhraseQuery query3 = new PhraseQuery();
for (int inx = 0; inx < termArr.length; inx++) {
query1.add(new Term(SchoolFinderConstant.ENTITY_NAME,termArr[inx]),inx);
query2.add(new Term(SchoolFinderConstant.ENTITY_ALIAS1,termArr[inx]),inx);
query3.add(new Term(SchoolFinderConstant.ENTITY_ALIAS2,termArr[inx]),inx);
}
BooleanQuery mainQuery = new BooleanQuery();
mainQuery.add(query1, Occur.SHOULD);
mainQuery.add(query2, Occur.SHOULD);
mainQuery.add(query3, Occur.SHOULD);
long start = System.currentTimeMillis();
TopDocs hits = is.search(mainQuery, 1000);
long end = System.currentTimeMillis();
System.err.println("Parser> Found " + hits.totalHits
+ " document(s) (in " + (end - start)
+ " milliseconds) that matched query '" + multiQuery + "':");
for (ScoreDoc scoreDoc : hits.scoreDocs) {
Document doc = is.doc(scoreDoc.doc);
System.out.println("Parser> " + scoreDoc.score + " :: "
+ doc.get("type") + " - " + doc.get("code") + " - "
+ doc.get("name") + ", " + doc.get("linenum"));
}
inxRead.close();
}

I played with your code with KeywordAnalyzer, obviously it did not work as the KeywordAnalyzer "Tokenizes" the entire stream as a single token. This is useful for data like zip codes, ids, and some product names. http://lucene.apache.org/core/4_5_0/analyzers-common/org/apache/lucene/analysis/core/KeywordAnalyzer.html , for this to work you need to specify entire token without any change.
Then i used WhitespaceAnalyzer it worked and able to find matches for your PhraseQuery. No change to rest of the code. Let me know if this worked for you.
The reason search worked with MultiFieldQueryParser is you are forced to use the analyzer at the time of query which matches to your index. So in short , you need to make sure your index analyzer and query time analyzers are similar.

how to add data to ARRAYLIST

try {
final List<String> ar = new ArrayList<String>();
final PRIvariable pri = new PRIvariable();
final BufferedReader reader = new BufferedReader(
new InputStreamReader(new FileInputStream("C:/cdr2.csv")));
while (reader.ready()) {
final String line = reader.readLine();
final String[] values = line.split(",");
pri.dateText = values[2] + " " + values[4];
pri.count = pri.count + 1;
pri.sum = pri.sum + Integer.parseInt(values[7]);
System.out.println(pri.dateText + " " + pri.sum + " " + pri.count);
ar.add(pri);
}
final String[] columnNames = { "Date", "TOTAL", "COUNTS" };
final String[][] cells = new String[ar.size()][3];
for (int i = 0; i < ar.size(); i++) {
cells[i][0] = ((PRIvariable) ar.get(i)).dateText;
cells[i][1] = "" + ((PRIvariable) ar.get(i)).sum;
cells[i][2] = "" + ((PRIvariable) ar.get(i)).count;
}
table = new JTable(cells, columnNames);
table.setSize(400, 400);
table.setVisible(true);
final JScrollPane js = new JScrollPane();
js.setViewportView(table);
js.setSize(400, 400);
js.setVisible(true);
add(js, java.awt.BorderLayout.CENTER);
} catch (final Exception e) {
System.out.println(e);
}
This is my code. Here i want to Read text file and put that data to Jtable. But in this code it shows every row of the Jtable filled with same data that contain in arraylist(ar) last row. ( i think there is problem in my arraylist). How can i solve this......

The problem is with the variable pri. It has to be created inside the while loop.
Like this
String line = null;
while ((line = reader.readLine()) != null) {
PRIvariable pri = new PRIvariable();
String[] values = line.split(",");
pri.dateText = values[2] + " " + values[4];
pri.count = pri.count + 1;
pri.sum = pri.sum + Integer.parseInt(values[7]);
System.out.println(pri.dateText + " " + pri.sum + " " + pri.count);
ar.add(pri);
}
In your code your want to create a separate instance of PRIvariable for every line in the file, but you are creating only once instance of PRIvariable at the beginning then you are always using that instance by overriding the previous value.