We Keep Coding

Open CSV Performance to write data

I came through a link: https://github.com/hyee/OpenCSV which drastically improves the writing time of the JDBC ResultSet to CSV due to setAsyncMode, RESULT_FETCH_SIZE
//Extract ResultSet to CSV file, auto-compress if the fileName extension is ".zip" or ".gz"
//Returns number of records extracted
public int ResultSet2CSV(final ResultSet rs, final String fileName, final String header, final boolean aync) throws Exception {
try (CSVWriter writer = new CSVWriter(fileName)) {
//Define fetch size(default as 30000 rows), higher to be faster performance but takes more memory
ResultSetHelperService.RESULT_FETCH_SIZE=10000;
//Define MAX extract rows, -1 means unlimited.
ResultSetHelperService.MAX_FETCH_ROWS=20000;
writer.setAsyncMode(aync);
int result = writer.writeAll(rs, true);
return result - 1;
}
}
But the problem is I don't know how I can merge above into my requirement. As the link has many other classes involved which I am not sure what they do and if I even need it for my requirement. Still, I tried but it fails to compile whenever I enable 2 commented line code. Below is my code.
Any help on how I can achieve this will be greatly appreciated.
package test;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Date;
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelperService;
public class OpenCSVTest1
{
static Connection con =null;
static Statement stmt = null;
static ResultSet rs = null;
public static void main(String args[]) throws Exception
{
connection ();
retrieveData(con);
}
private static void connection() throws Exception
{
try
{
Class.forName("<jdbcdriver>");
con = DriverManager.getConnection("jdbc:","<username>","<pass>");
System.out.println("Connection successful");
}
catch (Exception e)
{
System.out.println("Exception while establishing sql connection");
throw e;
}
}
private static void retrieveData(Connection con) throws Exception
{
try
{
stmt=con.createStatement();
stmt = con.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
String query = "SELECT * FROM dbo.tablename";
rs=stmt.executeQuery(query);
CSVWriter writer = new CSVWriter(new BufferedWriter(new FileWriter("C:\\Data\\File1.csv")));
ResultSetHelperService service = new ResultSetHelperService();
/*** ResultSetHelperService.RESULT_FETCH_SIZE=10000; ***/ // to add
service.setDateTimeFormat("yyyy-MM-dd HH:mm:ss.SSS");
System.out.println("**** Started writing Data to CSV **** " + new Date());
writer.setResultService(service);
/*** writer.setAsyncMode(aync); ***/ // to add
int lines = writer.writeAll(rs, true, true, false);
writer.flush();
writer.close();
System.out.println("** OpenCSV -Completed writing the resultSet at " + new Date() + " Number of lines written to the file " + lines);
}
catch (Exception e)
{
System.out.println("Exception while retrieving data" );
e.printStackTrace();
throw e;
}
finally
{
rs.close();
stmt.close();
con.close();
}
}
}
UPDATE
I have updated my code. Right now code is writing complete resultset in CSV at once using writeAll method which is resulting in time consumption.
Now what I want to do is write resultset to CSV in batches as resultset's first column will always have dynamically generated via SELECT query Auto Increment column (Sqno) with values as (1,2,3..) So not sure how I can read result sets first column and split it accoridngly to write in CSV. may be HashMap might help, so I have also added resultset-tohashmap conversion code if required.
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelperService;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class OpenCSVTest1
{
static int fetchlimit_src = 100;
static Connection con =null;
static Statement stmt = null;
static ResultSet rs = null;
static String filename = "C:\\Data\\filename.csv";
static CSVWriter writer;
public static void main(String args[])
{
try
{
connection();
retrieveData(con);
}
catch(Exception e)
{
System.out.println(e);
}
}
private static void connection() throws Exception
{
try
{
Class.forName("<jdbcdriver>");
con = DriverManager.getConnection("jdbc:","<username>","<pass>");
System.out.println("Connection successful");
}
catch (Exception e)
{
System.out.println("Exception while establishing sql connection");
throw e;
}
}
private static void retrieveData(Connection con) throws Exception
{
try
{
stmt=con.createStatement();
String query = "SELECT ROWNUM AS Sqno, * FROM dbo.tablename "; // Oracle
// String query = "SELECT ROW_NUMBER() OVER(ORDER BY Id ASC) AS Sqno, * FROM dbo.tablename "; // SQLServer
System.out.println(query);
stmt = con.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
stmt.setFetchSize(fetchlimit_src);
System.out.println("**** Started querying src **** " + new Date());
rs=stmt.executeQuery(query);
System.out.println("**** Completing querying src **** " + new Date());
// resultset_List(rs); // If required store resultset(rs) to HashMap
writetoCSV(rs,filename);
/** How to write resultset to CSV in batches instead of writing all at once to speed up write performance ?
* Hint: resultset first column is Autoincrement [Sqno] (1,2,3...) which might help to split result in batches.
*
**/
}
catch (Exception e)
{
System.out.println("Exception while retrieving data" );
e.printStackTrace();
throw e;
}
finally
{
rs.close();
stmt.close();
con.close();
}
}
private static List<Map<String, Object>> resultset_List(ResultSet rs) throws SQLException
{
ResultSetMetaData md = rs.getMetaData();
int columns = md.getColumnCount();
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
while (rs.next())
{
Map<String, Object> row = new HashMap<String, Object>(columns);
for(int i = 1; i <= columns; ++i)
{
row.put(md.getColumnName(i), rs.getObject(i));
}
rows.add(row);
}
// System.out.println(rows.toString());
return rows;
}
private static void writetoCSV(ResultSet rs, String filename) throws Exception
{
try
{
writer = new CSVWriter(new BufferedWriter(new FileWriter(filename)));
ResultSetHelperService service = new ResultSetHelperService();
service.setDateTimeFormat("yyyy-MM-dd HH:mm:ss.SSS");
long batchlimit = 1000;
long Sqno = 1;
ResultSetMetaData rsmd = rs.getMetaData();
String columnname = rsmd.getColumnLabel(1); // To retrieve columns with labels (for example SELECT ROWNUM AS Sqno)
System.out.println("**** Started writing Data to CSV **** " + new Date());
writer.setResultService(service);
int lines = writer.writeAll(rs, true, true, false);
System.out.println("** OpenCSV -Completed writing the resultSet at " + new Date() + " Number of lines written to the file " + lines);
}
catch (Exception e)
{
System.out.println("Exception while writing data" );
e.printStackTrace();
throw e;
}
finally
{
writer.flush();
writer.close();
}
}
}

You should be able to use the OpenCSV sample, pretty much exactly as it is provided in the documentation. So, there should be no need for you to write any of your own batching logic.
I was able to write a 6 million record result set to a CSV file in about 10 seconds. To be clear -that was just the file-write time, not the DB data-fetch time - but I think that should be fast enough for your needs.
Here is your code, with adaptations for using OpenCSV based on its documented approach... But please see the warning at the end of my notes!
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelperService;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Date;
import java.text.SimpleDateFormat;
public class OpenCSVDemo {
static int fetchlimit_src = 100;
static Connection con = null;
static Statement stmt = null;
static ResultSet rs = null;
static String filename = "C:\\Data\\filename.csv";
public static void main(String args[]) {
try {
connection();
retrieveData(con);
} catch (Exception e) {
System.out.println(e);
}
}
private static void connection() throws Exception {
try {
final String jdbcDriver = "YOURS GOES HERE";
final String dbUrl = "YOURS GOES HERE";
final String user = "YOURS GOES HERE";
final String pass = "YOURS GOES HERE";
Class.forName(jdbcDriver);
con = DriverManager.getConnection(dbUrl, user, pass);
System.out.println("Connection successful");
} catch (Exception e) {
System.out.println("Exception while establishing sql connection");
throw e;
}
}
private static void retrieveData(Connection con) throws Exception {
try {
stmt = con.createStatement();
String query = "select title_id, primary_title from imdb.title";
System.out.println(query);
stmt = con.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
stmt.setFetchSize(fetchlimit_src);
System.out.println("**** Started querying src **** " + new Date());
rs = stmt.executeQuery(query);
System.out.println("**** Completing querying src **** " + new Date());
// resultset_List(rs); // If required store resultset(rs) to HashMap
System.out.println();
String timeStamp = new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss").format(new Date());
System.out.println("Started writing CSV: " + timeStamp);
writeToCsv(rs, filename, null, Boolean.FALSE);
timeStamp = new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss").format(new Date());
System.out.println("Finished writing CSV: " + timeStamp);
System.out.println();
} catch (Exception e) {
System.out.println("Exception while retrieving data");
e.printStackTrace();
throw e;
} finally {
rs.close();
stmt.close();
con.close();
}
}
public static int writeToCsv(final ResultSet rs, final String fileName,
final String header, final boolean aync) throws Exception {
try (CSVWriter writer = new CSVWriter(fileName)) {
//Define fetch size(default as 30000 rows), higher to be faster performance but takes more memory
ResultSetHelperService.RESULT_FETCH_SIZE = 1000;
//Define MAX extract rows, -1 means unlimited.
ResultSetHelperService.MAX_FETCH_ROWS = 2000;
writer.setAsyncMode(aync);
int result = writer.writeAll(rs, true);
return result - 1;
}
}
}
Points to note:
1) I used "async" set to false:
writeToCsv(rs, filename, null, Boolean.FALSE);
You may want to experiment with this and the other settings to see if they make any significant difference for you.
2) Regarding your comment "the link has many other classes involved": The OpenCSV library's entire JAR file needs to be included in your project, as does the related disruptor JAR:
opencsv.jar
disruptor-3.3.6.jar
To get the JAR files, go to the GitHub page, click on the green button, select the zip download, unzip the zip file, and look in the "OpenCSV-master\release" folder.
Add these two JARs to your project in the usual way (depends on how you build your project).
3) WARNING: This code runs OK when you use Oracle's Java 8 JDK/JRE. If you try to use OpenJDK (e.g. for Java 13 or similar) it will not run. This is because of some changes behind the scenes to hidden classes. If you are interested, there are more details here.
If you need to use an OpenJDK version of Java, you may therefore have better luck with the library on which this CSV library is based: see here.

not able to write onto a file

I have written this code to read from a table and write onto a file, but I am unable to write to a file. The file gets created, but its empty. I don't have a mysql problem tried to insert comments and debug. Code compiles fine, but has a problem in creating a file:
import java.io.File;
import java.io.FileNotFoundException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Formatter;
import java.util.Scanner;
import java.util.logging.Level;
import java.util.logging.Logger;
public class tabletocsv {
public static void main(String[] args) {
System.out.println("enter the table name");
Scanner input = new Scanner(System.in);
String table = input.next();
System.out.println("enter number of columns");
int columns = input.nextInt();
createcsv(table, columns);
System.out.print(" csv created");
displaycsv(table, columns);
}
static void displaycsv(String table, int count) {
Scanner file123;
try {
file123 = new Scanner(new File("/Users/Tannishk/Documents/csv/" + table + ".csv"));
//System.out.printf("reading");
while (file123.hasNext()) {
System.out.println("going inside");
for (int i = 1; i <= count; i++) {
String a = file123.next();
System.out.print(a + " ");
}
System.out.println();
}
} catch (FileNotFoundException ex) {
Logger.getLogger(tabletocsv.class.getName()).log(Level.SEVERE, null, ex);
}
}
static void createcsv(String table, int count) {
Formatter x;
Connection connection;
Statement st;
try {
x = new Formatter("/Users/Tannishk/Documents/csv/" + table + ".csv");
Class.forName("com.mysql.jdbc.Driver");
connection = DriverManager.getConnection("jdbc:mysql://localhost:3306/db", "root", "password");
st = connection.createStatement();
ResultSet rs = st.executeQuery("select * from " + table + ";");
while (rs.next()) {
for (int i = 1; i <= count; i++) {
String a = rs.getString(i);
x.format("%s ", a);
// System.out.print(a);
}
x.format("\n");
}
} catch (Exception e) {
} finally {
}
}
}

Try flushing and closing the formatter

I think you have missed to flush the file.
static void createcsv(String table,int count)
{
Formatter x = null;
Connection connection;
Statement st;
try{
x = new Formatter("/Users/Tannishk/Documents/csv/"+table+".csv");
Class.forName("com.mysql.jdbc.Driver");
connection = DriverManager.getConnection("jdbc:mysql://localhost:3306/db","root","password");
st = connection.createStatement();
ResultSet rs= st.executeQuery("select * from "+table+";");
while(rs.next())
{
for(int i=1;i<=count;i++)
{
String a = rs.getString(i);
x.format("%s ",a);
// System.out.print(a);
}
x.format("\n");
}
}
catch(Exception e)
{
}
finally
{
try {
x.flush();
x.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
Try this ...

You need to close the output stream that you're using to write to the file. For example if you're using a FileWriter object named fout to write to the file, you'll have to do fout.close() to actually have anything written to the file as otherwise it would just be in memory.

JDBC - Double Value Not Showing Up in MySQL DB

I have a program that is grabbing prices from web pages, and then finds the difference of that price from the last grabbed price, and then it sends that value to my MySQL Database.
Things to note before looking at the code:
The actual price, which is a double, does get sent and is entered into my Database correctly, although my priceChange variable is not. I have tried changing it to a BigDecimal and that made not changes.
PriceGrabber.java (sloppy right now, I know. I'm going to slim the code down eventually. once I get my core functions working.)
import java.math.BigDecimal;
import java.text.SimpleDateFormat;
import java.util.TimerTask;
import java.util.Date;
import org.jsoup.*;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import java.util.*;
import java.lang.*;
import java.net.URL;
import java.io.*;
public class PriceGrabber extends TimerTask{
#Override
public void run() {
try {
if(Coinbase.getLastPrice() == Variables.getCoinbase()){
System.out.println();
}else {
System.out.println("Price Change Detected in Coinbase!");
System.out.println("Price Change: " + (Variables.getCoinbase() - Coinbase.getLastPrice()));
Coinbase.setLastPrice(Variables.getCoinbase());
Variables.addPrice(Variables.getCoinbase(), "Coinbase", (Variables.getCoinbase() - Coinbase.getLastPrice()));
}
if(BTCE.getLastPrice() == Variables.getBTCE()){
System.out.println();
}else {
System.out.println("Price Change Detected in BTC-E!");
System.out.println("Price Change: " + (Variables.getBTCE() - BTCE.getLastPrice()));
BTCE.setLastPrice(Variables.getBTCE());
Variables.addPrice(Variables.getBTCE(), "BTC-e", (Variables.getBTCE() - BTCE.getLastPrice()));
}
if(BitStamp.getLastPrice() == Variables.getBitStamp()){
System.out.println();
}else {
System.out.println("Price Change Detected in BitStamp Market!");
System.out.println("Price Change: " + (Variables.getBitStamp() - BitStamp.getLastPrice()));
BitStamp.setLastPrice(Variables.getBitStamp());
Variables.addPrice(Variables.getBitStamp(), "Bitstamp", (Variables.getBitStamp() - BitStamp.getLastPrice()));
}
if(Bitfinext.getLastPrice() == Variables.getBitfinext()){
System.out.println();
}else {
System.out.println("Price Change Detected in Bitfinext!");
System.out.println("Price Change: " + (Variables.getBitfinext() - Bitfinext.getLastPrice()));
Bitfinext.setLastPrice(Variables.getBitfinext());
Variables.addPrice(Variables.getBitfinext(), "Bitfinext", (Variables.getBitfinext() - Bitfinext.getLastPrice()));
}
//Variables.printPrices();
} catch (IOException e) {
}
}
}
Variables.java
import java.io.*;
import java.math.BigDecimal;
import java.sql.*;
import java.util.TimerTask;
import java.util.Date;
import org.jsoup.*;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import java.util.*;
import java.lang.*;
import java.net.URL;
import java.io.*;
import java.net.MalformedURLException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Enumeration;
public class Variables {
public static final String DB_URL = "jdbc:mysql://URL TO DB";
// Database credentials
public static final String USER = "username";
public static final String PASS = "password";
//Web Elements
public static URL url = null;
public static Document page = null;
public static Element priceElement = null;
public static Document doc = null;
public static String price;
public static String priceString;
public static String timeStamp;
//Different Market URL's
public static String coinbaseURL = "https://bitcoinwisdom.com/markets/coinbase/btcusd";
public static String btceURL = "https://bitcoinwisdom.com/markets/btce/btcusd";
public static String bitstampURL = "https://bitcoinwisdom.com/markets/bitstamp/btcusd";
public static String bitfinextURL = "https://bitcoinwisdom.com/markets/bitfinex/btcusd";
//Sets the URL
public static void setURL(String siteURL) throws MalformedURLException, IOException{
url = new URL(siteURL);
}
public static Double getCoinbase() throws IOException{
try{
setURL(coinbaseURL);
page = Jsoup.parse(url, 5000);
if(page.select("div.green").first() == null){
priceElement = page.select("div.red").first();
}else{
priceElement = page.select("div.green").first();
}
priceString = priceElement.toString();
doc = Jsoup.parse(priceString);
price = doc.body().text();
}catch(IOException e) {
System.out.println("something went wrong");
System.out.println(e.getMessage());
}
return Double.parseDouble(price);
}
public static Double getBTCE() throws IOException{
try{
setURL(btceURL);
page = Jsoup.parse(url, 5000);
if(page.select("div.green").first() == null){
priceElement = page.select("div.red").first();
}else{
priceElement = page.select("div.green").first();
}
priceString = priceElement.toString();
doc = Jsoup.parse(priceString);
price = doc.body().text();
}catch(IOException e){
System.out.println("oops! Something went wrong");
System.out.println(e.getMessage());
}
return Double.parseDouble(price);
}
public static Double getBitStamp() throws IOException {
try {
setURL(bitstampURL);
page = Jsoup.parse(url, 5000);
if (page.select("div.green").first() == null) {
priceElement = page.select("div.red").first();
} else {
priceElement = page.select("div.green").first();
}
priceString = priceElement.toString();
doc = Jsoup.parse(priceString);
price = doc.body().text();
} catch (IOException e) {
System.out.println("oops! Something went wrong");
System.out.println(e.getMessage());
}
return Double.parseDouble(price);
}
public static Double getBitfinext() throws IOException {
try {
setURL(bitfinextURL);
page = Jsoup.parse(url, 5000);
if (page.select("div.green").first() == null) {
priceElement = page.select("div.red").first();
} else {
priceElement = page.select("div.green").first();
}
priceString = priceElement.toString();
doc = Jsoup.parse(priceString);
price = doc.body().text();
} catch (IOException e) {
System.out.println("oops! Something went wrong");
System.out.println(e.getMessage());
}
return Double.parseDouble(price);
}
//***********************************************************************************\\
//TODO:******************************************************************************\\
//TODO: SEARCH FOR QUEUE SYSTEM TO AVOID HAVING TO RECONNECT FOR EACH ADDITION \\
//TODO:******************************************************************************\\
//***********************************************************************************\\
public static void addPrice(Double price, String market, Double priceChange){
java.sql.Connection conn = null;
Statement stmt = null;
try{
//STEP 2: Register JDBC driver
Class.forName("com.mysql.jdbc.Driver");
//STEP 3: Open a connection
conn = DriverManager.getConnection(DB_URL, USER, PASS);
java.sql.Timestamp sqlDate = new java.sql.Timestamp(new java.util.Date().getTime());
String query = " insert into prices (market, price, pricechange, time_of_change)"
+ " values (?, ?, ?, ?)";
PreparedStatement preparedStmt = conn.prepareStatement(query);
preparedStmt.setString (1, market);
preparedStmt.setDouble (2, price);
preparedStmt.setDouble(3, priceChange);
preparedStmt.setTimestamp(4, sqlDate);
// execute the preparedstatement
preparedStmt.execute();
conn.close();
}catch(SQLException se){
//Handle errors for JDBC
se.printStackTrace();
}catch(Exception e){
//Handle errors for Class.forName
e.printStackTrace();
}finally{
//finally block used to close resources
try{
if(stmt!=null)
stmt.close();
}catch(SQLException se2){
}// nothing we can do
try{
if(conn!=null)
conn.close();
}catch(SQLException se){
se.printStackTrace();
}//end finally try
}//end try
}//end main
}
I'm going to show you the source of Coinbase.java, it is the exact same as the other classes (BTCE.java, Bitfinext.java, etc.) Just to save space
Coinbase.java (Same as other market classes)
/**
* Created by Sullivan4653 on 12/8/2014.
*/
public class Coinbase {
public static double lastPrice = 0.0;
public static void setLastPrice(double price){
lastPrice = price;
}
public static double getLastPrice(){
return lastPrice;
}
}
**Finally my MySQL Information: **
My MySQL Database/Table and columns all work except my priceChange one. The value-type is set to Double. Default value = null. That is the SAME as my price column, which seems to be working because I'm getting values of the prices. (AKA not 0 every time like my PriceChange column)
My question is, why is this double now showing in my MySQL Database?
I apologize for the length of the post but I want to make sure I don't leave out any details that may help someone understand the problem. I've been struggling with it for a few days now and can't find the error!
Thanks!

Stop using static classes, go deeper into Java if you already want to make stuff like this :)
Also, if you want to provide code to others, please use some nice naming conventions, instead of making everything look like default variables (IE coinbaseURL, which should be something like COINBASE_URL, based on your modifiers).
In your code you're declaring the old value with a new value, so you're overriding it (you cannot use it anymore, because it changed to the new one).
Here's what I mean:
Coinbase.setLastPrice(Variables.getCoinbase());
Variables.addPrice(Variables.getCoinbase(), "Coinbase", (Variables.getCoinbase() - Coinbase.getLastPrice()));
I've simplified your code and improved it, this should work.
import java.io.IOException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* #author JKetelaar
*/
public class PriceGrabber implements Runnable {
private Coinbase coinbase;
private Variables variables;
public PriceGrabber(){
this.coinbase = new Coinbase(0);
this.variables = new Variables();
this.variables.connect();
}
public static void main(String[] args){
ScheduledExecutorService scheduleTaskExecutor = Executors.newScheduledThreadPool(5);
scheduleTaskExecutor.scheduleAtFixedRate(new PriceGrabber(), 0, 10, TimeUnit.SECONDS);
}
#Override
public void run() {
try {
if (coinbase.getPrice() == variables.getCoinbase()) {
System.out.println("No changes found for Coinbase...");
} else {
System.out.println("Price Change detected in Coinbase!");
Double cbase = variables.getCoinbase();
Double cprice = coinbase.getPrice();
System.out.println("Old price: " + cprice + "\nNew price: " + cbase + "\nPrice Change: " + (cbase - cprice));
coinbase.setPrice(cbase);
variables.addPrice(cbase, "Coinbase", (cprice - cprice));
}
} catch (IOException ignored) {
}
}
}
-
public class Coinbase {
private double price = 0.0;
public Coinbase(double price) {
this.price = price;
}
public void setPrice(double price) {
this.price = price;
}
public double getPrice() {
return price;
}
}
-
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import java.io.IOException;
import java.net.URL;
import java.sql.*;
public class Variables {
public static final String DB_URL = "jdbc:mysql://160.153.49.168:3306/btcprogram";
// Database credentials
public static final String USER = "sully";
public static final String PASS = "asweq123e";
//Web Elements
public static URL url = null;
public static Document page = null;
public static Element priceElement = null;
public static Document doc = null;
public static String price;
public static String priceString;
public static String coinbaseURL = "https://bitcoinwisdom.com/markets/coinbase/btcusd";
private Connection connection;
//Sets the URL
public void setURL(String siteURL) throws IOException {
url = new URL(siteURL);
}
public Double getCoinbase() throws IOException {
try {
setURL(coinbaseURL);
page = Jsoup.parse(url, 5000);
if (page.select("div.green").first() == null) {
priceElement = page.select("div.red").first();
} else {
priceElement = page.select("div.green").first();
}
priceString = priceElement.toString();
doc = Jsoup.parse(priceString);
price = doc.body().text();
} catch (IOException e) {
System.out.println("something went wrong");
System.out.println(e.getMessage());
}
return Double.parseDouble(price);
}
public void addPrice(Double price, String market, Double priceChange) {
/**
* In your table set the time of change to a default value, so the table will do the time itself.
* Makes it easier for you and doesn't get complicated if you want to get others to insert prices.
*/
this.query("INSERT INTO prices (market, price, pricechange) VALUES (?, ?, ?)", new Object[]{
price, market, priceChange
});
}
public ResultSet query(String q, Object[] args) {
if (connection == null) {
System.out.println("No connection to the database.");
return null;
}
try {
PreparedStatement preparedStatement = connection.prepareStatement(q);
for (int i = 1; i <= args.length; i++) {
if (args[i] instanceof Double){
preparedStatement.setDouble(i, Double.parseDouble(String.valueOf(args[i])));
}else{
preparedStatement.setString(i, String.valueOf(args[i - 1]));
}
}
if (q.toLowerCase().startsWith("select")) {
return preparedStatement.executeQuery();
} else {
preparedStatement.executeUpdate();
}
} catch (SQLException e) {
e.printStackTrace();
}
return null;
}
public void connect() {
try {
Class.forName("com.mysql.jdbc.Driver");
connection = DriverManager
.getConnection("jdbc:mysql://160.153.49.168:3306/btcprogram?"
+ "user=sully&password=asweq123e");
if (!connection.isClosed()) {
System.out.println("Successfully connected to the database...\nReady for SQL queries!");
}
} catch (ClassNotFoundException | SQLException e) {
e.printStackTrace();
}
}
}

JDBC - Retrieve entire data as a List

I'm trying to build a class that retrieves each row from a MySQL table and store it to a List but my code doesn't compile and throws me an error
Error: Main method not found in class
MySQLAccountsDatabankReader,
please define the main method as: public static void main(String[]
args)
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
public class MySQLAccountsDatabankReader {
private static List<String[]> accounts;
public MySQLAccountsDatabankReader(String[] args) {
Connection con = null;
Statement st = null;
ResultSet rs = null;
String url = "jdbc:mysql://localhost:3306/test";
String user = "root";
String password = "mysql";
try {
con = DriverManager.getConnection(url, user, password);
st = con.createStatement();
rs = st.executeQuery("SELECT * FROM test.accounts;");
if (rs.next()) {
ArrayList<String[]> accounts = new ArrayList<String[]>();
rs = st.executeQuery("SELECT * FROM test.accounts;");
int columnCount = rs.getMetaData().getColumnCount();
while (rs.next()) {
String[] row = new String[columnCount];
for (int i = 0; i < columnCount; i++) {
row[i] = rs.getString(i + 1);
}
accounts.add(row);
}
}
} catch (SQLException ex) {
Logger lgr = Logger.getLogger(MySQLAccountsDatabankReader.class
.getName());
lgr.log(Level.SEVERE, ex.getMessage(), ex);
} finally {
try {
if (rs != null) {
rs.close();
}
if (st != null) {
st.close();
}
if (con != null) {
con.close();
}
} catch (SQLException ex) {
Logger lgr = Logger.getLogger(MySQLAccountsDatabankReader.class
.getName());
lgr.log(Level.WARNING, ex.getMessage(), ex);
}
}
return accounts;
}
}
Revision 0.2 - I've got the following compilation error - Exception in thread "main" java.lang.Error: Unresolved compilation problems:
accounts cannot be resolved
accounts cannot be resolved to a variable
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
public class MySQLAccountsDatabankReader {
public static void main(String[] args) {
MySQLAccountsDatabankReader reader = new MySQLAccountsDatabankReader();
List<String[]> accounts = reader.getAccounts();
}
public List<String[]> getAccounts() {
Connection con = null;
Statement st = null;
ResultSet rs = null;
String url = "jdbc:mysql://localhost:3306/test";
String user = "root";
String password = "mysql";
try {
con = DriverManager.getConnection(url, user, password);
st = con.createStatement();
rs = st.executeQuery("SELECT * FROM test.accounts;");
if (rs.next()) {
rs = st.executeQuery("SELECT * FROM test.accounts;");
int columnCount = rs.getMetaData().getColumnCount();
while (rs.next()) {
String[] row = new String[columnCount];
for (int i = 0; i < columnCount; i++) {
row[i] = rs.getString(i + 1);
}
accounts.add(row);
}
return accounts;
}
} catch (SQLException ex) {
Logger lgr = Logger.getLogger(MySQLAccountsDatabankReader.class
.getName());
lgr.log(Level.SEVERE, ex.getMessage(), ex);
} finally {
try {
if (rs != null) {
rs.close();
}
if (st != null) {
st.close();
}
if (con != null) {
con.close();
}
} catch (SQLException ex) {
Logger lgr = Logger.getLogger(MySQLAccountsDatabankReader.class
.getName());
lgr.log(Level.WARNING, ex.getMessage(), ex);
}
}
}
}

Compiler errors are usually a good indicator of what is wrong - you need a main method.
Also, non-declarative statements need to be in a method, constructor or static initializer rather than in the class code block. Place your JDBC code within a method. Here you need a method that will return accounts to match your return statement. Also you need to create an instance of List, e.g. ArrayList to return.
private List<String[]> getAccounts() {
List<String[]> accounts = new ArrayList<String[]>();
...
return accounts;
}
and add a main method entry point:
public static void main(String[] args) {
MySQLAccountsDatabankReader reader = new MySQLAccountsDatabankReader();
List<String[]> accounts = reader.getAccounts();
...
}

Add main method:
public static void main(String [] args) {
new MySQLAccountsDatabankReader(args);
}

Problems in Database Management - sqlite with Java (IDE: NetBeans)

I have some problems in managing the storage of data via query (NetBeans-> Java-> Sqlite):
1) I have a folder with some txt file, containing several lines of text (the files do not exceed 2 Kb)
2) The program opens the files in sequence, and stores each word in a table
3)When the program comes to analyze too much data (some time more than 40 files ore more then 82) returns the following error
Exception in thread "main" java.sql.SQLException: unable to open database file
at org.sqlite.DB.throwex (DB.java: 288)
at org.sqlite.DB.executeBatch (DB.java: 236)
at org.sqlite.PrepStmt.executeBatch (PrepStmt.java: 83)
The error is in int [] upCountsb = prepb.executeBatch();
Here the code:
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
public static void main(String[] args) throws ClassNotFoundException, SQLException, IOException, InterruptedException {
Class.forName("org.sqlite.JDBC");
Connection conn = DriverManager.getConnection("jdbc:sqlite:C:/Users/.../test.db");
Statement stmt;
stmt = conn.createStatement();
stmt.executeUpdate("DROP TABLE IF EXISTS words");
stmt.executeUpdate("CREATE TABLE words (words)");
String path_dir ="C:/Users/.../good";
File currentDIR = new File("C:/Users/.../good");
File files[]=currentDIR.listFiles();
String tmp="";
ArrayList app = new ArrayList();
//Search in DIR for Files
for( File f1 : files ){
String nameFile = f1.getName();
FileReader f = null;
BufferedReader fIN = null;
String s;
//Open the file xxx.txt
try{
f = new FileReader(path_dir+"/"+nameFile);
fIN = new BufferedReader(f);
s = fIN.readLine();
while(s != null) {
StringTokenizer st = new StringTokenizer(s);
while(st.hasMoreTokens()) {
String str = st.nextToken().toString().toLowerCase();
Pattern pattern =Pattern.compile("\\W", Pattern.MULTILINE);
String newAll = pattern.matcher(str).replaceAll("").trim();
tmp=newAll;
app.add(tmp); //Add all data in the ArrayList app
} // Close While 'hasMoreTokens'
s = fIN.readLine();
} //Close While on File
} //Close TRAY
catch (FileNotFoundException ex) {
ex.printStackTrace();
} catch (IOException ex) {
ex.printStackTrace();
}
f.close(); //Close FileReader
} //Close Scan DIR for FILE
//Add all data in the Tbl od Database
PreparedStatement prep = conn.prepareStatement("insert into words values (?);");
for (int z=0; z<app.size();z++){
prep.setString(1,app.get(z).toString().toLowerCase());
prep.addBatch();
conn.setAutoCommit(false);
prep.executeBatch(); ***//Here I get the error although i use int [] Count =prep.executeBatch();***
conn.setAutoCommit(true);
}
prep.close();
} //Close MAIN

You need to release the resources you are using once you are done with them. E.g. prepb.close() after executing the statement.
The same thing goes for your file handles.
Also, the point of batching is lost if you execute the statement for every insert.
Since your files are very small you might as well prepare all the data in memory before you persist it to a database.
package stackoverflow.wordanalysis;
import java.io.*;
import java.sql.*;
import java.util.*;
import java.util.regex.*;
public class WordFrequencyImporter {
public static void main(String[] args) throws Exception {
List<String> words = readWords("the-directory-from-which-to-read-the-files");
Class.forName("org.sqlite.JDBC");
Connection conn = DriverManager.getConnection("jdbc:sqlite:test.db");
try {
createWordsTable(conn);
persistWords(words, conn);
} finally {
conn.close();
}
}
private static void persistWords(List<String> words, Connection conn)
throws SQLException {
System.out.println("Persisting " + words.size() + " words");
PreparedStatement prep = conn
.prepareStatement("insert into words values (?);");
try {
for (String word : words) {
prep.setString(1, word);
prep.addBatch();
}
conn.setAutoCommit(false);
prep.executeBatch();
conn.setAutoCommit(true);
} finally {
prep.close();
}
}
private static void createWordsTable(Connection conn) throws SQLException {
Statement stmt = conn.createStatement();
try {
stmt.executeUpdate("DROP TABLE IF EXISTS words");
stmt.executeUpdate("CREATE TABLE words (words)");
} finally {
stmt.close();
}
}
private static List<String> readWords(String path_dir) throws IOException {
Pattern pattern = Pattern.compile("\\W", Pattern.MULTILINE);
List<String> words = new ArrayList<String>();
for (File file : new File(path_dir).listFiles()) {
BufferedReader reader = new BufferedReader(new FileReader(file));
System.out.println("Reading " + file);
try {
String s;
while ((s = reader.readLine()) != null) {
StringTokenizer st = new StringTokenizer(s);
while (st.hasMoreTokens()) {
String token = st.nextToken().toString().toLowerCase();
String word = pattern.matcher(token).replaceAll("")
.trim();
words.add(word);
}
}
} finally {
reader.close();
}
}
return words;
}
}