We Keep Coding

out of memory when trying to select data from oracle database and insert it into sqlite

we get out of memory (the process go up and consumes %100 of ram) trying to select data and insert it into SQLite DB, the program written in java, and the data are so big, we even made pagination for it, it selects 100000 rows and inserts it into SQLite database, to figure out the problem, we commented out all line from the code that insert the data, and we saw that consumption of ram stop's at %6
package dbex;
import java.sql.*;
import java.util.ArrayList;
public class DBEX {
String url = "jdbc:oracle:thin:#192.168.120.46:1521:db";
String user = "dbuser";
String password = "dbpass";
ArrayList<Table> Tales = new ArrayList<>();
public static void main(String[] args) {
DBEX db = new DBEX();
db.CreateDB();
}
public int CountRecords(String table) {
int result = 0;
try {
Class.forName("oracle.jdbc.driver.OracleDriver");
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
Connection getCountCon = DriverManager.getConnection(url, user, password);
Statement CountSt = getCountCon.createStatement();
ResultSet countRs = CountSt.executeQuery("SELECT COUNT(*) as num FROM " + table);
while (countRs.next()) {
result = countRs.getInt("num");
}
} catch (Exception ex) {
}
return result;
}
public void dumpData() {
try {
int incrementRecord = 100000;
Class.forName("org.sqlite.JDBC");
Class.forName("oracle.jdbc.driver.OracleDriver");
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
Table table = null;
ArrayList<String> columns = new ArrayList<>();
Connection getDataCon = DriverManager.getConnection(url, user, password);
getDataCon.setAutoCommit(false);
Statement dataSt = getDataCon.createStatement();
dataSt.setFetchSize(incrementRecord);
ResultSet dataRs = null;
Connection LiteInsertCon = DriverManager.getConnection("jdbc:sqlite:" + "DBEX" + ".db");
LiteInsertCon.setAutoCommit(false);
PreparedStatement insertStatement = null;
int numRecord = 0;
for (int i = 0; i < Tales.size(); i++) {
table = Tales.get(i);
String tableName = table.name;
System.out.println("table::" + tableName);
System.out.println("Dumping data from " + tableName);
String InsertSQL = "INSERT INTO " + tableName + " ( ";
String vals = "VALUES (";
String selectSQL = "SELECT * FROM ( SELECT b.*, ROWNUM RN FROM ( SELECT ";
System.out.println("geting column name...");
columns = table.Columns;
for (int j = 0; j < columns.size(); j++) {
String columnName = columns.get(j);
if (j == 0) {
vals += " ? ";
InsertSQL += columnName + " ";
selectSQL += columnName + " ";
}else {
InsertSQL += " , " + columnName + " ";
vals += ", ? ";
selectSQL += " , " + columnName + " ";
}
}
System.out.println("Number of column of table " + tableName + " is " + columns.size());
vals += ")";
InsertSQL += ")" + vals;
selectSQL += "FROM " + tableName + " ORDER BY "+table.primary_key+" ASC ) b WHERE ROWNUM <= :TO ) WHERE RN > :FROM ";
System.out.println(selectSQL);
System.out.println(InsertSQL);
int parIndex = 0;
String colName = null;
String data = null;
numRecord = CountRecords(tableName);
int from = 0;
int to = incrementRecord;
if (to > numRecord) {
System.out.println("Table data is less than "+incrementRecord+" geting all data "+numRecord+" " );
dataRs = dataSt.executeQuery(selectSQL.replace(":TO", numRecord + "").replace(":FROM", "0"));
while (dataRs.next()) {
insertStatement = LiteInsertCon.prepareStatement(InsertSQL);
insertStatement.setFetchSize(incrementRecord);
for (; parIndex < columns.size(); parIndex++) {
colName = columns.get(parIndex);
data = dataRs.getString(colName);
if (data != null) {
insertStatement.setString(parIndex + 1, data);
} else {
insertStatement.setString(parIndex + 1, " ");
}
parIndex++;
}
parIndex = 0;
insertStatement.executeUpdate();
insertStatement = null;
}
} else {
while (numRecord >= to) {
System.out.println("page "+(to/incrementRecord)+" of "+(numRecord/incrementRecord) );
dataRs = dataSt.executeQuery(selectSQL.replace(":TO", to + "").replace(":FROM", from+""));
while (dataRs.next()) {
insertStatement = LiteInsertCon.prepareStatement(InsertSQL);
insertStatement.setFetchSize(incrementRecord);
for (; parIndex < columns.size(); parIndex++) {
colName = columns.get(parIndex);
data = dataRs.getString(colName);
if (data != null) {
insertStatement.setString(parIndex + 1, data);
} else {
insertStatement.setString(parIndex + 1, " ");
}
parIndex++;
}
parIndex = 0;
insertStatement.executeUpdate();
insertStatement = null;
}
dataRs=null;
from = to;
to += incrementRecord;
System.gc();
}
}
}
} catch (Exception ex) {
System.out.println(ex.toString());
}
}
class Table {
public String name;
public String primary_key;
ArrayList< String> Columns = new ArrayList<>();
public Table(String name, String primary_key, ArrayList< String> Columns) {
this.name = name;
this.primary_key = primary_key;
this.Columns = Columns;
}
}
public void CreateDB() {
try {
Class.forName("oracle.jdbc.driver.OracleDriver");
System.out.println("Oracle JDBC driver found");
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
Connection getTableCon = DriverManager.getConnection(url, user, password);
Connection getColumnCon = DriverManager.getConnection(url, user, password);
Connection getKeyCon = DriverManager.getConnection(url, user, password);
System.out.println("connected to database");
Statement getTableSt = getTableCon.createStatement();
PreparedStatement getColumnSt = getColumnCon.prepareStatement("SELECT DISTINCT column_name FROM all_tab_cols WHERE table_name = ? ");
PreparedStatement getKeySt = getKeyCon.prepareStatement("SELECT cols.table_name, cols.column_name, cols.position, cons.status, cons.owner FROM all_constraints cons, all_cons_columns cols WHERE cols.table_name = ? AND cons.constraint_type = 'P' AND cons.constraint_name = cols.constraint_name AND cons.owner = cols.owner ORDER BY cols.table_name, cols.position");
ResultSet getTableRs = getTableSt.executeQuery("SELECT * FROM all_tables WHERE OWNER not in('SYS','OUTLN','SYSTEM')");
ResultSet getColumnRs;
ResultSet getKeyRs;
Class.forName("org.sqlite.JDBC");
System.out.println("Sqlite JDBC driver found");
Connection LiteCreateTableCon = null;
LiteCreateTableCon = DriverManager.getConnection("jdbc:sqlite:" + "DBEX" + ".db");
Statement statement = LiteCreateTableCon.createStatement();
String keyName = "";
while (getTableRs.next()) {
String tableName = getTableRs.getString("table_name");
System.out.println("table: " + tableName);
getColumnSt.setString(1, tableName);
System.out.println("SELECT * FROM all_tab_cols WHERE table_name = " + tableName + "");
getColumnRs = getColumnSt.executeQuery();
String createTableSQL = "create table if not exists " + tableName + " ( ";
int index = 0;
ArrayList<String> Columns = new ArrayList<>();
while (getColumnRs.next()){
String columnName = getColumnRs.getString("column_name");
Columns.add(columnName);
System.out.println("column: " + columnName);
if (index == 0) {
createTableSQL += columnName + " string";
} else {
createTableSQL += " , " + columnName + " string";
}
index++;
}
createTableSQL += ")";
System.out.println(createTableSQL);
statement.executeUpdate(createTableSQL);
getKeySt.setString(1, tableName);
getKeyRs = getKeySt.executeQuery();
keyName = "1";
while (getKeyRs.next()) {
keyName = getKeyRs.getString("column_name");
}
Tales.add(new Table(tableName, keyName, Columns));
}
getTableCon.close();
getColumnCon.close();
getKeyCon.close();
System.out.println("All table created...");
}catch (Exception ex) {
System.out.println(ex.toString());
}
dumpData();
}
}

You need to issue/excute commit on SQLite after inserting every batch of records.
And try to come-up with suitable batch size (may be 100).

Trouble Updating a Table Row in Java

Connection conn = SqlConnection.getConnection("jdbc:mysql://localhost:3306/stocks");
Statement statement = conn.createStatement();
File path = new File("/Users/Zack/Desktop/JavaDB/BALANCESHEETS");
for(File file: path.listFiles()) {
if (file.isFile()) {
String fileName = file.getName();
String ticker = fileName.split("\\_")[0];
if (ticker.equals("ASB") || ticker.equals("FRC")) {
if (ticker.equals("ASB")) {
ticker = ticker + "PRD";
}
if (ticker.equals("FRC")) {
ticker = ticker + "PRD";
}
}
//CSVReader reader = new CSVReader(new FileReader(file));
//List entries = reader.readAll();
//ArrayList<String> entry = new ArrayList<String>();
Reader reader = new BufferedReader(new FileReader(file));
StringBuilder builder = new StringBuilder();
int c;
while ((c = reader.read()) != -1) {
builder.append((char) c);
}
String string = builder.toString();
ArrayList<String> stringResult = new ArrayList<String>();
if (string != null) {
String[] splitData = string.split("\\s*,\\s*|\\n");
for (int i = 0; i <splitData.length; i++) {
if (!(splitData[i] == null) || !(splitData[i].length() ==0)) {
stringResult.add(splitData[i].trim());
}
}
}
String columnName = null;
int yearCount = 0;
for (int i = 0; i < stringResult.size(); i++) {
int sL = stringResult.get(i).length();
for (int x = 0; x < sL; x++) {
if (Character.isLetter(stringResult.get(i).charAt(x))) {
yearCount = 0;
System.out.println("index: " + i);
columnName = stringResult.get(i);
columnName = columnName.replace(" ", "_");
System.out.println(columnName);
i++;
break;
}
}
yearCount++;
String value = stringResult.get(i);
System.out.println("Year: " + stringResult.get(yearCount) + " Value: " + value + " Stock: " + ticker + " Column: " + columnName );
if (!(columnName == null)) {
String writeValues = "INSERT INTO BalanceSheet (ticker, Year, " + columnName + ") "
+ "VALUE ('" + ticker + "','" + stringResult.get(yearCount) + "','" + value + "')";
String writeValues2 = "UPDATE BalanceSheet "
+ "SET ticker = '" + ticker + "', "
+ "Year = '" + stringResult.get(yearCount) + "', "
+ columnName + " = '" + value + "' "
+ "WHERE ticker = '" + ticker + "'";
statement.executeUpdate(writeValues2);
}
}
Towards the bottom of the code are two queries I tried, I'm trying to get all data organized by ticker and year into a table, "writeColumns" works but it's making a new row for every new "value" put into "columnName". My second attempt "writeColumns2" doesn't do anything.
I want to update the same row with a certain year for all values and then move onto the next year, then next ticker.

If I have understood your question correctly, you want to insert a row if it doesn't exists but update the values if it already does. You need to use ON DUPLICATE KEY UPDATE
String writeValues = "INSERT INTO BalanceSheet (ticker, Year, " + columnName + ") "
+ "VALUES (?,?,?) "
+"ON DUPLICATE KEY UPDATE " + columnName +"=?";
Statement statement = conn.prepareStatement(writeValues);
statement.setString(1,ticker);
statement.setString(2,stringResult.get(yearCount));
statement.setString(3, value);
This will solve your immidiate problem provided you create a UNIQUE index on ticker,year
However there are lot's of other issues here.
An update for each column - Currently you are doing an insert/update for each column on the table. What you are supposed to do is to insert update all the columns at one.
You are not using prepared statements addressed in my code sample
You shouldn't be doing this at all the best way to batch process data is to use MYSQL's built in LOAD DATA INFILE command. If your data is not in a format that can be easily imported into mysql, what your Java code can do is to transform it into a format that can be. Such a code will be a lot simpler and neater than what you have now

how to edit/update jtable cell values into database

am trying to edit and update jtable cells as in my code below. my problem is that a single row when updated all the other rows get the same values. i mean only one row is updated and all other a duplicated. can any one help with a good approach. thanks
int count = Table_purchase.getRowCount();
int col = Table_purchase.getColumnCount();
String pod_id[] = new String[count];
String po_id[] = new String[count];
String order_qty[] = new String[count];
String item_id[] = new String[count];
String unit_price[] = new String[count];
String recived_qty[] = new String[count];
String rejected_qty[] = new String[count];
for (int i = 0; i < count; i++) {
po_id[i] = Table_purchase.getValueAt(i,0).toString();
pod_id[i] = Table_purchase.getValueAt(i,1).toString();
order_qty[i] = Table_purchase.getValueAt(i,2).toString();
item_id[i] = Table_purchase.getValueAt(i,3).toString();
unit_price[i] = Table_purchase.getValueAt(i,4).toString();
recived_qty[i] = Table_purchase.getValueAt(i, 5).toString();
rejected_qty[i] = Table_purchase.getValueAt(i,6).toString();
try {
String sql = "update purchase.purchase_detail set pod_id='" + pod_id[i] + "',order_qty='" + order_qty[i] + "',item_id='" + item_id[i] + "', unit_price='" + unit_price[i] + "', recived_qty='" + recived_qty[i] + "',rejected_qty='" + rejected_qty[i] + "'where po_id= '" + po_id[i] + "'";
pst = conn.prepareStatement(sql);
pst.execute();
JOptionPane.showMessageDialog(null, "updated");
} catch (Exception e) {
JOptionPane.showMessageDialog(null, e);
}
}

Your sql statement does not contain a where-clause, hence all rows in the database table are updated for each iteration of the swing-table-rows, and in the end, all the database-rows will have the values from the last swing-table-row.
(And, use pst.setParameter (http://docs.oracle.com/javase/tutorial/jdbc/basics/prepared.html) and do not mix sql into gui-code.)

SQL-Exception in listing

I'm programming a small application in Java, a very basic database containing (fictional) student data. It works quiet okay so far, but I've encountered a problem when trying to make a list containing the curriculum of the university the students are frequenting. I want to list every possible degree, all of which have 6 semesters, along with the courses that can be taken in every category, how many hours per week they take and so on. The first bit works, but I encounter a problem when trying to fill the different categories with their respective courses over each semester. Heres the code (things I deem relevant to the problem are enclosed with two stars):
#Override
public List<List<String>> getStudyingplan(Studies s) throws ApplicationException {
List<List<String>> curriculum= new ArrayList<List<String>>();
ArrayList<String> line = new ArrayList<String>();
ArrayList<String> categories = new ArrayList<String>();
String currentCategory;
String name = s.getName();
String abb= s.getAbbreviation();
String category;
String categoryHeader= ("Mod E C P Cr");
line.add("Course of studies\n" + name + " (" + abb+ ")");
for (int i = 1; i <= 6; i++) {
line.add(i + ". Semester");
}
line.add("");
curriculum.add(line);
line= new ArrayList();
line.add("Category");
for (int i = 1; i <= 6; i++) {
line.add(categoryHeader);
}
line.add("Sum");
curriculum.add(line);
line= new ArrayList();
//Connect to database:
try {
Class.forName(driver).newInstance();
con = DriverManager.getConnection(database);
PreparedStatement giveCategories = con.prepareStatement("SELECT distinct k.* "
+ "FROM CATEGORY c, CURRICULUM cu, MODULE m "
+ "WHERE cu.SABB = ? AND cu.MABB = m.MABB AND m.KABB = c.KABB "
+ "ORDER BY c.INDEX");
**PreparedStatement giveModule = con.prepareStatement("SELECT distinct m.*" +
"FROM MODULE m, STUDIES st, CURRICULUM c" +
"WHERE st.SKABB = ? AND c.SEM = ? "
+ "AND c.MABB = m.MABB AND m.KABB = ?");**
giveCategories.setString(1, abb);
**giveModule.setString(1, abb);**
ResultSet categoriesGiven= giveCategories.executeQuery();
while (categoriesGiven.next()) {
categories.add(categoriesGiven.getString("NAME") + " (" +
categoriesGiven.getString("KABB") + ")");
}
**for (int i = 0; i < 6; i++) {
currentCategory = categories.get(i);
line.add(currentCategory);
giveModule.setString(3, currentCategory);
for (int j = 0; j < 6; j++) {
Integer seme = new Integer(j);
seme++;
giveModule.setString(2, seme.toString());
ResultSet current Modules = giveModules.executeQuery();
while (currentModules.next()) {
zeile.add(currentModules.getString("MABB"));
}
}
line.add("Sum");
curriculum.add(line);
line= new ArrayList();
}**
*A bunch of other stuff happens*
} catch (Exception e) {
System.out.println("getStudyingPlan has encountered an error.");
e.printStackTrace();
} finally {
if (con != null) {
try {
con.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
return curriculum;
}
The error I keep getting says:
java.sql.SQLSyntaxErrorException: Syntax error: Encountered "st" at line 1, column 73
so I'm assuming that there is an error in the PreparedStatement somewhere, but I can't seem to find it. Any help, even just a nudge, would be greatly appreciated while I'm trying to figure out the problem by myself.

Your query is running together, put a space after the c.
"FROM MODULE m, STUDIES st, CURRICULUM c" +
"WHERE st.SKABB = ? AND c.SEM = ? "
Should become
"FROM MODULE m, STUDIES st, CURRICULUM c " + // Note space
"WHERE st.SKABB = ? AND c.SEM = ? "

concatenation output problem (toString Array) - java

I am trying to display the output as "1(10) 2(23) 3(29)" but instead getting output as "1 2 3 (10)(23)(29)". I would be grateful if someone could have a look the code and possible help me. I don't want to use arraylist.
the code this
// int[] Groups = {10, 23, 29}; in the constructor
public String toString()
{
String tempStringB = "";
String tempStringA = " ";
String tempStringC = " ";
for (int x = 1; x<=3; x+=1)
{
tempStringB = tempStringB + x + " ";
}
for(int i = 0; i < Group.length;i++)
{
tempStringA = tempStringA + "(" + Groups[i] + ")";
}
tempStringC = tempStringB + tempStringA;
return tempStringC;
}

The problem is that you are appending all of the indices to one String and all of the elements to another, and then concatenating the two.
Instead, try building one String (and remember to use StringBuffer/StringBuilder, since it is more efficient than String concatenation):
public String toString() {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < Groups.length; i++) {
sb.append(i+1).append('(').append(Groups[i]).append(')');
}
return sb.toString();
}

You should use :
// int[] Groups = {10, 23, 29}; in the constructor
public String toString()
{
String tempStringB = "";
for(int i = 0; i < Group.length;i++)
{
tempStringB = (i==0?"":" ")+ tempStringB + (i+1) + " "+ "(" + Groups[i] + ")";
}
return tempStringB;
}
But by the way using a StringBuffer would be clever especially if your Group become bigger