I am trying to map a particular character to show the full word so in this code below I have a translate method, a select method that is selecting from the remote database and update method that updates those column to my local database.
My remote database has a column called profile_ind this columns only has single chars B,C,F,P,S,W in the data, and once I do the update I want to be able to translate it to the full word only without the chars. B = BENIGN, C = CUSTOMER, F = FRAME, P = PPCOS, S = STANDARD, W = W-RED
And in my update method I want to call that translate method. cos_profile_type is the column on my local database that I want to update to either BENIGN, CUSTOMER, FRAME,PPCOS,STANDARD,W-RED depending on profile_ind column in the remote db. How can I implement it ? how can i call that translate method in the update method when I set the ? in that statement.
public static String translate(String indicator) throws Exception {
if (indicator == null || indicator.length() == 0) return "";
if (indicator.equals("B")) return "BENIGN";
if (indicator.equals("C")) return "CUSTOMER";
if (indicator.equals("F")) return "FRAME";
if (indicator.equals("P")) return "PPCOS";
if (indicator.equals("S")) return "STANDARD";
if (indicator.equals("W")) return "W-RED";
System.out.println(indicator);
return "";
}
public static void selectRecordsIcore() throws SQLException {
Connection dbConnection = null;
PreparedStatement preparedStatement = null;
Statement statement = null;
java.util.Date date= new java.util.Date();
String selectTableSQL = "SELECT profile_id, ingress_flag, egress_flag, ce_ingress_flag, ce_egress_flag, profile_ind from COS_PROFILE"
+ " WHERE profile_id >= ? AND profile_id <= ? ;";
try {
dbConnection = getInformixConnection(); //connects to ICORE database
System.out.println("ICORE Select Statement: " + selectTableSQL);
//Gets the max profile_id record
statement = dbConnection.createStatement();
ResultSet r = statement.executeQuery("SELECT max(profile_id) AS rowcount FROM COS_PROFILE");
r.next();
int maxCount = r.getInt("rowcount");
System.out.println("COS_PROFILE table has " + maxCount + " row(s).");
preparedStatement = dbConnection.prepareStatement(selectTableSQL);
preparedStatement.setInt(1, 1);
preparedStatement.setInt(2, maxCount);
// execute select SQL statement
rs = preparedStatement.executeQuery();
updateRecordIntoBids();
} catch (SQLException e) {
System.out.println(e.getMessage());
} finally {
System.out.println("inside the finally block ICORE " + new Timestamp(date.getTime()));
if (rs != null) {
rs.close();
}
if (statement != null) {
statement.close();
System.out.println("ICORE Statement Connection is closed");
}
if (preparedStatement != null) {
preparedStatement.close();
System.out.println("ICORE PreparedStatement Connection is closed");
}
if (dbConnection != null) {
dbConnection.close();
System.out.println("Database ICORE Connection is closed");
}
}
}
private static void updateRecordIntoBids() throws SQLException {
Connection dbConnection = null;
PreparedStatement preparedStatement = null;
Statement statement = null;
dbConnection = getOracleConnection(); //connects to BIDS database
java.util.Date date= new java.util.Date();
//Gets the max profile_id record
statement = dbConnection.createStatement();
dbConnection.setAutoCommit(false);
ResultSet r = statement.executeQuery("SELECT count (*) AS rowcount FROM traffic_profile_temp");
r.next();
int maxCount = r.getInt("rowcount");
System.out.println("Traffic_profile_temp table before update has " + maxCount + " row(s).");
String updateTableSQL =
"UPDATE traffic_profile_temp SET pe_ingress_flag = ?, "
+ " pe_egress_flag = ?,"
+ " ce_ingress_flag = ?,"
+ " ce_egress_flag = ?, "
+ " cos_profile_type = ? "
+ " WHERE traffic_profile_id = ? ";
// execute update SQL statement
System.out.println("BIDS Update Statement: " + updateTableSQL);
preparedStatement = dbConnection.prepareStatement(updateTableSQL);
System.out.println("Updating Bids Database in process ...");
try {
int rowCount = 0;
int expectedId = 1;
int first = 0;
while (rs.next()) {
String ingressflag = rs.getString("ingress_flag"); //BIDS column is pe_ingress_flag
String egressflag = rs.getString("egress_flag"); //BIDS column is pe_egress_flag
String ceingressflag = rs.getString("ce_ingress_flag"); //BIDS column is ce_ingress_flag
String ceegressflag = rs.getString("ce_egress_flag"); //BIDS column is ce_egress_flag
String profileind = rs.getString("profile_ind"); //BIDS column is cos_profile_type
int profileid = rs.getInt("profile_id"); //BIDS column is traffic_profile_id
preparedStatement.setString(1, ingressflag);
preparedStatement.setString(2, egressflag);
preparedStatement.setString(3, ceingressflag);
preparedStatement.setString(4, ceegressflag);
preparedStatement.setString(5, profileind);
preparedStatement.setInt(6, profileid);
preparedStatement.addBatch();
rowCount++;
if(expectedId == profileid){
if(first == 0){
first = expectedId-1;
}
}
if(expectedId != profileid){
if(first > 0){
System.out.println ("Profile id "+first+" to "+(expectedId-1)+" update.");
first = 0;
}
System.out.println ("Profile id "+expectedId+" to "+(profileid-1)+" missing.");
expectedId = profileid;
}
expectedId++;
}
if(first > 0){
System.out.println("Profile id "+first+" to "+(expectedId-1)+" update.");
first = 0;
}
preparedStatement.executeBatch();
dbConnection.commit();
System.out.println("Record(s) Updated : " + rowCount + new Timestamp(date.getTime()));
} catch (SQLException e) {
System.out.println("Update records Failed!! " + e.getMessage());
} finally {
System.out.println("inside the finally block BIDS" + " timestamp " + new Timestamp(date.getTime()));
if (statement != null) {
statement.close();
System.out.println("BIDS Statement Connection is closed");
}
if (preparedStatement != null) {
preparedStatement.close();
System.out.println("BIDS PreparedStatement Connection is closed");
}
if (dbConnection != null) {
dbConnection.close();
System.out.println("Database BIDS Connection is closed");
}
}
}
Related
we get out of memory (the process go up and consumes %100 of ram) trying to select data and insert it into SQLite DB, the program written in java, and the data are so big, we even made pagination for it, it selects 100000 rows and inserts it into SQLite database, to figure out the problem, we commented out all line from the code that insert the data, and we saw that consumption of ram stop's at %6
package dbex;
import java.sql.*;
import java.util.ArrayList;
public class DBEX {
String url = "jdbc:oracle:thin:#192.168.120.46:1521:db";
String user = "dbuser";
String password = "dbpass";
ArrayList<Table> Tales = new ArrayList<>();
public static void main(String[] args) {
DBEX db = new DBEX();
db.CreateDB();
}
public int CountRecords(String table) {
int result = 0;
try {
Class.forName("oracle.jdbc.driver.OracleDriver");
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
Connection getCountCon = DriverManager.getConnection(url, user, password);
Statement CountSt = getCountCon.createStatement();
ResultSet countRs = CountSt.executeQuery("SELECT COUNT(*) as num FROM " + table);
while (countRs.next()) {
result = countRs.getInt("num");
}
} catch (Exception ex) {
}
return result;
}
public void dumpData() {
try {
int incrementRecord = 100000;
Class.forName("org.sqlite.JDBC");
Class.forName("oracle.jdbc.driver.OracleDriver");
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
Table table = null;
ArrayList<String> columns = new ArrayList<>();
Connection getDataCon = DriverManager.getConnection(url, user, password);
getDataCon.setAutoCommit(false);
Statement dataSt = getDataCon.createStatement();
dataSt.setFetchSize(incrementRecord);
ResultSet dataRs = null;
Connection LiteInsertCon = DriverManager.getConnection("jdbc:sqlite:" + "DBEX" + ".db");
LiteInsertCon.setAutoCommit(false);
PreparedStatement insertStatement = null;
int numRecord = 0;
for (int i = 0; i < Tales.size(); i++) {
table = Tales.get(i);
String tableName = table.name;
System.out.println("table::" + tableName);
System.out.println("Dumping data from " + tableName);
String InsertSQL = "INSERT INTO " + tableName + " ( ";
String vals = "VALUES (";
String selectSQL = "SELECT * FROM ( SELECT b.*, ROWNUM RN FROM ( SELECT ";
System.out.println("geting column name...");
columns = table.Columns;
for (int j = 0; j < columns.size(); j++) {
String columnName = columns.get(j);
if (j == 0) {
vals += " ? ";
InsertSQL += columnName + " ";
selectSQL += columnName + " ";
}else {
InsertSQL += " , " + columnName + " ";
vals += ", ? ";
selectSQL += " , " + columnName + " ";
}
}
System.out.println("Number of column of table " + tableName + " is " + columns.size());
vals += ")";
InsertSQL += ")" + vals;
selectSQL += "FROM " + tableName + " ORDER BY "+table.primary_key+" ASC ) b WHERE ROWNUM <= :TO ) WHERE RN > :FROM ";
System.out.println(selectSQL);
System.out.println(InsertSQL);
int parIndex = 0;
String colName = null;
String data = null;
numRecord = CountRecords(tableName);
int from = 0;
int to = incrementRecord;
if (to > numRecord) {
System.out.println("Table data is less than "+incrementRecord+" geting all data "+numRecord+" " );
dataRs = dataSt.executeQuery(selectSQL.replace(":TO", numRecord + "").replace(":FROM", "0"));
while (dataRs.next()) {
insertStatement = LiteInsertCon.prepareStatement(InsertSQL);
insertStatement.setFetchSize(incrementRecord);
for (; parIndex < columns.size(); parIndex++) {
colName = columns.get(parIndex);
data = dataRs.getString(colName);
if (data != null) {
insertStatement.setString(parIndex + 1, data);
} else {
insertStatement.setString(parIndex + 1, " ");
}
parIndex++;
}
parIndex = 0;
insertStatement.executeUpdate();
insertStatement = null;
}
} else {
while (numRecord >= to) {
System.out.println("page "+(to/incrementRecord)+" of "+(numRecord/incrementRecord) );
dataRs = dataSt.executeQuery(selectSQL.replace(":TO", to + "").replace(":FROM", from+""));
while (dataRs.next()) {
insertStatement = LiteInsertCon.prepareStatement(InsertSQL);
insertStatement.setFetchSize(incrementRecord);
for (; parIndex < columns.size(); parIndex++) {
colName = columns.get(parIndex);
data = dataRs.getString(colName);
if (data != null) {
insertStatement.setString(parIndex + 1, data);
} else {
insertStatement.setString(parIndex + 1, " ");
}
parIndex++;
}
parIndex = 0;
insertStatement.executeUpdate();
insertStatement = null;
}
dataRs=null;
from = to;
to += incrementRecord;
System.gc();
}
}
}
} catch (Exception ex) {
System.out.println(ex.toString());
}
}
class Table {
public String name;
public String primary_key;
ArrayList< String> Columns = new ArrayList<>();
public Table(String name, String primary_key, ArrayList< String> Columns) {
this.name = name;
this.primary_key = primary_key;
this.Columns = Columns;
}
}
public void CreateDB() {
try {
Class.forName("oracle.jdbc.driver.OracleDriver");
System.out.println("Oracle JDBC driver found");
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
Connection getTableCon = DriverManager.getConnection(url, user, password);
Connection getColumnCon = DriverManager.getConnection(url, user, password);
Connection getKeyCon = DriverManager.getConnection(url, user, password);
System.out.println("connected to database");
Statement getTableSt = getTableCon.createStatement();
PreparedStatement getColumnSt = getColumnCon.prepareStatement("SELECT DISTINCT column_name FROM all_tab_cols WHERE table_name = ? ");
PreparedStatement getKeySt = getKeyCon.prepareStatement("SELECT cols.table_name, cols.column_name, cols.position, cons.status, cons.owner FROM all_constraints cons, all_cons_columns cols WHERE cols.table_name = ? AND cons.constraint_type = 'P' AND cons.constraint_name = cols.constraint_name AND cons.owner = cols.owner ORDER BY cols.table_name, cols.position");
ResultSet getTableRs = getTableSt.executeQuery("SELECT * FROM all_tables WHERE OWNER not in('SYS','OUTLN','SYSTEM')");
ResultSet getColumnRs;
ResultSet getKeyRs;
Class.forName("org.sqlite.JDBC");
System.out.println("Sqlite JDBC driver found");
Connection LiteCreateTableCon = null;
LiteCreateTableCon = DriverManager.getConnection("jdbc:sqlite:" + "DBEX" + ".db");
Statement statement = LiteCreateTableCon.createStatement();
String keyName = "";
while (getTableRs.next()) {
String tableName = getTableRs.getString("table_name");
System.out.println("table: " + tableName);
getColumnSt.setString(1, tableName);
System.out.println("SELECT * FROM all_tab_cols WHERE table_name = " + tableName + "");
getColumnRs = getColumnSt.executeQuery();
String createTableSQL = "create table if not exists " + tableName + " ( ";
int index = 0;
ArrayList<String> Columns = new ArrayList<>();
while (getColumnRs.next()){
String columnName = getColumnRs.getString("column_name");
Columns.add(columnName);
System.out.println("column: " + columnName);
if (index == 0) {
createTableSQL += columnName + " string";
} else {
createTableSQL += " , " + columnName + " string";
}
index++;
}
createTableSQL += ")";
System.out.println(createTableSQL);
statement.executeUpdate(createTableSQL);
getKeySt.setString(1, tableName);
getKeyRs = getKeySt.executeQuery();
keyName = "1";
while (getKeyRs.next()) {
keyName = getKeyRs.getString("column_name");
}
Tales.add(new Table(tableName, keyName, Columns));
}
getTableCon.close();
getColumnCon.close();
getKeyCon.close();
System.out.println("All table created...");
}catch (Exception ex) {
System.out.println(ex.toString());
}
dumpData();
}
}
You need to issue/excute commit on SQLite after inserting every batch of records.
And try to come-up with suitable batch size (may be 100).
How do I tell my java program to retrieve the next matching record into my default table model.
Below is my home work so far.using jTable tb1 and default table model dtm is compulsory for me.
private void Show_My_LettersActionPerformed(java.awt.event.ActionEvent evt) {
Connection conn = null;
Statement stmt = null;
try {
Class.forName("com.mysql.jdbc.Driver");
System.out.println("Connecting to a selected database...");
conn = DriverManager.getConnection(url, "root", "root");
System.out.println("Connected database successfully...");
System.out.println("Creating statement...");
stmt = conn.createStatement();
String sql = "SELECT * from LCI where SUB_ID = '" + SUB_ID_.getText() + "' AND L_DATE = '" + DATE.getText() + "'";
ResultSet rs1, rs2, rs3, rs4, rs5, rs6, rs7;
rs1=j.getData("select COUNT(*) from LCI");
try (ResultSet rs = stmt.executeQuery(sql)) {
DefaultTableModel dtm = (DefaultTableModel) tb1.getModel();
while (rs.next()) {
String L_ID_ = rs.getString("L_ID");
String L_DATE_ = rs.getString("L_DATE");
String heading = rs.getString("HEADING");
String sub_id = rs.getString("SUB_ID");
System.out.print("ID: " + L_ID_);
System.out.print(", Letter date: " + L_DATE_);
System.out.print(", Heading " + heading);
System.out.println(", Subject ID " + sub_id);
/* This gives the correct out put when debug is done.
But the below code doesn't retrive the full out put.
It gives only the very first record matching with the user inputs*/
Vector v = new Vector();
Vector v1 = new Vector();
Vector v2 = new Vector();
Vector v3 = new Vector();
JOptionPane.showMessageDialog(this, "Done");
dtm.getColumnName(1);
v.addElement(rs.getString(1));
dtm.addColumn(v);
dtm.getColumnName(3);
v1.addElement(rs.getString(3));
dtm.addColumn(v1);
dtm.getColumnName(10);
v2.addElement(rs.getString(10));
dtm.addColumn(v2);
// stmt.executeQuery("SELECT * FROM LCI WHERE L_ID IN(SELECT (L_ID + 1)FROM LCI WHERE L_DATE = '"+DATE.getText()+"'");
dtm.addRow(v3);
//stmt.executeQuery("SELECT * FROM LCI WHERE L_ID IN(SELECT (L_ID '"+(+1)+"')FROM LCI WHERE L_DATE = '"+DATE.getText()+"'");
}
}
} catch (SQLException se) {
se.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (stmt != null) {
conn.close();
}
} catch (SQLException se) {
se.printStackTrace();
}
try {
if (conn != null) {
conn.close();
}
} catch (SQLException se) {
se.printStackTrace();
}
}// TODO add your handling code here:
}
I have a sqlite db file created by other program, and checked everything is fine.
Then after doing select query to get some data, some of the row disappear after this process. I tried to use prepareStatement and though it worked but this remained.
my code
private ForecastTableItem selectItemPrepareStatement(String tableName, String columnName, String name) {
ForecastTableItem item = null;
Connection conn = null;
PreparedStatement pstmt = null;
ResultSet rs = null;
try {
Class.forName("org.sqlite.JDBC");
conn = DriverManager.getConnection("jdbc:sqlite:" + dbLocation);
System.out.println("Selecting item from tableName: "+tableName + " of col: "+columnName + " : "+name);
String query = "SELECT * FROM " + tableName + " WHERE " + columnName + "=? COLLATE NOCASE";
pstmt = conn.prepareStatement(query);
pstmt.setString(1, name);
rs = pstmt.executeQuery();
if (rs.next()) {
if (tableName.equalsIgnoreCase("mainTable")) {
item = new ForecastTableItem();
item.setId(rs.getInt("Id"));
item.setTitle(rs.getString("title"));
item.setLink(rs.getString("link").toLowerCase());
item.setPositionType(rs.getString("positionType"));
item.setPackageName(rs.getString("packageName"));
item.setCsvFilePath(rs.getString("csvFilePath"));
item.setSubpackageName(rs.getString("subpackageName"));
item.setTimeFrame(rs.getString("timeFrame"));
item.setForecastDate(rs.getString("forecastDate"));
item.setTargetDate(rs.getString("targetDate"));
}
}
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
} finally{
if(pstmt != null) try{ pstmt.close();} catch(SQLException e){};
if(rs != null) try{ rs.close();} catch(SQLException e){};
if(conn != null) try{ conn.close();} catch(SQLException e){};
}
return item;
}
after COLLATE NOCASE use semi-column
"SELECT * FROM " + tableName + " WHERE " + columnName + "= ? COLLATE NOCASE;";
I currently have a very large file which contains a few million lines of entries, and want them inserted into a database. The connection established from java to SQL works as I have tried inserting the data singularly and it works, however, when I switched to using executeBatch and addBatch, it seems to loop though but not populating anything into my database.
Code is as follows:
import java.io.BufferedReader;
import java.io.FileReader;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
import java.sql.SQLException;
public class fedOrganiser6 {
private static String directory = "C:\\Users\\x\\Desktop\\Files\\";
private static String file = "combined.fed";
private static String mapperValue = "";
public static void main(String[] args) throws Exception {
Connection conn = null;
try {
BufferedReader mapper = new BufferedReader(new FileReader(directory + file));
String dbURL = "jdbc:sqlserver://localhost\\SQLExpress;database=TIMESTAMP_ORGANISER;integratedSecurity=true";
String user = "sa";
String pass = "password";
conn = DriverManager.getConnection(dbURL, user, pass);
if (conn != null) {
DatabaseMetaData dm = (DatabaseMetaData) conn.getMetaData();
System.out.println("Driver name: " + dm.getDriverName());
System.out.println("Driver version: " + dm.getDriverVersion());
System.out.println("Product name: " + dm.getDatabaseProductName());
System.out.println("Product version: " + dm.getDatabaseProductVersion());
System.out.println("clearing database");
conn.createStatement().executeUpdate("truncate table TimestampsStorage");
System.out.println("bulk insert into database");
System.out.println("complete");
int i = 0;
int records = 0;
String query = "INSERT INTO TimestampsStorage " + "values(" + "'" + mapperValue.toString() + "'"+ ")";
conn.prepareStatement(query);
for (mapperValue = mapper.readLine(); mapperValue != null; mapperValue = mapper.readLine()) {
i++;
records++;
System.out.println("Batching " + records + " records...");
conn.createStatement().addBatch(query);
if (i == 100000) {
conn.createStatement().executeBatch();
i = 0;
}
}
}
conn.createStatement().executeBatch();
conn.createStatement().close();
System.out.print("Done");
} catch (SQLException ex) {
ex.printStackTrace();
} finally {
try {
if (conn != null && !conn.isClosed()) {
conn.close();
}
} catch (SQLException ex) {
ex.printStackTrace();
}
}
}
}
createStatement() creates a new statement object, so you're execute a different statement than the one you're batching on. You should create the PreparedStatement once, add several batches to it, and then execute on the same object:
String query = "INSERT INTO TimestampsStorage VALUES (?)";
PreparedStatement ps = conn.prepareStatement(query);
for (mapperValue = mapper.readLine();
mapperValue != null;
mapperValue = mapper.readLine()) {
i++;
records++;
System.out.println("Batching " + records + " records...");
ps.setString(1, mapperValue);
ps.addBatch();
if (i == 100000) {
ps.executeBatch();
i = 0;
}
}
I think you are a bit mistaken on how batch processing for JDBC works.
You are creating a new Statement each time you call conn.createStatement().
Instead, you will want to use a PreparedStatement. First, change your query to include a ? where you want your values to go.
String query = "INSERT INTO TimestampsStorage VALUES(?)";
Then, when you call conn.prepareStatement(query), store the returned PreparedStatement.
PreparedStatement ps = conn.prepareStatement(query);
This PreparedStatement will then 'remember' your query, and you can simply change the values you want where the ? is on each iteration of your loop.
ps.setString(1, mapperValue);
The setString method will take your mapperValue and use it instead of the first ? it finds in your query (since you pass in the index 1).
Then, instead of calling conn.createStatement().addBatch(), you would call ps.addBatch().
Then, outside of your loop, you can call ps.executeBatch(). (There is no need to call this inside your loop, so you can remove your if (i == 100000) condition).
Finally, if you are using Java 7+, you can use a try with resources, so that you don't need to worry about closing the PreparedStatement or Connection in a finally block.
Here is what your end result should look like.
String query = "INSERT INTO TimestampsStorage VALUES (?)";
try (Connection con = DriverManager.getConnection(dbURL, user, pass); PreparedStatement ps = con.prepareStatement(query);) {
for (mapperValue = mapper.readLine(); mapperValue != null; mapperValue = mapper.readLine()) {
records++;
ps.setString(1, mapperValue);
ps.addBatch();
}
System.out.println("Executing batch of " + records + " records...");
ps.executeBatch();
} catch (SQLException ex) {
//handle exception
}
you are throwing away the prepared statement
String query = "INSERT INTO TimestampsStorage VALUES (?)";
PreparedStatement statement = conn.prepareStatement(query);
for (mapperValue = mapper.readLine(); mapperValue != null; mapperValue = mapper.readLine()) {
i++;
records++;
System.out.println("Batching " + records + " records...");
statement.setString(1,mapperValue);
statement.addBatch();
if (i == 100000) {
statement.executeBatch();
i = 0;
}
I have a problem, I cannot get my list from MS SQL Server Database.
This is my code:
public List<Location> showLocations(String nmlocation) {
List<Location> location = new ArrayList<Location>();
try {
String sql = "SELECT deskripsi FROM T_Place WHERE ID_Place='%" + nmlocation + "%'";
PreparedStatement statement = (PreparedStatement) dm.logON().prepareStatement(sql);
boolean result = statement.execute();
if (result) {
ResultSet rs = statement.getResultSet();
int i = 0;
while (rs.next())
{
Location loc = new Location(rs.getString("deskripsi"));
System.out.println(loc);
location.add(loc);
i++;
}
rs.close();
}
} catch (SQLException sqle) {
System.out.println("Error on Listing Location Data showLocation Status : " + sqle.getSQLState()
+ " \n Error Message : " + sqle.getMessage()
+ " \n Error State : " + sqle.toString());
}
return location;
}
But it shows empty resultset. How to fix it? thanks in advance.