I want to print the result set obtained from JDBC connection and want it to print each result set in aloop.So my code is below:
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.sql.*;
import java.util.*;
public class zone {
List<model> listOfBlogs;
public zone() {
Connection conn = null;
listOfBlogs = new ArrayList<model>();
// connect to the database
conn = connectToDatabaseOrDie();
// get the data
populateListOfTopics(conn, listOfBlogs);
//print the data
printTopics(listOfBlogs);
}
public List<model> printTopics(List<model> listOfBlogs) {
Iterator<model> it = listOfBlogs.iterator();
while (it.hasNext()) {
model blog = (model) it.next();
System.out.println(blog.day + " " + blog.hour + " " + blog.zone_id);
}
return listOfBlogs;
}
private void populateListOfTopics(Connection conn, List<model> listOfBlogs) {
try {
for (int j = 9; j < 10; j++) {
for (int i = 10; i < 12; i++) {
String sql = "SELECT day,hour,zone_id FROM public.zone_block_table where day=? and hour=?";
PreparedStatement pstmt = conn.prepareStatement(sql);
pstmt.setInt(1, j);
pstmt.setInt(2, i);
ResultSet rs = pstmt.executeQuery();
while (rs.next()) {
model blog = new model();
blog.day = rs.getInt("day");
blog.hour = rs.getInt("hour");
blog.zone_id = rs.getInt("zone_id");
listOfBlogs.add(blog);
}
convertToCsv(rs, i, j);
rs.close();
pstmt.close();
}
}
} catch (SQLException se) {
System.err.println("Threw a SQLException creating the list of state.");
System.err.println(se.getMessage());
} catch (Exception e) {
System.out.println("Err");
e.printStackTrace();
}
}
private Connection connectToDatabaseOrDie() {
Connection conn = null;
try {
Class.forName("org.postgresql.Driver");
String url = "jdbc:postgresql://localhost:5432/mvc_data_base";
conn = DriverManager.getConnection(url, "postgres", "122333");
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
}
return conn;
}
public void convertToCsv(ResultSet rs, int k, int l) throws SQLException, FileNotFoundException {
PrintWriter csvWriter = new PrintWriter(new File("state_" + k + "_" + l + ".csv"));
ResultSetMetaData meta = rs.getMetaData();
int numberOfColumns = meta.getColumnCount();
String dataHeaders = "\"" + meta.getColumnName(1) + "\"";
for (int i = 2; i < numberOfColumns + 1; i++) {
dataHeaders += ",\"" + meta.getColumnName(i) + "\"";
}
csvWriter.println(dataHeaders);
while (rs.next()) {
String row = "\"" + rs.getString(1) + "\"";
for (int i = 2; i < numberOfColumns + 1; i++) {
row += ",\"" + rs.getString(i) + "\"";
}
csvWriter.println(row);
}
csvWriter.close();
}
}
But the resultset is printing perfectly.But I could not find any csv files generated in any of the destination.Any help is appreciated.
There is a Direct method to create CSV file from resultset using opencsv. Try that
CSVWriter writer = new CSVWriter(new FileWriter(csvtemp), ','
,CSVWriter.NO_QUOTE_CHARACTER , CSVWriter.NO_ESCAPE_CHARACTER , "\r\n");
writer.writeAll(myResultSet, true);
writer.close();
Related
we get out of memory (the process go up and consumes %100 of ram) trying to select data and insert it into SQLite DB, the program written in java, and the data are so big, we even made pagination for it, it selects 100000 rows and inserts it into SQLite database, to figure out the problem, we commented out all line from the code that insert the data, and we saw that consumption of ram stop's at %6
package dbex;
import java.sql.*;
import java.util.ArrayList;
public class DBEX {
String url = "jdbc:oracle:thin:#192.168.120.46:1521:db";
String user = "dbuser";
String password = "dbpass";
ArrayList<Table> Tales = new ArrayList<>();
public static void main(String[] args) {
DBEX db = new DBEX();
db.CreateDB();
}
public int CountRecords(String table) {
int result = 0;
try {
Class.forName("oracle.jdbc.driver.OracleDriver");
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
Connection getCountCon = DriverManager.getConnection(url, user, password);
Statement CountSt = getCountCon.createStatement();
ResultSet countRs = CountSt.executeQuery("SELECT COUNT(*) as num FROM " + table);
while (countRs.next()) {
result = countRs.getInt("num");
}
} catch (Exception ex) {
}
return result;
}
public void dumpData() {
try {
int incrementRecord = 100000;
Class.forName("org.sqlite.JDBC");
Class.forName("oracle.jdbc.driver.OracleDriver");
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
Table table = null;
ArrayList<String> columns = new ArrayList<>();
Connection getDataCon = DriverManager.getConnection(url, user, password);
getDataCon.setAutoCommit(false);
Statement dataSt = getDataCon.createStatement();
dataSt.setFetchSize(incrementRecord);
ResultSet dataRs = null;
Connection LiteInsertCon = DriverManager.getConnection("jdbc:sqlite:" + "DBEX" + ".db");
LiteInsertCon.setAutoCommit(false);
PreparedStatement insertStatement = null;
int numRecord = 0;
for (int i = 0; i < Tales.size(); i++) {
table = Tales.get(i);
String tableName = table.name;
System.out.println("table::" + tableName);
System.out.println("Dumping data from " + tableName);
String InsertSQL = "INSERT INTO " + tableName + " ( ";
String vals = "VALUES (";
String selectSQL = "SELECT * FROM ( SELECT b.*, ROWNUM RN FROM ( SELECT ";
System.out.println("geting column name...");
columns = table.Columns;
for (int j = 0; j < columns.size(); j++) {
String columnName = columns.get(j);
if (j == 0) {
vals += " ? ";
InsertSQL += columnName + " ";
selectSQL += columnName + " ";
}else {
InsertSQL += " , " + columnName + " ";
vals += ", ? ";
selectSQL += " , " + columnName + " ";
}
}
System.out.println("Number of column of table " + tableName + " is " + columns.size());
vals += ")";
InsertSQL += ")" + vals;
selectSQL += "FROM " + tableName + " ORDER BY "+table.primary_key+" ASC ) b WHERE ROWNUM <= :TO ) WHERE RN > :FROM ";
System.out.println(selectSQL);
System.out.println(InsertSQL);
int parIndex = 0;
String colName = null;
String data = null;
numRecord = CountRecords(tableName);
int from = 0;
int to = incrementRecord;
if (to > numRecord) {
System.out.println("Table data is less than "+incrementRecord+" geting all data "+numRecord+" " );
dataRs = dataSt.executeQuery(selectSQL.replace(":TO", numRecord + "").replace(":FROM", "0"));
while (dataRs.next()) {
insertStatement = LiteInsertCon.prepareStatement(InsertSQL);
insertStatement.setFetchSize(incrementRecord);
for (; parIndex < columns.size(); parIndex++) {
colName = columns.get(parIndex);
data = dataRs.getString(colName);
if (data != null) {
insertStatement.setString(parIndex + 1, data);
} else {
insertStatement.setString(parIndex + 1, " ");
}
parIndex++;
}
parIndex = 0;
insertStatement.executeUpdate();
insertStatement = null;
}
} else {
while (numRecord >= to) {
System.out.println("page "+(to/incrementRecord)+" of "+(numRecord/incrementRecord) );
dataRs = dataSt.executeQuery(selectSQL.replace(":TO", to + "").replace(":FROM", from+""));
while (dataRs.next()) {
insertStatement = LiteInsertCon.prepareStatement(InsertSQL);
insertStatement.setFetchSize(incrementRecord);
for (; parIndex < columns.size(); parIndex++) {
colName = columns.get(parIndex);
data = dataRs.getString(colName);
if (data != null) {
insertStatement.setString(parIndex + 1, data);
} else {
insertStatement.setString(parIndex + 1, " ");
}
parIndex++;
}
parIndex = 0;
insertStatement.executeUpdate();
insertStatement = null;
}
dataRs=null;
from = to;
to += incrementRecord;
System.gc();
}
}
}
} catch (Exception ex) {
System.out.println(ex.toString());
}
}
class Table {
public String name;
public String primary_key;
ArrayList< String> Columns = new ArrayList<>();
public Table(String name, String primary_key, ArrayList< String> Columns) {
this.name = name;
this.primary_key = primary_key;
this.Columns = Columns;
}
}
public void CreateDB() {
try {
Class.forName("oracle.jdbc.driver.OracleDriver");
System.out.println("Oracle JDBC driver found");
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
Connection getTableCon = DriverManager.getConnection(url, user, password);
Connection getColumnCon = DriverManager.getConnection(url, user, password);
Connection getKeyCon = DriverManager.getConnection(url, user, password);
System.out.println("connected to database");
Statement getTableSt = getTableCon.createStatement();
PreparedStatement getColumnSt = getColumnCon.prepareStatement("SELECT DISTINCT column_name FROM all_tab_cols WHERE table_name = ? ");
PreparedStatement getKeySt = getKeyCon.prepareStatement("SELECT cols.table_name, cols.column_name, cols.position, cons.status, cons.owner FROM all_constraints cons, all_cons_columns cols WHERE cols.table_name = ? AND cons.constraint_type = 'P' AND cons.constraint_name = cols.constraint_name AND cons.owner = cols.owner ORDER BY cols.table_name, cols.position");
ResultSet getTableRs = getTableSt.executeQuery("SELECT * FROM all_tables WHERE OWNER not in('SYS','OUTLN','SYSTEM')");
ResultSet getColumnRs;
ResultSet getKeyRs;
Class.forName("org.sqlite.JDBC");
System.out.println("Sqlite JDBC driver found");
Connection LiteCreateTableCon = null;
LiteCreateTableCon = DriverManager.getConnection("jdbc:sqlite:" + "DBEX" + ".db");
Statement statement = LiteCreateTableCon.createStatement();
String keyName = "";
while (getTableRs.next()) {
String tableName = getTableRs.getString("table_name");
System.out.println("table: " + tableName);
getColumnSt.setString(1, tableName);
System.out.println("SELECT * FROM all_tab_cols WHERE table_name = " + tableName + "");
getColumnRs = getColumnSt.executeQuery();
String createTableSQL = "create table if not exists " + tableName + " ( ";
int index = 0;
ArrayList<String> Columns = new ArrayList<>();
while (getColumnRs.next()){
String columnName = getColumnRs.getString("column_name");
Columns.add(columnName);
System.out.println("column: " + columnName);
if (index == 0) {
createTableSQL += columnName + " string";
} else {
createTableSQL += " , " + columnName + " string";
}
index++;
}
createTableSQL += ")";
System.out.println(createTableSQL);
statement.executeUpdate(createTableSQL);
getKeySt.setString(1, tableName);
getKeyRs = getKeySt.executeQuery();
keyName = "1";
while (getKeyRs.next()) {
keyName = getKeyRs.getString("column_name");
}
Tales.add(new Table(tableName, keyName, Columns));
}
getTableCon.close();
getColumnCon.close();
getKeyCon.close();
System.out.println("All table created...");
}catch (Exception ex) {
System.out.println(ex.toString());
}
dumpData();
}
}
You need to issue/excute commit on SQLite after inserting every batch of records.
And try to come-up with suitable batch size (may be 100).
I have written the below Java code, which is executing fine in order to export the results in xlsx file from Oracle DB. But can someone help me to improve the performance of this code by which this below code can extract the result very fast.
Below is the Java Code which I have tried as of now; But it is slow while exporting result as compared to excel export from SQL Developer
public class FilesFromFolder {
private Workbook writeWorkbook;
public void ExportService(DBConnection con) {
writeWorkbook = new XSSFWorkbook();
Sheet desSheet = writeWorkbook.createSheet("Data");
Statement stmt = null;
ResultSet rs = null;
int columnsNumber = 0;
ResultSetMetaData rsmd = null;
FileOutputStream fileOut = null;
Connection cntn = null;
String filePath = "C:\\Users\\Desktop\\OracleExport";
File files = new File(filePath);
File[] file = files.listFiles();
String fileNameWithOutExt = null;
if (file != null) {
for (int i = 0; i < file.length; i++) {
if (file[i].isFile()) {
String tempFilename = file[i].getName();
fileNameWithOutExt = tempFilename.replaceFirst("[.][^.]+$", "");
File fileTemp = file[i];
try {
String fileContent = FileUtils.readFileToString(fileTemp, "UTF8");
// System.out.println(fileContent);
cntn = con.getConnection();
stmt = cntn.createStatement();
rs = stmt.executeQuery(fileContent);
rsmd = rs.getMetaData();
columnsNumber = rsmd.getColumnCount();
Row desRow1 = desSheet.createRow(0);
for (int col = 0; col < columnsNumber; col++) {
Cell newpath = desRow1.createCell(col);
newpath.setCellValue(rsmd.getColumnLabel(col + 1));
}
while (rs.next()) {
System.out.println("Row number -->" + rs.getRow());
Row desRow = desSheet.createRow(rs.getRow());
for (int col = 0; col < columnsNumber; col++) {
Cell newpath = desRow.createCell(col);
newpath.setCellValue(rs.getString(col + 1));
}
String outputFile = "C:\\Users\\Desktop\\OracleExport\\" + fileNameWithOutExt
+ ".xlsx";
fileOut = new FileOutputStream(outputFile);
writeWorkbook.write(fileOut);
}
System.out.println(fileNameWithOutExt + " export complete");
} catch (IOException e) {
e.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
} finally {
if (fileOut!= null) {
try {
fileOut.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (cntn != null) {
con.closeConnection();
}
}
}
}
}
}
}
Quick questions...
I'm trying to make a Dynamic JTree but I can't get to put every database I have into one single node for each one. This is my code so far:
jTree2 = new javax.swing.JTree();
try {
String DSN = "jdbc:mysql://localhost";
String user = "root";
String password = "";
conexion = DriverManager.getConnection(DSN, user, password);
}
catch(Exception e) {
System.out.println("ERROR");
}
try {
sentencia = conexion.createStatement(
ResultSet.TYPE_SCROLL_INSENSITIVE,
ResultSet.CONCUR_READ_ONLY);
}
catch(Exception e) {
System.out.println("ERROR2");
}
try {
String hi = "";
ResultSet rs1 = conexion.getMetaData().getCatalogs();
ResultSetMetaData rsmd = rs1.getMetaData();
int columnCount = rsmd.getColumnCount();
while (rs1.next()) {
for (int i = 1; i <= columnCount; i++ ) {
hi = hi + rs1.getString(i) + ", ";
}
//for
String sb = hi.substring(0, hi.length()-2);
jTree2.setModel(new FileSystemModel(new File(sb)));
}
}
catch(Exception ae) {
System.out.println("ERROR3");
}
jScrollPane3.setViewportView(jTree2);
And the result I get is this:
Every database is splitted by a "," but I want them to be on a single node for each one. Any help?
This should do it for you:
DefaultMutableTreeNode parent = new DefaultMutableTreeNode("Databases", true);
while (rs1.next()) {
for (int i = 1; i <= columnCount; i++) {
DefaultMutableTreeNode node = new DefaultMutableTreeNode(rs1.getString(i), true);
parent.add(node);
}
jTree2.setModel(new DefaultTreeModel(parent));
}
so i found this code on the internet, basically what supposedly it can do is backup all the tables from a db, my question is on this line:
res = st.executeQuery("select * from xcms." + tableName);
i get the following excpetion exception: SQLException information
what does xcms. stands for? what else can i put here?
res = st.executeQuery("select * from " + tableName);
btw if i erase xcms. and put it like this ^, i can save only the first table not all the tables, thx
the sourcecode webpage:
https://gauravmutreja.wordpress.com/2011/10/13/exporting-your-database-to-csv-file-in-java/#comment-210
public static void main(String[] args) {
Connection con = null;
String url = "jdbc:mysql://localhost:3306/";
String db = "gg";
String driver = "com.mysql.jdbc.Driver";
String user = "root";
String pass = "";
FileWriter fw;
try {
Class.forName(driver);
con = DriverManager.getConnection(url + db, user, pass);
Statement st = con.createStatement();
ResultSet res = st.executeQuery("SELECT table_name FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = 'gg'");
List<String> tableNameList = new ArrayList<String>();
while (res.next()) {
tableNameList.add(res.getString(1));
}
String filename = "C:\\Users\\Angel Silva\\Documents";
for (String tableName : tableNameList) {
int k = 0;
int j = 1;
System.out.println(tableName);
List<String> columnsNameList = new ArrayList<String>();
res = st.executeQuery("select * from " + tableName);
int columnCount = getColumnCount(res);
try {
fw = new FileWriter("C:\\Users\\Angel Silva\\Documents\\popo1121.csv");
for (int i = 1; i <= columnCount; i++) {
fw.append(res.getMetaData().getColumnName(i));
fw.append(",");
}
fw.append(System.getProperty("line.separator"));
while (res.next()) {
for (int i = 1; i <= columnCount; i++) {
if (res.getObject(i) != null) {
String data = res.getObject(i).toString();
fw.append(data);
fw.append(",");
} else {
String data = "null";
fw.append(data);
fw.append(",");
}
}
fw.append(System.getProperty("line.separator"));
}
fw.flush();
fw.close();
} catch (IOException ioe) {
ioe.printStackTrace();
}
}
con.close();
} catch (ClassNotFoundException cnfe) {
System.err.println("Could not load JDBC driver");
cnfe.printStackTrace();
}catch (SQLException sqle) {System.err.println("SQLException information");}
}
public static int getRowCount(ResultSet res) throws SQLException {
res.last();
int numberOfRows = res.getRow();
res.beforeFirst();
return numberOfRows;
}
public static int getColumnCount(ResultSet res) throws SQLException {
return res.getMetaData().getColumnCount();
}
}
This is what I used:
public void sqlToCSV (String query, String filename){
log.info("creating csv file: " + filename);
try {
FileWriter fw = new FileWriter(filename + ".csv");
if(conn.isClosed()) st = conn.createStatement();
ResultSet rs = st.executeQuery(query);
int cols = rs.getMetaData().getColumnCount();
for(int i = 1; i <= cols; i ++){
fw.append(rs.getMetaData().getColumnLabel(i));
if(i < cols) fw.append(',');
else fw.append('\n');
}
while (rs.next()) {
for(int i = 1; i <= cols; i ++){
fw.append(rs.getString(i));
if(i < cols) fw.append(',');
}
fw.append('\n');
}
fw.flush();
fw.close();
log.info("CSV File is created successfully.");
conn.close();
} catch (Exception e) {
log.fatal(e);
}
}
The xms stands for the Database name, in your Connection in the java program you already are establishing the connection to the Database:
(DriverManager.getConnection(url + db, user, pass);
The string db is the name of the DB to connect to.
So no need to have the xms. .just for example use this query:
"SELECT * FROM "+tableName+";"
This is executed in the database you are connected to, for example ggin your code.
For writting the CSV file chillyfacts.com/export-mysql-table-csv-file-using-java/ may help!
SELECT * FROM <MENTION_TABLE_NAME_HERE> INTO OUTFILE <FILE_NAME> FIELDS TERMINATED BY ',';
Example :
SELECT * FROM topic INTO OUTFILE 'D:\5.csv' FIELDS TERMINATED BY ',';
use opencsv dependency to export SQL data to CSV using minimal lines of code.
import com.opencsv.CSVWriter;
import java.io.FileWriter;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class CsvWriter {
public static void main(String[] args) throws Exception {
CSVWriter writer = new CSVWriter(new FileWriter("filename.csv"), '\t');
Boolean includeHeaders = true;
Statement statement = null;
ResultSet myResultSet = null;
Connection connection = null;
try {
connection = //make database connection here
if (connection != null) {
statement = connection.createStatement();
myResultSet = statement.executeQuery("your sql query goes here");
writer.writeAll(myResultSet, includeHeaders);
}
} catch (SQLException throwables) {
throwables.printStackTrace();
}
}
}
This is my code for database class method to get data in resultset and return an general array in this I have only one field in database.
Problem : I am not able to get the array abc[] and its contents and when I return it also says arrayIndexoutOfBond error "AWT-EventQueue-0" java.lang.ArrayIndexOutOfBoundsException: 1
public String[] getTableContents(String tableName) {
ResultSet results = null;
String[] abc = null;
int a = 0;
try {
System.out.println(conn);
stmt = conn.createStatement();
results = stmt.executeQuery("select * from " + tableName);
ResultSetMetaData rsmd = results.getMetaData();
// int numberCols = rsmd.getColumnCount();
for (int i = 1; i <= rsmd.getColumnCount(); i++) {
// print Column Names
System.out.print(rsmd.getColumnLabel(i) + "\t\t");
}
System.out.println("\n----------------------------------------");
while (results.next()) {
System.out.println(results.getString(2) + " 1");
String em = (results.getString(2));
System.out.println(em + " 2");
abc = em.split(" ");
System.out.println(abc + " 3");
}
results.close();
stmt.close();
} catch (SQLException suresh) {
System.out.println(suresh);
}
System.out.println(abc + " 4");
return abc;
}