We Keep Coding

Open CSV Performance to write data

I came through a link: https://github.com/hyee/OpenCSV which drastically improves the writing time of the JDBC ResultSet to CSV due to setAsyncMode, RESULT_FETCH_SIZE
//Extract ResultSet to CSV file, auto-compress if the fileName extension is ".zip" or ".gz"
//Returns number of records extracted
public int ResultSet2CSV(final ResultSet rs, final String fileName, final String header, final boolean aync) throws Exception {
try (CSVWriter writer = new CSVWriter(fileName)) {
//Define fetch size(default as 30000 rows), higher to be faster performance but takes more memory
ResultSetHelperService.RESULT_FETCH_SIZE=10000;
//Define MAX extract rows, -1 means unlimited.
ResultSetHelperService.MAX_FETCH_ROWS=20000;
writer.setAsyncMode(aync);
int result = writer.writeAll(rs, true);
return result - 1;
}
}
But the problem is I don't know how I can merge above into my requirement. As the link has many other classes involved which I am not sure what they do and if I even need it for my requirement. Still, I tried but it fails to compile whenever I enable 2 commented line code. Below is my code.
Any help on how I can achieve this will be greatly appreciated.
package test;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Date;
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelperService;
public class OpenCSVTest1
{
static Connection con =null;
static Statement stmt = null;
static ResultSet rs = null;
public static void main(String args[]) throws Exception
{
connection ();
retrieveData(con);
}
private static void connection() throws Exception
{
try
{
Class.forName("<jdbcdriver>");
con = DriverManager.getConnection("jdbc:","<username>","<pass>");
System.out.println("Connection successful");
}
catch (Exception e)
{
System.out.println("Exception while establishing sql connection");
throw e;
}
}
private static void retrieveData(Connection con) throws Exception
{
try
{
stmt=con.createStatement();
stmt = con.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
String query = "SELECT * FROM dbo.tablename";
rs=stmt.executeQuery(query);
CSVWriter writer = new CSVWriter(new BufferedWriter(new FileWriter("C:\\Data\\File1.csv")));
ResultSetHelperService service = new ResultSetHelperService();
/*** ResultSetHelperService.RESULT_FETCH_SIZE=10000; ***/ // to add
service.setDateTimeFormat("yyyy-MM-dd HH:mm:ss.SSS");
System.out.println("**** Started writing Data to CSV **** " + new Date());
writer.setResultService(service);
/*** writer.setAsyncMode(aync); ***/ // to add
int lines = writer.writeAll(rs, true, true, false);
writer.flush();
writer.close();
System.out.println("** OpenCSV -Completed writing the resultSet at " + new Date() + " Number of lines written to the file " + lines);
}
catch (Exception e)
{
System.out.println("Exception while retrieving data" );
e.printStackTrace();
throw e;
}
finally
{
rs.close();
stmt.close();
con.close();
}
}
}
UPDATE
I have updated my code. Right now code is writing complete resultset in CSV at once using writeAll method which is resulting in time consumption.
Now what I want to do is write resultset to CSV in batches as resultset's first column will always have dynamically generated via SELECT query Auto Increment column (Sqno) with values as (1,2,3..) So not sure how I can read result sets first column and split it accoridngly to write in CSV. may be HashMap might help, so I have also added resultset-tohashmap conversion code if required.
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelperService;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class OpenCSVTest1
{
static int fetchlimit_src = 100;
static Connection con =null;
static Statement stmt = null;
static ResultSet rs = null;
static String filename = "C:\\Data\\filename.csv";
static CSVWriter writer;
public static void main(String args[])
{
try
{
connection();
retrieveData(con);
}
catch(Exception e)
{
System.out.println(e);
}
}
private static void connection() throws Exception
{
try
{
Class.forName("<jdbcdriver>");
con = DriverManager.getConnection("jdbc:","<username>","<pass>");
System.out.println("Connection successful");
}
catch (Exception e)
{
System.out.println("Exception while establishing sql connection");
throw e;
}
}
private static void retrieveData(Connection con) throws Exception
{
try
{
stmt=con.createStatement();
String query = "SELECT ROWNUM AS Sqno, * FROM dbo.tablename "; // Oracle
// String query = "SELECT ROW_NUMBER() OVER(ORDER BY Id ASC) AS Sqno, * FROM dbo.tablename "; // SQLServer
System.out.println(query);
stmt = con.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
stmt.setFetchSize(fetchlimit_src);
System.out.println("**** Started querying src **** " + new Date());
rs=stmt.executeQuery(query);
System.out.println("**** Completing querying src **** " + new Date());
// resultset_List(rs); // If required store resultset(rs) to HashMap
writetoCSV(rs,filename);
/** How to write resultset to CSV in batches instead of writing all at once to speed up write performance ?
* Hint: resultset first column is Autoincrement [Sqno] (1,2,3...) which might help to split result in batches.
*
**/
}
catch (Exception e)
{
System.out.println("Exception while retrieving data" );
e.printStackTrace();
throw e;
}
finally
{
rs.close();
stmt.close();
con.close();
}
}
private static List<Map<String, Object>> resultset_List(ResultSet rs) throws SQLException
{
ResultSetMetaData md = rs.getMetaData();
int columns = md.getColumnCount();
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
while (rs.next())
{
Map<String, Object> row = new HashMap<String, Object>(columns);
for(int i = 1; i <= columns; ++i)
{
row.put(md.getColumnName(i), rs.getObject(i));
}
rows.add(row);
}
// System.out.println(rows.toString());
return rows;
}
private static void writetoCSV(ResultSet rs, String filename) throws Exception
{
try
{
writer = new CSVWriter(new BufferedWriter(new FileWriter(filename)));
ResultSetHelperService service = new ResultSetHelperService();
service.setDateTimeFormat("yyyy-MM-dd HH:mm:ss.SSS");
long batchlimit = 1000;
long Sqno = 1;
ResultSetMetaData rsmd = rs.getMetaData();
String columnname = rsmd.getColumnLabel(1); // To retrieve columns with labels (for example SELECT ROWNUM AS Sqno)
System.out.println("**** Started writing Data to CSV **** " + new Date());
writer.setResultService(service);
int lines = writer.writeAll(rs, true, true, false);
System.out.println("** OpenCSV -Completed writing the resultSet at " + new Date() + " Number of lines written to the file " + lines);
}
catch (Exception e)
{
System.out.println("Exception while writing data" );
e.printStackTrace();
throw e;
}
finally
{
writer.flush();
writer.close();
}
}
}

You should be able to use the OpenCSV sample, pretty much exactly as it is provided in the documentation. So, there should be no need for you to write any of your own batching logic.
I was able to write a 6 million record result set to a CSV file in about 10 seconds. To be clear -that was just the file-write time, not the DB data-fetch time - but I think that should be fast enough for your needs.
Here is your code, with adaptations for using OpenCSV based on its documented approach... But please see the warning at the end of my notes!
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelperService;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Date;
import java.text.SimpleDateFormat;
public class OpenCSVDemo {
static int fetchlimit_src = 100;
static Connection con = null;
static Statement stmt = null;
static ResultSet rs = null;
static String filename = "C:\\Data\\filename.csv";
public static void main(String args[]) {
try {
connection();
retrieveData(con);
} catch (Exception e) {
System.out.println(e);
}
}
private static void connection() throws Exception {
try {
final String jdbcDriver = "YOURS GOES HERE";
final String dbUrl = "YOURS GOES HERE";
final String user = "YOURS GOES HERE";
final String pass = "YOURS GOES HERE";
Class.forName(jdbcDriver);
con = DriverManager.getConnection(dbUrl, user, pass);
System.out.println("Connection successful");
} catch (Exception e) {
System.out.println("Exception while establishing sql connection");
throw e;
}
}
private static void retrieveData(Connection con) throws Exception {
try {
stmt = con.createStatement();
String query = "select title_id, primary_title from imdb.title";
System.out.println(query);
stmt = con.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
stmt.setFetchSize(fetchlimit_src);
System.out.println("**** Started querying src **** " + new Date());
rs = stmt.executeQuery(query);
System.out.println("**** Completing querying src **** " + new Date());
// resultset_List(rs); // If required store resultset(rs) to HashMap
System.out.println();
String timeStamp = new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss").format(new Date());
System.out.println("Started writing CSV: " + timeStamp);
writeToCsv(rs, filename, null, Boolean.FALSE);
timeStamp = new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss").format(new Date());
System.out.println("Finished writing CSV: " + timeStamp);
System.out.println();
} catch (Exception e) {
System.out.println("Exception while retrieving data");
e.printStackTrace();
throw e;
} finally {
rs.close();
stmt.close();
con.close();
}
}
public static int writeToCsv(final ResultSet rs, final String fileName,
final String header, final boolean aync) throws Exception {
try (CSVWriter writer = new CSVWriter(fileName)) {
//Define fetch size(default as 30000 rows), higher to be faster performance but takes more memory
ResultSetHelperService.RESULT_FETCH_SIZE = 1000;
//Define MAX extract rows, -1 means unlimited.
ResultSetHelperService.MAX_FETCH_ROWS = 2000;
writer.setAsyncMode(aync);
int result = writer.writeAll(rs, true);
return result - 1;
}
}
}
Points to note:
1) I used "async" set to false:
writeToCsv(rs, filename, null, Boolean.FALSE);
You may want to experiment with this and the other settings to see if they make any significant difference for you.
2) Regarding your comment "the link has many other classes involved": The OpenCSV library's entire JAR file needs to be included in your project, as does the related disruptor JAR:
opencsv.jar
disruptor-3.3.6.jar
To get the JAR files, go to the GitHub page, click on the green button, select the zip download, unzip the zip file, and look in the "OpenCSV-master\release" folder.
Add these two JARs to your project in the usual way (depends on how you build your project).
3) WARNING: This code runs OK when you use Oracle's Java 8 JDK/JRE. If you try to use OpenJDK (e.g. for Java 13 or similar) it will not run. This is because of some changes behind the scenes to hidden classes. If you are interested, there are more details here.
If you need to use an OpenJDK version of Java, you may therefore have better luck with the library on which this CSV library is based: see here.

How to get the list of all database names from a Sybase DB Server in java

I want to get the list of all database names from a Sybase DB server. I can connect the Sybase db through Java, but don't know how to get the list of database names. I am using jconn4 jar file.
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import com.sybase.jdbc4.jdbc.SybDriver;
public class ConnectToSybase {
public static Connection conn = null;
public static Statement stmt = null;
public static SybDriver sybDriver = null;
public static ResultSet rs = null;
public static String dbServerIP = "10.10.10.11";
public static String portNo = "5000";
public static String dbName = "NewDB";
public static void main(String[] args) {
try {
Class.forName("com.sybase.jdbc4.jdbc.SybDriver").newInstance();
System.out.println("Driver loaded");
conn = DriverManager.getConnection("jdbc:sybase:Tds:" + dbServerIP + ":" + portNo, "usrname", "password");
stmt = conn.createStatement();
} catch (Exception e) {
System.out.println("In exception");
e.printStackTrace();
} finally {
try {
rs.close();
stmt.close();
conn.close();
} catch (Exception e) {
}
}
}
}

I found myself a working code.
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import com.sybase.jdbc4.jdbc.SybDriver;
public class ConnectToSybase {
public static Connection conn = null;
public static Statement stmt = null;
public static SybDriver sybDriver = null;
public static ResultSet rs = null;
public static String dbServerIP = "10.10.10.11";
public static String portNo = "5000";
public static String dbName = "NewDB";
public static void main(String[] args) {
try {
Class.forName("com.sybase.jdbc4.jdbc.SybDriver").newInstance();
System.out.println("Driver loaded");
conn = DriverManager.getConnection("jdbc:sybase:Tds:" + dbServerIP + ":" + portNo, "usrname", "password");
stmt = conn.createStatement();
List<String> dbList = new ArrayList<String>();
// getting list of DB names from the DB server
ResultSet rs = conn.getMetaData().getCatalogs();
while (rs.next()) {
dbList.add(rs.getString(1));
}
for (String list : dbList) {
System.out.println(list);
}
} catch (Exception e) {
System.out.println("In exception");
e.printStackTrace();
} finally {
try {
rs.close();
stmt.close();
conn.close();
} catch (Exception e) {
}
}
}
}

ClassNotFoundException in Apache Tomcat 8.5.14 [duplicate]

This question already has answers here:
java.lang.ClassNotFoundException: sun.jdbc.odbc.JdbcOdbcDriver Error [duplicate]
(2 answers)
Connect Java to a MySQL database
(14 answers)
Closed 4 years ago.
This is my simple code to test connectivity with MySQL :
import java.sql.*;
import java.util.*;
public class PreparedStatementTest {
public static void main(String[] args) throws Exception {
String[] str = {"ram", "shyam", "radhe", "lakhan"};
Class.forName("sun.jdbc.odbc.JdbcOdbcDriver");
Connection con = DriverManager.getConnection("jdbc:odbc:shubh", "sa", "shubham");
PreparedStatement ps = con.prepareStatement("insert into shubham_table values(?,?)");
for (int i = 0; i < 4; i++) {
ps.setInt(1, i);
ps.setString(2, str[i]);
ps.executeUpdate();
}
PreparedStatement prs = con.prepareStatement("select *from shubham_table where id=?");
for (int i = 0; i < 4; i++) {
prs.setInt(1, i);
ResultSet rs = prs.executeQuery();
while (rs.next()) {
System.out.print("id = " + rs.getInt(1));
System.out.println("name = " + rs.getString(2));
}
}
rs.close();
prs.close();
ps.close();
con.close();
}
}
This code is working properly and updating my already existing table in database but when I try to create a connection in my web app on Apache it throws ClassNotFoundException. The source code in my application is
import javax.servlet.*;
import java.io.*;
import java.sql.*;
import java.util.*;
public class RegFormServlet implements Servlet {
public void init(ServletConfig sc) throws ServletException {
System.out.println("created");
}
public ServletConfig getServletConfig() {
return null;
}
public void service(ServletRequest req, ServletResponse res) throws
ServletException, IOException {
System.out.println("before con mysql");
try {
Class.forName("sun.jdbc.odbc.JdbcOdbcDriver");
Connection con = DriverManager.getConnection("jdbc:odbc:shubh", "sa", "shubham");
String name = req.getParameter("name");
String email = req.getParameter("email");
String address = req.getParameter("address");
Statement st = con.createStatement();
String ddl =
"create table shubham_table3 (name varchar(30),e-mail
varchar(20)
,address varchar
(100))";
st.execute(ddl);
PreparedStatement ps = con.prepareStatement(
"insert into shubham_table
values( ?, ?, ?)
");
ps.setString(1, name);
ps.setString(2, email);
ps.setString(3, address);
PrintWriter out = res.getWriter();
out.println("You Are Registered Successfully yeah!!!!");
st.close();
ps.close();
con.close();
} catch (Exception e) {
System.out.println(e);
}
}
public String getServletInfo() {
return null;
}
public void destroy() {
}
}
How can I fix this?

You are using Type1 jdbc driver to connect to Mysql database. Please check Tomcat and its jdk version. From Jdk8 onwards Type1 driver is not allowed to use.
Alternatively you can use Mysql Type4 driver to do operation. Here is link
Once downloaded mysql connector driver add it to /WEB-INF/lib folder.
Hopefully It should solve issue.

Error in connect to SQL Server 2008 R2 using JDBC

I am using Java to connect Microsoft SQL Server 2008 R2 and sqljdbc4.jar, but there is a problem with the initial connection. After running, 3 is displayed in the browser only.
This is my code:
import java.sql.*;
import com.microsoft.sqlserver.jdbc.*;
public class Database {
static Connection con = null;
public static int dbConnect() {
int x = 3;
try {
// Establish the connection.
SQLServerDataSource ds = new SQLServerDataSource();
ds.setIntegratedSecurity(true);
ds.setServerName("172.18.16.10");
ds.setPortNumber(1433);
ds.setDatabaseName("my-database");
ds.setUser("my-user");
ds.setPassword("my-pass");
con = ds.getConnection();
x = 5;
if (con != null) {
x = 1;
}else{
x = 0;
}
} catch (Exception e) {
System.out.println(e.getMessage());
}
return x;
}}
and this is the error I receive:
Login failed. The login is from an untrusted domain and cannot be used
with Windows authentication.
ClientConnectionId:af711e9c-941f-4535-9ccb-b6ef31a42fdf
The older questions about my problem in SO were not helpful for me. I added sqljdbc-auth.jar into /java/bin and path also in /java/lib

I add sqljdbc.jar and import that :
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import com.microsoft.sqlserver.jdbc.*;
I define all need variable in Global ans Static :
private static SQLServerDataSource ds = null;
private static Connection con = null;
and define a method for connect:
public static boolean getConnection() {
try {
System.out.println(ServerName);
ds = new SQLServerDataSource();
ds.setServerName(ServerName);
ds.setPortNumber(1433);
ds.setDatabaseName(DatabaseName);
ds.setUser(UserName);
ds.setPassword(Password);
con = ds.getConnection();
if (con != null) {
System.out.println("Success");
}
stmt = con.createStatement();
return true;
} // Handle any errors that may have occurred.
catch (Exception e) {
System.out.println(e.getMessage());
return false;
}
}

Try doing something like this.
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
public class Database {
static Connection con = null
public static int dbConnect() {
int x = 3;
try {
// Establish the connection.
Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
con = DriverManager.getConnection("jdbc:sqlserver://172.18.16.10:1433;databaseName=my-database;user=my-user;password=my-pass;");
x = 5;
if (con != null) {
x = 1;
}else{
x = 0;
}
} catch (Exception e) {
System.out.println(e.getMessage());
}
return x;
}
}
You will get a proper connection using the above code.
Morever, the driver Name ("com.microsoft.sqlserver.jdbc.SQLServerDriver") and the connection url should always be read from a config file. Try taking the above two things out of your code into a config file.

Invalid operation for read only resultset when using select for update nowait in multithreaded environment

For an Oracle database, the following program will throw SQL exceptions only for some threads. Why downgrading resultSetConcurrency from CONCUR_UPDATABLE to CONCUR_READ_ONLY? In a single thread environment this is not happening.
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
public class Main extends Thread {
public static final String DBURL = "jdbc:oracle:thin:#localhost:1521:DB";
public static final String DBUSER = "USER";
public static final String DBPASS = "PASS";
public static void main(String[] args) {
// TODO Auto-generated method stub
for(int i=0; i<20; i++)
new Main().start();
}
#Override
public void run() {
try
{
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
Connection con = DriverManager.getConnection(DBURL, DBUSER, DBPASS);
con.setAutoCommit(false);
try(PreparedStatement pstmt = con.prepareStatement("SELECT COLUMN1 FROM TABLE1 FOR UPDATE NOWAIT",
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_UPDATABLE))
{
ResultSet rs = pstmt.executeQuery();
if (rs.next()) {
rs.updateString(1, "12345");
rs.updateRow();
}
}
finally
{
con.commit();
con.close();
}
}
catch(SQLException e)
{
if(!e.toString().contains("NOWAIT"))
e.printStackTrace();
}
}
}

You can look at the warnings raised against the result set/statement/connection to see why it was downgraded. With this added after the executeQuery() call:
SQLWarning warning = pstmt.getWarnings();
while (warning != null)
{
System.out.println("Warning: " + warning.getSQLState()
+ ": " + warning.getErrorCode());
System.out.println(warning.getMessage());
warning = warning.getNextWarning();
}
In this case you'll sometimes see:
Warning: 99999: 17091
Warning: Unable to create resultset at the requested type and/or concurrency level: ORA-00054: resource busy and acquire with NOWAIT specified or timeout expired
You're looking for a NOWAIT exception, but you're getting a warning. What isn't clear to me is why you still get a result set in that scenario; but you can at least trap that warning and not go into the result set loop if you see it.