We Keep Coding

Transferring data to Db issue, how to increase the transferring speed of data from java application to database?

In my application I am taking data from file and transferring to database. I have 400 000 records. First it transfers data fast up to 10 000 records after that it updating very slowly. Hw to increase the performance of transferring data to db?
is there any problem with gc?
This is my code:
package com.fileupload;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Iterator;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import javax.swing.text.ZoneView;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import java.io.InputStream;
import java.util.Iterator;
import org.apache.poi.xssf.eventusermodel.XSSFReader;
import org.apache.poi.xssf.model.SharedStringsTable;
import org.apache.poi.xssf.usermodel.XSSFRichTextString;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.xml.sax.Attributes;
import org.xml.sax.ContentHandler;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.helpers.XMLReaderFactory;
public class SendDataToDb extends HttpServlet{
PreparedStatement ps = null;
HttpSession hs;
Connection con1;
#Override
public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
hs = request.getSession(false);
try {
Class.forName("com.mysql.jdbc.Driver");
con1 = DriverManager.getConnection("jdbc:mysql://localhost:3306/xlsx","root","Inf123#");
ps = con1.prepareStatement("INSERT INTO userdetails(ID, NAME, AGE, GENDER,ADDRESS, ZONEID, LOCATION) VALUES(?, ?, ?, ?, ?, ?, ?)");
} catch (ClassNotFoundException e1) {
e1.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
}
try {
processOneSheet("C:/Users/Penchalaiah/Desktop/New folder/"+hs.getAttribute("filename1"));
System.out.println("clossing the connnection");
ps.close();
con1.close();
} catch (Exception e) {
e.printStackTrace();
}
}
public void processOneSheet(String filename) throws Exception {
OPCPackage pkg = OPCPackage.open(filename);
XSSFReader r = new XSSFReader( pkg );
SharedStringsTable sst = r.getSharedStringsTable();
XMLReader parser = fetchSheetParser(sst);
// To look up the Sheet Name / Sheet Order / rID,
// you need to process the core Workbook stream.
// Normally it's of the form rId# or rSheet#
InputStream sheet2 = r.getSheet("rId2");
InputSource sheetSource = new InputSource(sheet2);
parser.parse(sheetSource);
sheet2.close();
}
public XMLReader fetchSheetParser(SharedStringsTable sst) throws SAXException {
XMLReader parser = XMLReaderFactory.createXMLReader("org.apache.xerces.parsers.SAXParser");
ContentHandler handler = new SheetHandler(sst);
parser.setContentHandler(handler);
return parser;
}
/**
* See org.xml.sax.helpers.DefaultHandler javadocs
*/
private class SheetHandler extends DefaultHandler {
private SharedStringsTable sst;
private String lastContents;
private boolean nextIsString;
String id;
String names;
String age;
String gender;
String address;
int i = 1;
private SheetHandler(SharedStringsTable sst) {
this.sst = sst;
}
public void startElement(String uri, String localName, String name,
Attributes attributes) throws SAXException {
// c => cell
if(name.equals("c")) {
// Print the cell reference
// Figure out if the value is an index in the SST
String cellType = attributes.getValue("t");
if(cellType != null && cellType.equals("s")) {
nextIsString = true;
} else {
nextIsString = false;
}
}
// Clear contents cache
lastContents = "";
//System.out.println("===>"+lastContents+"<====");
}
public void endElement(String uri, String localName, String name)
throws SAXException {
// Process the last contents as required.
// Do now, as characters() may be called more than once
if(nextIsString) {
int idx = Integer.parseInt(lastContents);
lastContents = new XSSFRichTextString(sst.getEntryAt(idx)).toString();
nextIsString = false;
}
// v => contents of a cell
// Output after we've seen the string contents
if(name.equals("v")) {
System.out.print(lastContents+"\t");
if(i == 1){
id = lastContents;
System.out.print(lastContents+"("+i+")");
}
if(i == 2){
names = lastContents;
System.out.print(lastContents+"("+i+")");
}
if(i == 3){
age = lastContents;
System.out.print(lastContents+"("+i+")");
}
if(i == 4){
gender = lastContents;
System.out.print(lastContents+"("+i+")");
}
if(i == 5){
address = lastContents;
System.out.print(lastContents+"("+i+")");
insertInToDb(id, names, age, gender, address);
i = 0;
}
i++;
}
}
public void characters(char[] ch, int start, int length)
throws SAXException {
lastContents += new String(ch, start, length);
}
}
public void insertInToDb(String id,String names,String age, String gender,String address){
try {
ps.setString(1, id);
ps.setString(2, names);
ps.setString(3, age);
ps.setString(4, gender);
ps.setString(5, address);
ps.setString(6, (String)hs.getAttribute("zoneId1"));
ps.setString(7, (String)hs.getAttribute("location1"));
ps.executeUpdate();
} catch (SQLException e) {
e.printStackTrace();
}
}
}

Executing one batch operation for multiple records is much faster than executing each insert query for multiple records.
You can create a batch of 10000 or whatever you want and then execute the batch.
Connection con = null;
PreparedStatement pstm = null;
try {
Class.forName("driver class");
con = DriverManager.
getConnection("connectionUrlString","password");
con.setAutoCommit(false);
pstm = con.prepareStatement("your insert command );
pstm .setInt(1, 3000); //set all parameters
pst.addBatch();
int count[] = pst.executeBatch();
for(int i=1;i<=count.length;i++){
System.out.println("Query "+i+" has effected "+count[i]+" records");
}
con.commit();
pst.close();
con.close();

SQLException Operation not allowed after ResultSet closed

I am attempting to write a JUnit test for a query which is retrieved via a textbox in an html form. The text retrieval has been tested and works but my unit test is failing. I am using 2 relevant classes: QueryController and QueryControllerTest. I have been playing around with when and what I am closing in these two classes and keep getting the error: Operation not allowed after ResultSet closed.
QueryControllerTest.java
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import org.junit.Test;
import static org.junit.Assert.*;
public class QueryControllerTest {
#Test
public void testQuery() {
ResultSet testRs = null;
Connection conn = null;
try {
Class.forName("com.mysql.jdbc.Driver");
String connectionUrl = "jdbc:mysql://localhost:3306/test";
String connectionUser = "root";
String connectionPassword = "GCImage";
conn = DriverManager.getConnection(connectionUrl,
connectionUser, connectionPassword);
Query testQuery = new Query();
testQuery
.setQuery("select * from service_request where FN_contact = 'Greg'");
testRs = QueryController.executeSelect(conn, testQuery);
assertEquals("Laughlin", testRs.getString("LN_contact"));
assertEquals("Hello World", testRs.getString("Notes"));
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
} finally {
try {
testRs.close();
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}
QueryController.java
import java.util.Map;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.RequestMapping;
#Controller
public class QueryController {
#RequestMapping(value = "/query")
public String processRegistration(#ModelAttribute("query") Query query,
Map<String, Object> model) {
String queryString = query.getQuery();
if (queryString != null && !queryString.isEmpty()) {
System.out.println("query (from controller): " + queryString);
Connection conn = null;
try {
Class.forName("com.mysql.jdbc.Driver");
String connectionUrl = "jdbc:mysql://localhost:3306/test";
String connectionUser = "root";
String connectionPassword = "GCImage";
conn = DriverManager.getConnection(connectionUrl,
connectionUser, connectionPassword);
if (queryString.toLowerCase().startsWith("select")) {
ResultSet rs = executeSelect(conn, query);
} else {
int rowsUpdated = executeUpdate(conn, query);
System.out.println(rowsUpdated + " rows updated");
}
} catch (Exception e) {
e.printStackTrace();
}
}
return "query";
}
public static ResultSet executeSelect(Connection conn, Query query) {
ResultSet rs = null;
Statement stmt = null;
try {
stmt = conn.createStatement();
rs = stmt.executeQuery(query.getQuery());
while (rs.next()) {
String id = rs.getString("ID");
String firstName = rs.getString("FN_Contact");
String lastName = rs.getString("LN_Contact");
String notes = rs.getString("Notes");
System.out.println("ID: " + id + ", First Name: " + firstName
+ ", Last Name: " + lastName + ", Notes: " + notes);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if(rs!=null){
rs.close();
}
if(stmt != null){
stmt.close();
}
if (conn != null)
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return rs;
}
}

QueryController.executeSelect is calling rs.close(), but then your assertEquals in QueryControllerTest.testQuery are calling methods on testRS. As executeSelect is returning the resultset, closing it first doesn't make sense. Further, executeSelect is being passed the connection, so it shouldn't be closing that either (what happens if the caller wants to do two different selects on the same connection?).

I think the problem is because you are creating two connections. Try to only instantiate the connection of QueryController class for your test. You will need to provide the connection. After you store it in a variable to run the query.
Connection con = QueryController.getConnection ();

Why the INSERT query doesn't work in this HttpServer code

Im trying to make a little server for my homework.This is very simple project yet i cant insert some variables (which i took from the client ,in an object form ,through serialization ) into the database .
It shows no errors! That's what i find strange and also the client receive the response without problems.
my Server class is as the following :
package server;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import org.ietf.jgss.Oid;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpServer;
public class Server {
public static void main(String[] args) throws Exception {
HttpServer server = HttpServer.create(new InetSocketAddress(3333), 0);
server.createContext("/", new MyHandler());
server.setExecutor(null); // creates a default executor
server.start();
}
static class MyHandler implements HttpHandler {
public void handle(HttpExchange t) throws IOException {
ObjectInputStream ios = new ObjectInputStream(t.getRequestBody());
final String url = "jdbc:mysql://localhost/httpServer";
final String user = "root";
final String password = "";
try {
Send oin = (Send) ios.readObject();
String response = "Kjo eshte nje pergjigje nga serveri! \n"
+ "Clienti me id "
+ oin.getId()
+ " dhe me emer "
+ oin.getName()
+ " ka pasur "
+ oin.getAmount()
+ "$ ne llogarine e tij ,por me pas ka terhequr "
+ oin.getPaid()
+ "$ nga llogaria \n"
+ "Kjo terheqe eshte ruajtur ne database dhe tani gjendja e re eshte "
+ (oin.getAmount() - oin.getPaid()) + "$ \n";
t.sendResponseHeaders(200, response.length());
OutputStream os = t.getResponseBody();
os.write(response.getBytes());
os.close();
int id = oin.getId();
String emri = oin.getName();
int amount = oin.getAmount();
int paid = oin.getPaid();
try {
Class.forName("com.mysql.jdbc.Driver");
Connection con = DriverManager.getConnection(url, user,
password);
try {
Statement s = con.createStatement();
s.executeUpdate("INSERT INTO person VALUES ('" + id
+ "','" + emri + "','" + amount + "','" + paid
+ "')");
} catch (SQLException s) {
System.out
.println("Tabel or column or data type is not found!");
}
} catch (Exception e) {
e.printStackTrace();
}
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
}
}
}
can you please help me ?
Or have any idea what the problem may is ?
Edit:
Maybe i am doing something wrong in the Client:
package server;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.net.HttpURLConnection;
import java.net.InetAddress;
import java.net.Socket;
import java.net.URL;
class Send implements Serializable {
// duhet te implementoje interfacin serizable ne menyre qe tja dergoj
// serverit
private static final long serialVersionUID = 1L;
public int getId() {
return id;
}
public int getAmount() {
return amount;
}
public int getPaid() {
return paid;
}
int id = 1;
int amount = 2000;
int paid = 800;
String name = "Andi Domi";
public String getName() {
return name;
}
}
public class Client {
public static void main(String[] args) throws Exception {
try {
URL url = new URL("http://localhost:3333");
HttpURLConnection s = (HttpURLConnection) url.openConnection();
s.setDoOutput(true);
s.setDoInput(true);
s.setRequestMethod("POST");
s.setUseCaches(false);
Send obj = new Send();
ObjectOutputStream objOut = new ObjectOutputStream(
s.getOutputStream());
objOut.writeObject(obj);
InputStream in = s.getInputStream();
InputStreamReader isr = new InputStreamReader(in);
BufferedReader br = new BufferedReader(isr);
int c;
while ((c = br.read()) != -1) {
System.out.print((char) c);
}
objOut.close();
s.disconnect();
} catch (IOException ex) {
System.err.println(ex);
System.err.print("gabimi eshte ketu");
}
}
}

After your executeUpdate statement you need to do.
con.commit();
to save the transaction.
EDIT: Based on the chat discussion, we learned that the column named emri is actually Emri in the table and was throwing:
com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException: Unknown column 'emri' in 'field list'
Changing the name resolves the issue.
Now unrelated to your problem, you should be using a PreparedStatement instead and should be closing your connection and statement
try {
PreparedStatement s = con.prepareStatement("INSERT INTO person(id, emri, amount, paid) VALUES (?,?,?,?)");
s.setInt(1,id);
s.setString(2,emri);
s.setInt(3,amount);
s.setInt(4,paid);
int count = s.executeUpdate();
con.commit();
} catch(Exception e){
e.printStackTrace();
//something bad happened rollback
//any uncommitted changes
con.rollback();
} finally {
if (con != null) {
con.close();
}
}

first, use prepared statement[docs] to avoid from SQL INJECTION
String sql = "INSERT INTO person VALUES (?,?,?,?)";
PreparedStatement prest = con.prepareStatement(sql);
prest.setString(1,id);
prest.setString(2,emri); // or use setInt for integer
prest.setString(3,amount); // or use setInt for integer
prest.setString(4,paid);
prest.executeUpdate()
second, if the the number of values does not match the total number of columns in your table, it will also fail because you are using the implicit type of INSERT statement. To solve it, just supply the column names where you want the values should be stored, eg
String sql = "INSERT INTO person (col1, col2, col3, col4) VALUES (?,?,?,?)";

selecting data from cassandar using cql

i am using cassandra-jdbc to perform the operation on data in cassandra but when i run this simple program i get exception.
this is my code:
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.*;
import javax.sql.*;
public class Operations
{
public static void main(String[] args){
try
{
Class.forName("org.apache.cassandra.cql.jdbc.CassandraDriver");
Connection con = DriverManager.getConnection("jdbc:cassandra://localhost:9160/temp");
String qry = "select name FROM cql";
Statement smt = con.createStatement();
ResultSet resultSet = smt.executeQuery(qry);
System.out.println(resultSet);
while(resultSet.next())
{
System.out.println(resultSet);
}
}
catch(Exception e)
{
System.out.println(" : "+e.getMessage());
}
}
}
i got :cannot parse 'name' as hex bytes

Try:
import static org.apache.cassandra.utils.Hex.bytesToHex;
...
String name = bytesToHex("name".getBytes());
String qry = "select '" + name + "' FROM cql";

Cannot find place to insert finally block to get rid of the error: Insert Finally to complete TryStatement

I've tried several spots to insert the finally block but no matter what I try it ends up making the code worse.
Here is my code, the 4th to last ending curly bracket is the one giving me the error. Any thoughts?
package com.tunestore.action;
import java.io.InputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.PreparedStatement;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.struts.action.Action;
import org.apache.struts.action.ActionForm;
import org.apache.struts.action.ActionForward;
import org.apache.struts.action.ActionMapping;
import org.apache.struts.action.ActionMessage;
import org.apache.struts.action.ActionMessages;
import org.apache.struts.action.DynaActionForm;
import java.io.*;
import javax.servlet.*;
import javax.servlet.http.*;
import org.owasp.validator.html.*;
import org.owasp.esapi.*;
public class DownloadAction extends Action
{
private static final Log log = LogFactory.getLog(DownloadAction.class);
public static String DB_URL;
static
{
if (System.getProperty("tunestore.db.location") != null)
{
DB_URL = "jdbc:derby://localhost:1527/" + System.getProperty("tunestore.db.location");
}
else
{
DB_URL = "jdbc:derby://localhost:1527/" + System.getProperty("user.home") + "/.tunestore";
}
System.setProperty("jdbc.tunestore.url", DB_URL);
}
public static Connection getConnection() throws Exception
{
log.info("Opening database at " + DB_URL);
Class.forName("org.apache.derby.jdbc.ClientDriver").newInstance();
Connection conn = DriverManager.getConnection(DB_URL);
return conn;
}
public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServletRequest request, HttpServletResponse response)
throws Exception
{
DynaActionForm daf = (DynaActionForm)form;
String user = (String)request.getSession(true).getAttribute("USERNAME");
if(user != null)
{
Connection conn = null;
try
{
conn = DownloadAction.getConnection();
String sql2 = "SELECT ID FROM CD WHERE CD.BITS = ?";
PreparedStatement stmt2 = conn.prepareStatement(sql2);
stmt2.setString(1, request.getParameter("cd"));
ResultSet rs2 = stmt2.executeQuery();
rs2.next();
String sql = "SELECT COUNT(*) "
+ "FROM TUNEUSER_CD "
+ "WHERE TUNEUSER_CD.TUNEUSER = ? AND TUNEUSER_CD.CD = ?";
PreparedStatement stmt = conn.prepareStatement(sql);
stmt.setString(1, user);
stmt.setInt(2, rs2.getInt(1));
ResultSet rs = stmt.executeQuery();
rs.next();
int owned = rs.getInt(1);
if(owned == 1)
{
try
{
// Try to open the stream first - if there's a goof, it'll be here
InputStream is = this.getServlet().getServletContext().getResourceAsStream("/WEB-INF/bits/" + request.getParameter("cd"));
if (is != null)
{
response.setContentType("audio/mpeg");
response.setHeader("Content-disposition", "attachment; filename=" + daf.getString("cd"));
byte[] buff = new byte[4096];
int bread = 0;
while ((bread = is.read(buff)) >= 0)
{
response.getOutputStream().write(buff, 0, bread);
}
}
else
{
ActionMessages errors = getErrors(request);
errors.add(ActionMessages.GLOBAL_MESSAGE, new ActionMessage("download.error"));
saveErrors(request, errors);
return mapping.findForward("error");
}
}
catch (Exception e)
{
e.printStackTrace();
ActionMessages errors = getErrors(request);
errors.add(ActionMessages.GLOBAL_MESSAGE, new ActionMessage("download.error"));
saveErrors(request, errors);
return mapping.findForward("error");
}
return null;
}
}
}
}
}

That bracket is the location where your outer try block ends. It has no catch block and no finally block, so you get an error. Just add one or the other immediately after the bracket, or remove the try if it's not needed.

You only have one catch block, but two trys. Add a catch block for the first try-catch and you should have your issue solved.
Edit: Why are you nesting try-catch in the first place? I don't believe there is any need to do so.