We Keep Coding

Trying to read 700k+ of data and the Error "GC Overhead Limit Exceeded" occurred

Alright so I need help in reviewing my codes because I'm kinda still new in programming (currently in my second year of Diploma in Computer Science). I got this error as in the title GC Overhead Limit Exceeded when I tried running my code below.
A brief explanation of this code, I'm trying to read data from a CSV File and then transfer it to a database. FYI, there are actually 10 tables/CSV files that I need to read, but on this I'll show this one table Tickets because the error only occurred when I tried to read that table/file. The other tables have hundreds of rows/data only while the table Tickets have 735,504 of rows/data. Furthermore, I've succeeded in reading 450,028 of data after 6 hours of running the code before the error occurred.
What can I do to fix this error? What can be modified to improve my code? I really appreciate it if you guys can help me :)
public class Demo2 {
public static void main(String[] args) {
String url = "jdbc:mysql://localhost:3306/database";
String username = "root";
String password = "password";
try {
//Connect to the database
Connection connection = DriverManager.getConnection(url, username, password);
//Test on one table only
String tableName = "Tickets";
System.out.println("Connecting to TABLE " +tableName +"...");
readCSVFile(tableName, connection);
System.out.println();
System.out.println("THE END");
connection.close();//close connection to the database
}
catch (SQLException e) {
System.out.println("ERROR at main(): SQLException!!");
e.printStackTrace();
}
}
static int countNewRow = 0;
static int countUpdatedRow = 0;
//Method to read the CSV File
static void readCSVFile(String tableName, Connection conn) {
//Read CSV File
try {
String path = tableName +".csv";
BufferedReader br = new BufferedReader(new FileReader(path));
br.readLine();//skip the first line
String inData;
//Read The Remaining Line
while((inData=br.readLine()) != null)
{
String[] rowData = inData.split(",");
ArrayList <String> rowDataList = new ArrayList<String>();
for (int i=0; i<rowData.length; i++)
rowDataList.add(rowData[i]);
//To combine String that starts and ends with "
for(int i=0; i<rowDataList.size(); i++) {
if (rowDataList.get(i).charAt(0) == '"') {
String string1 = rowDataList.get(i).substring(1, rowDataList.get(i).length());
String string2 = rowDataList.get(i+1).substring(0, rowDataList.get(i+1).length()-1);
String combined = string1 +"," +string2;
rowDataList.set(i, combined);
rowDataList.remove(i+1);
break;
}
}
//Remove the RM
for(int i=0; i<rowDataList.size(); i++) {
if (rowDataList.get(i).startsWith("RM")) {
String string = rowDataList.get(i).substring(2);
rowDataList.set(i, string);
}
}
//This is just to keep track of the data that has been read
System.out.println("[" +rowDataList.get(0) +"]");
//Transfer the data to the database
insertToDatabase(conn, tableName, rowDataList);
}
System.out.println("New Row Added : " +countNewRow);
System.out.println("Updated Row : " +countUpdatedRow);
System.out.println("== Process Completed ==");
br.close();
}
catch (FileNotFoundException e) {
System.out.println("ERROR at readCSVFile(): FileNotFoundException!!");
e.printStackTrace();
}
catch (IOException e) {
System.out.println("ERROR at readCSVFile(): IOException!!");
e.printStackTrace();
}
catch (SQLException e) {
System.out.println("ERROR at readCSVFile(): SQLException!!");
e.printStackTrace();
}
catch (ParseException e) {
System.out.println("ERROR at readCSVFile(): ParseException!!");
e.printStackTrace();
}
}
static void insertToDatabase(Connection connection, String tableName, ArrayList <String> rowDataList) throws SQLException, ParseException {
String tableIdName = tableName;
if (tableIdName.charAt(tableIdName.length()-1) == 's')
tableIdName = tableIdName.substring(0, tableIdName.length()-1);
//To read row
String rowID = rowDataList.get(0);
String selectSQL = "SELECT * FROM " +tableName +" "
+"WHERE " +tableIdName +"_ID = " +rowID;
Statement statement = connection.createStatement();
ResultSet result = statement.executeQuery(selectSQL);
boolean value = result.next();
//INSERT # UPDATE row
if (value == true) { //Update Row if the data is already existed
updateStatementt(tableName, connection, rowDataList);
countUpdatedRow++;
}
else { //Insert New Row
insertStatementt(tableName, connection, rowDataList);
countNewRow++;
}
}
//Method to insert data to the database
static void insertStatementt(String tableType, Connection conn, ArrayList <String> rowDataList) throws SQLException, ParseException {
//Generate Question Mark
String generateQuestionMark = null;
if(rowDataList.size() == 1)
generateQuestionMark = "?";
else
generateQuestionMark = "?, ";
for(int i=1; i<rowDataList.size(); i++) {
if(i!=rowDataList.size()-1)
generateQuestionMark += "?, ";
else
generateQuestionMark += "?";
}
//Insert sql
String sql = "INSERT INTO " +tableType +" VALUES (" +generateQuestionMark +")";
PreparedStatement insertStatement = conn.prepareStatement(sql);
//Insert data
//There are other 'if' and 'else if' statements here for other tables
else if (tableType.equals("Tickets")) {
int ticketID = Integer.parseInt(rowDataList.get(0));
int movieId = Integer.parseInt(rowDataList.get(1));
int theaterId = Integer.parseInt(rowDataList.get(2));
String[] date = rowDataList.get(3).split("/");
String dateString = date[2] +"-" +date[1] +"-" +date[0];
Date showDate = Date.valueOf(dateString);
int showTimeId = Integer.parseInt(rowDataList.get(4));
int cptId = Integer.parseInt(rowDataList.get(5));
int pcId = Integer.parseInt(rowDataList.get(6));
float amountPaid = Float.parseFloat(rowDataList.get(7));
int year = Integer.parseInt(rowDataList.get(8));
String month = rowDataList.get(9);
insertStatement.setInt(1, ticketID);
insertStatement.setInt(2, movieId);
insertStatement.setInt(3, theaterId);
insertStatement.setDate(4, showDate);
insertStatement.setInt(5, showTimeId);
insertStatement.setInt(6, cptId);
insertStatement.setInt(7, pcId);
insertStatement.setFloat(8, amountPaid);
insertStatement.setInt(9, year);
insertStatement.setString(10, month);
}
insertStatement.executeUpdate();
insertStatement.close();
}
//Method to update the data from the database
static void updateStatementt(String tableType, Connection conn, ArrayList <String> rowDataList) throws SQLException {
Statement statement = conn.createStatement();
String sql = "UPDATE " +tableType;
//There are other 'if' and 'else if' statements here for other tables
else if (tableType.equals("Tickets")) {
String[] date = rowDataList.get(3).split("/");
String dateString = date[2] +"-" +date[1] +"-" +date[0];
sql += " SET movie_id = " +rowDataList.get(1) +","
+ " theater_id = " +rowDataList.get(2) +","
+ " showdate = \"" +dateString +"\","
+ " showtime_id = " +rowDataList.get(4) +","
+ " costperticket_id = " +rowDataList.get(5) +","
+ " personcategory_id = " +rowDataList.get(6) +","
+ " amount_paid = " +rowDataList.get(7) +","
+ " year = " +rowDataList.get(8) +","
+ " month = \"" +rowDataList.get(9) +"\""
+ " WHERE ticket_id = " +rowDataList.get(0);
}
statement.executeUpdate(sql);
}
}

For short, read a single line and do whatever you want to do with it. You don't have enough memory for all 700k lines.

You should add statement.close() for the update Statement.

If you really want to read all this data into the Java heap, increase the heap size using, for example, the -Xmx command-line switch. Because of the way textual data is encoded in the JVM, you'll probably need much more heap that the total data size would suggest.
In addition, there might be some places in your code where you can take the strain off the JVM's memory management system. For example, concatenating strings using "+" can generate a lot of temporary data, which will increase the load on the garbage collector. Assembling strings using a StringBuilder might be a simple, less resource-hungry, alternative.

ArrayIndexOutOfBounds: 0 when trying to add an image as a BLOB to an sqlite database in java

I am trying to insert an image as a blob to my sqlite database but I just can't get it to work as I am catching the error java.lang.ArrayIndexOutOfBoundsException: 0
I tried to do it with one try catch statement but again i couldn't figure it out. The url used to create the connection to the database is correct since I have been using the same url throught the other methods and were working fine. Any help would be greatly appreciated.
This is my code:
public Boolean insertIntoProducts(String name, String description, String filePath, Double price, int quantity) {
String url = "jdbc:sqlite:C:/Users/User/Desktop/MyDatabase.db";
Boolean success = false;
String sqlSelectID = "SELECT ID FROM Products ORDER BY ID DESC";
// Establishing a connection to the database
try {
// Initialising the image file
File image = new File(filePath);
FileInputStream fileInput = new FileInputStream(image);
ByteArrayOutputStream byteArray = new ByteArrayOutputStream();
byte[] buf = new byte[1024];
for (int i; (i = fileInput.read(buf)) != -1;) {
// read all the bytes and store them in the array. Stores the binary data of the
// image
byteArray.write(buf, 0, i);
// Closing the file input to prevent memory leaks
fileInput.close();
try (Connection conn = DriverManager.getConnection(url)){
Statement statement = conn.();
ResultSet result = statement.executeQuery(sqlSelectID);
int id = 0;
if (result.next()) {
id = result.getInt("ID");
}
String sql = "INSERT INTO Products(ID,Name,Description,Quantity,Price,Image,isAuthorised) "
+ "VALUES (" + id + ",'" + name + "','" + description + "'," + quantity + ",'" + price
+ "','?','0');";
PreparedStatement pstatement = conn.prepareStatement(sql);
pstatement.setBytes(1, byteArray.toByteArray());
// Creating a variable to check if the insertion was successful depending if a
// row was added to the table
int row = 0;
row = pstatement.executeUpdate();
if (row > 0) {
success = true;
}
} catch (SQLException e) {
System.out.println(e);
}
}
} catch (Exception e) {
System.out.println(e);
}
return success;
}
The error is in the line pstatement.setBytes(1, byteArray.toByteArray());
I have checked the length of the array and it has a correct length but I don't know what it is trying to access in order for this error to be created

Java Servlet store users info into Java DB

I have write a java servlet to insert and display users info from database. I am inserting a photo to all users. The problem is when I want to display the byte array for the photos it displays just the first bytes e.g.: [B#43809cd3
private void doAddUser(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
InputStream inputStream = null;
Part filePart = request.getPart("image");
if (filePart != null) {
// prints out some information for debugging
System.out.println(filePart.getName());
System.out.println(filePart.getSize());
System.out.println(filePart.getContentType());
// obtains input stream of the upload file
inputStream = filePart.getInputStream();
}
String message = null;
try {
// Register JDBC driver
Class.forName("org.apache.derby.jdbc.ClientDriver");
// Open a connection
Connection conn = DriverManager.getConnection("jdbc:derby://127.0.0.1:1527/tutoriumDB", "db", "db");
// Execute SQL query
PreparedStatement st= conn.prepareStatement("insert into users(ID,NAME,LAT,LONG,IMAGE) values(?,?,?,?,?)");
st.setString(1, request.getParameter("id1"));
st.setString(2, request.getParameter("name"));
st.setString(3, request.getParameter("lat"));
st.setString(4, request.getParameter("long"));
if (inputStream != null) {
//st.setBlob(5, inputStream);
st.setBinaryStream(5, inputStream, (int)filePart.getSize());
}
int row = st.executeUpdate();
if (row > 0) {
message = "File uploaded and saved into database";
}
st.close();
conn.close();
try (PrintWriter out = response.getWriter()) {
out.println("Der Benutzer mit der ID: " +request.getParameter("id1")+" und Name: " +
request.getParameter("name")+" wurde eingefuegt");
}
} catch (ClassNotFoundException | SQLException ex) {
Logger.getLogger(TutServlet.class.getName()).log(Level.SEVERE, null, ex);
}
// sets the message in request scope
request.setAttribute("Message", message);
// forwards to the message page
getServletContext().getRequestDispatcher("/submit.jsp").forward(
request, response);
}
private void getAllUsers(HttpServletRequest request, HttpServletResponse response) throws IOException
{
try {
// Register JDBC driver
Class.forName("org.apache.derby.jdbc.ClientDriver");
// Open a connection
Connection conn = DriverManager.getConnection("jdbc:derby://127.0.0.1:1527/tutoriumDB", "db", "db");
// Execute SQL query
String sql = "SELECT * FROM users order by ID";
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql);
byte[] imgData = null ;
ArrayList<String> users = new ArrayList<String>();
while (rs.next()) {
int id = rs.getInt("id");
String name = rs.getString("name");
double lat = rs.getDouble("lat");
double longit = rs.getDouble("long");
byte[] bytes = rs.getBytes("image");
users.add(id + ", " + name+ ", " +lat+ ", " +longit+ ", " +bytes);
}
stmt.close();
conn.close();
try (PrintWriter out = response.getWriter()) {
for (String s : users)
{
out.println(s);
}
}
} catch (ClassNotFoundException | SQLException ex) {
Logger.getLogger(TutServlet.class.getName()).log(Level.SEVERE, null, ex);
}
}
Result of getAllUser

Ok let me check i analize your code and saw the image, the image is correct because you are using
users.add(id + ", " + name+ ", " +lat+ ", " +longit+ ", " +bytes);
as you can see the var called bytes is this:
byte[] bytes = rs.getBytes("image");
why is it getting [B#43809cd3 ? because the method toString is not overriding in byte.
What could you do? if your byte is not null then iterate it and make your own like this:
byte[] bytes = new byte[2];
bytes[0] = 1;
bytes[1] = 127;
System.out.println(bytes);
StringBuilder byteAppender = new StringBuilder("");
if(null != bytes){
for(byte b: bytes){
byteAppender.append(b);
}
System.out.println("Ok bites are " +byteAppender);
}
So now you will put byteAppender instead of bytes so this will be:
users.add(id + ", " + name+ ", " +lat+ ", " +longit+ ", " +byteAppender );
So your coude could be like this:
private void getAllUsers(HttpServletRequest request, HttpServletResponse response) throws IOException
{
try {
// Register JDBC driver
Class.forName("org.apache.derby.jdbc.ClientDriver");
// Open a connection
Connection conn = DriverManager.getConnection("jdbc:derby://127.0.0.1:1527/tutoriumDB", "db", "db");
// Execute SQL query
String sql = "SELECT * FROM users order by ID";
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql);
byte[] imgData = null ;
ArrayList<String> users = new ArrayList<String>();
StringBuilder byteAppender = new StringBuilder(""); //here is the String you will make
while (rs.next()) {
int id = rs.getInt("id");
String name = rs.getString("name");
double lat = rs.getDouble("lat");
double longit = rs.getDouble("long");
byte[] bytes = rs.getBytes("image");
if(null != bytes){ //you ask the object is not null
for(byte b: bytes){ //you itarete it
byteAppender.append(b); //you add it
}
}
users.add(id + ", " + name+ ", " +lat+ ", " +longit+ ", " +byteAppender); //you put it
byteAppender.setLength(0); //you restart it
}
stmt.close();
conn.close();
try (PrintWriter out = response.getWriter()) {
for (String s : users)
{
out.println(s);
}
}
} catch (ClassNotFoundException | SQLException ex) {
Logger.getLogger(TutServlet.class.getName()).log(Level.SEVERE, null, ex);
}
}
Also you could read this: how to convert byte array to string and vice versa
Hope it help you.
Cya

Tamil utf-8 encoding using java and mySQL

I am using Eclipse, with MY SQL Server 5.0.
Using JFrame, I insert Tamil words into a textfield. However, the MY SQL console shows question marks instead of the Tamil words. I also see question marks in the textfield, but I am able to type there in Tamil using Alt+Shift key.
(Tamil being an indian language with special characters)
How can I properly switch to Unicode?

enter code here
try
{
try
{
Class.forName ("com.mysql.jdbc.Driver");
try
{
java.sql.Connection conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/mydbase?useUnicode=true&characterEncoding=utf-8", "root", "ALPHAS");
String sql = "insert into morph values(?,?,?)";
String sql1 = "insert into word values(?)";
String sql2 = "insert into stems values(?)";
String sql3 = "insert into suffixs values(?)";
java.sql.PreparedStatement psmt = conn.prepareStatement(sql);
java.sql.PreparedStatement psmt1 = conn.prepareStatement(sql1);
java.sql.PreparedStatement psmt2 = conn.prepareStatement(sql2);
java.sql.PreparedStatement psmt3 = conn.prepareStatement(sql3);
String s= jTextField1.getText() ;
String sq = "select*from word";
String sqlA = "select Word from morph where Word ="+"'"+jTextField1.getText()+"'";
try
{
java.sql.Statement stmtB = conn.createStatement();
java.sql.ResultSet rsq = stmtB.executeQuery(sqlA);
while(rsq.next())
{
String sA = jTextField3.getText() ;
String ds = "WordExists";
if (s== sA)
{
jTextField3.setText(rsq.getString("Word"));
jTextField1.setText(rsq.getString("ds"));
}
}
}
catch (SQLException e )
{
e.printStackTrace();
}
String sA = jTextField3.getText() ;
char[] ch = s.toCharArray();
int y = s.length();
int k=1;
for (int m = y-1 ; m >=0; m--)
{
String pl1 = new String(ch,0,m);
if(pl1.length()>=2)
{
char[] dh = new char[y];
int c=y-1;
int g = y-1;
for (int j=c; j>=0; j--)
{
if(m>=2)
{
dh[g] = ch[j];
String pl = new String(ch,0,m);
String mpm = new String(dh,0,y);
--m;
// System.out.println(""+pl+"/"+mpm.trim()+"");
// System.out.println(""+pl+"/"+mpm.trim()+"");
g--;
k++;
if(k==y-1)
{
System.out.println(""+pl+"/"+mpm.trim()+"");
psmt.setString(1,s);
psmt.setString(2,pl);
psmt.setString(3,mpm.trim()+"");
psmt1.setString(1,s);
psmt2.setString(1,pl);
psmt3.setString(1,mpm.trim()+"");
psmt.executeUpdate();
psmt1.executeUpdate();
psmt2.executeUpdate();
psmt3.executeUpdate();
}
}
}
}
}
}
catch(Exception E)
{
E.printStackTrace();
}
}
catch(Exception E)
{
E.printStackTrace();
}
}
catch(Exception E)
{
E.printStackTrace();
}

See UTF-8 all the way through
Question marks come from this:
The client has a valid character (good), and
The SET NAMES agrees with the encoding that the client has (good), but
The target column's CHARACTER SET does not include the intended character (bad).
The Tamil words are lost. You will need to fix the CHARACTER SET and re-INSERT the text.

MD5 java and mysql md5 match and look the same but aren't matching with .equals()

Thanks for looking. I'm having an issue with getting my database and connection to match passwords. MySQL and my java md5s are the same, but when I check to see if it's equal in comparing, it says they're not the same. I'm not really sure what I'm doing wrong here.
For example, here are the generated MD5's for both:
MySQL: 798da231909aa3645eced61dde9f9bfa
Java: 798da231909aa3645eced61dde9f9bfa
So, I'm not sure why they wouldn't be equal when I check.
import java.security.MessageDigest;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.*;
/**
* Created by Ohlaph on 8/29/2015.
*/
public class Conn {
private Connection con = null;
private String JDBC = "com.mysql.jdbc.Driver";
private String username = "root", password = "password";
private Statement statement;
private ResultSet rs;
private String user_name, user_password;
private String dbname = "jdbc:mysql://localhost/thon";
public Conn(String user, String password) {
this.user_name = user;
this.user_password = MD5(password);
}
public void Connect() throws Exception {
try {
Class.forName(JDBC);
con = DriverManager.getConnection("jdbc:mysql://localhost/thon", username, password);
statement = con.createStatement();
} catch (Exception e) {
System.out.println(e);
System.out.println("Failure");
}
}//end CONNECT()
public boolean checkIt() throws Exception {
String check = "select * from users";
try {
rs = statement.executeQuery(check);
while (rs.next()) {
String usr = rs.getString("nickname");
String pwd = rs.getString("password");
System.out.println("user name " + usr + ", Password " + pwd);
if (user_name.equals(usr) && user_password.equals(pwd)) {
System.out.println("Access Granted");
return true;
} else {
System.out.println("Access Denied");
return false;
}
}
} catch (Exception e) {
System.out.println("Error " + e);
}
con.close();
return false;
}//end checkIt()
public static String MD5( String source ) {
try {
MessageDigest md = MessageDigest.getInstance( "MD5" );
byte[] bytes = md.digest( source.getBytes("UTF-8") );
return getString( bytes );
} catch( Exception e ) {
e.printStackTrace();
return null;
}
}//end MD5()
private static String getString( byte[] bytes ) {
StringBuffer sb = new StringBuffer();
for( int i=0; i<bytes.length; i++ )
{
byte b = bytes[ i ];
String hex = Integer.toHexString((int) 0x00FF & b);
if (hex.length() == 1)
{
sb.append("0");
}
sb.append( hex );
}
return sb.toString();
}// end getString()
}//end Conn.java

The line user_password = MD5(user_password); is inside the while loop, so it will MD5 the value on first user record, then MD5 the already MD5'd value on the second user record, and so on, so that by the time you get to the user in question you have a very mangled value.
Maybe it would be better to do the MD5 in the constructor.
Unrelated, but instead of querying the entire user table every time, you should use something like:
String sql = "select password from users where nickname = ?";
and use a PreparedStatement to set the marker value (?).
Update
Your error is that you do the System.out.println("Access Denied"); return false; inside the loop, so the loop never gets to look at the second record.
Adding where nickname = ? to the SQL would help fix that, since only one record would be returned (assuming nickname is unique), but the loop logic is still flawed.