We Keep Coding

Trying to read 700k+ of data and the Error "GC Overhead Limit Exceeded" occurred

Alright so I need help in reviewing my codes because I'm kinda still new in programming (currently in my second year of Diploma in Computer Science). I got this error as in the title GC Overhead Limit Exceeded when I tried running my code below.
A brief explanation of this code, I'm trying to read data from a CSV File and then transfer it to a database. FYI, there are actually 10 tables/CSV files that I need to read, but on this I'll show this one table Tickets because the error only occurred when I tried to read that table/file. The other tables have hundreds of rows/data only while the table Tickets have 735,504 of rows/data. Furthermore, I've succeeded in reading 450,028 of data after 6 hours of running the code before the error occurred.
What can I do to fix this error? What can be modified to improve my code? I really appreciate it if you guys can help me :)
public class Demo2 {
public static void main(String[] args) {
String url = "jdbc:mysql://localhost:3306/database";
String username = "root";
String password = "password";
try {
//Connect to the database
Connection connection = DriverManager.getConnection(url, username, password);
//Test on one table only
String tableName = "Tickets";
System.out.println("Connecting to TABLE " +tableName +"...");
readCSVFile(tableName, connection);
System.out.println();
System.out.println("THE END");
connection.close();//close connection to the database
}
catch (SQLException e) {
System.out.println("ERROR at main(): SQLException!!");
e.printStackTrace();
}
}
static int countNewRow = 0;
static int countUpdatedRow = 0;
//Method to read the CSV File
static void readCSVFile(String tableName, Connection conn) {
//Read CSV File
try {
String path = tableName +".csv";
BufferedReader br = new BufferedReader(new FileReader(path));
br.readLine();//skip the first line
String inData;
//Read The Remaining Line
while((inData=br.readLine()) != null)
{
String[] rowData = inData.split(",");
ArrayList <String> rowDataList = new ArrayList<String>();
for (int i=0; i<rowData.length; i++)
rowDataList.add(rowData[i]);
//To combine String that starts and ends with "
for(int i=0; i<rowDataList.size(); i++) {
if (rowDataList.get(i).charAt(0) == '"') {
String string1 = rowDataList.get(i).substring(1, rowDataList.get(i).length());
String string2 = rowDataList.get(i+1).substring(0, rowDataList.get(i+1).length()-1);
String combined = string1 +"," +string2;
rowDataList.set(i, combined);
rowDataList.remove(i+1);
break;
}
}
//Remove the RM
for(int i=0; i<rowDataList.size(); i++) {
if (rowDataList.get(i).startsWith("RM")) {
String string = rowDataList.get(i).substring(2);
rowDataList.set(i, string);
}
}
//This is just to keep track of the data that has been read
System.out.println("[" +rowDataList.get(0) +"]");
//Transfer the data to the database
insertToDatabase(conn, tableName, rowDataList);
}
System.out.println("New Row Added : " +countNewRow);
System.out.println("Updated Row : " +countUpdatedRow);
System.out.println("== Process Completed ==");
br.close();
}
catch (FileNotFoundException e) {
System.out.println("ERROR at readCSVFile(): FileNotFoundException!!");
e.printStackTrace();
}
catch (IOException e) {
System.out.println("ERROR at readCSVFile(): IOException!!");
e.printStackTrace();
}
catch (SQLException e) {
System.out.println("ERROR at readCSVFile(): SQLException!!");
e.printStackTrace();
}
catch (ParseException e) {
System.out.println("ERROR at readCSVFile(): ParseException!!");
e.printStackTrace();
}
}
static void insertToDatabase(Connection connection, String tableName, ArrayList <String> rowDataList) throws SQLException, ParseException {
String tableIdName = tableName;
if (tableIdName.charAt(tableIdName.length()-1) == 's')
tableIdName = tableIdName.substring(0, tableIdName.length()-1);
//To read row
String rowID = rowDataList.get(0);
String selectSQL = "SELECT * FROM " +tableName +" "
+"WHERE " +tableIdName +"_ID = " +rowID;
Statement statement = connection.createStatement();
ResultSet result = statement.executeQuery(selectSQL);
boolean value = result.next();
//INSERT # UPDATE row
if (value == true) { //Update Row if the data is already existed
updateStatementt(tableName, connection, rowDataList);
countUpdatedRow++;
}
else { //Insert New Row
insertStatementt(tableName, connection, rowDataList);
countNewRow++;
}
}
//Method to insert data to the database
static void insertStatementt(String tableType, Connection conn, ArrayList <String> rowDataList) throws SQLException, ParseException {
//Generate Question Mark
String generateQuestionMark = null;
if(rowDataList.size() == 1)
generateQuestionMark = "?";
else
generateQuestionMark = "?, ";
for(int i=1; i<rowDataList.size(); i++) {
if(i!=rowDataList.size()-1)
generateQuestionMark += "?, ";
else
generateQuestionMark += "?";
}
//Insert sql
String sql = "INSERT INTO " +tableType +" VALUES (" +generateQuestionMark +")";
PreparedStatement insertStatement = conn.prepareStatement(sql);
//Insert data
//There are other 'if' and 'else if' statements here for other tables
else if (tableType.equals("Tickets")) {
int ticketID = Integer.parseInt(rowDataList.get(0));
int movieId = Integer.parseInt(rowDataList.get(1));
int theaterId = Integer.parseInt(rowDataList.get(2));
String[] date = rowDataList.get(3).split("/");
String dateString = date[2] +"-" +date[1] +"-" +date[0];
Date showDate = Date.valueOf(dateString);
int showTimeId = Integer.parseInt(rowDataList.get(4));
int cptId = Integer.parseInt(rowDataList.get(5));
int pcId = Integer.parseInt(rowDataList.get(6));
float amountPaid = Float.parseFloat(rowDataList.get(7));
int year = Integer.parseInt(rowDataList.get(8));
String month = rowDataList.get(9);
insertStatement.setInt(1, ticketID);
insertStatement.setInt(2, movieId);
insertStatement.setInt(3, theaterId);
insertStatement.setDate(4, showDate);
insertStatement.setInt(5, showTimeId);
insertStatement.setInt(6, cptId);
insertStatement.setInt(7, pcId);
insertStatement.setFloat(8, amountPaid);
insertStatement.setInt(9, year);
insertStatement.setString(10, month);
}
insertStatement.executeUpdate();
insertStatement.close();
}
//Method to update the data from the database
static void updateStatementt(String tableType, Connection conn, ArrayList <String> rowDataList) throws SQLException {
Statement statement = conn.createStatement();
String sql = "UPDATE " +tableType;
//There are other 'if' and 'else if' statements here for other tables
else if (tableType.equals("Tickets")) {
String[] date = rowDataList.get(3).split("/");
String dateString = date[2] +"-" +date[1] +"-" +date[0];
sql += " SET movie_id = " +rowDataList.get(1) +","
+ " theater_id = " +rowDataList.get(2) +","
+ " showdate = \"" +dateString +"\","
+ " showtime_id = " +rowDataList.get(4) +","
+ " costperticket_id = " +rowDataList.get(5) +","
+ " personcategory_id = " +rowDataList.get(6) +","
+ " amount_paid = " +rowDataList.get(7) +","
+ " year = " +rowDataList.get(8) +","
+ " month = \"" +rowDataList.get(9) +"\""
+ " WHERE ticket_id = " +rowDataList.get(0);
}
statement.executeUpdate(sql);
}
}

For short, read a single line and do whatever you want to do with it. You don't have enough memory for all 700k lines.

You should add statement.close() for the update Statement.

If you really want to read all this data into the Java heap, increase the heap size using, for example, the -Xmx command-line switch. Because of the way textual data is encoded in the JVM, you'll probably need much more heap that the total data size would suggest.
In addition, there might be some places in your code where you can take the strain off the JVM's memory management system. For example, concatenating strings using "+" can generate a lot of temporary data, which will increase the load on the garbage collector. Assembling strings using a StringBuilder might be a simple, less resource-hungry, alternative.

Java Format Timestamp

I have below Java code to convert string format to Timestamp object
public class TestUtil{
Object result;
Public Object convertFormat(String format, String value, String type){
String format = "yyyyMMddHHmmss";
String value = "20050225144824";
SimpleDateFormat dformat = new SimpleDateFormat(format);
java.util.Date date = dformat.parse(value);
result = new Timestamp(date.getTime);
System.out.println("Result::"+ result);
}
}
Expected outcome:
I was expecting the outcome should be like below
20050225144824
Actual outcome:
2005-02-25 14:48:24.0
Could anyone tell me what I am missing here? To get "20050225144824" this result

The below code runs fine for me.
Adding few print statements to explain the different behaviors.
import java.util.Date;
import java.text.SimpleDateFormat;
import java.sql.Timestamp;
public class Main
{
public static void main(String[] args) {
String myFormat = "yyyyMMddHHmmss";
String value = "20050225144824";
try {
SimpleDateFormat dformat = new SimpleDateFormat("yyyyMMddHHmmss");
Date date = dformat.parse(value);
Timestamp ts = new Timestamp(date.getTime());
Object result = new Timestamp(date.getTime());
System.out.println("Timestamp Format with yyyyMMddHHmmss : " +dformat.format(ts));
System.out.println("Object Format with yyyyMMddHHmmss : " +result);
System.out.println("Object Format with yyyyMMddHHmmss : " +dformat.format(result));
} catch(Exception e) {
e.printStackTrace();
}
}
}
Here is the output of the different behaviors :
Timestamp Format with yyyyMMddHHmmss : 20050225144824
Object Format with yyyyMMddHHmmss : 2005-02-25 14:48:24.0
Object Format with yyyyMMddHHmmss : 20050225144824

If you expect Timestamp to return your custom output then you need to override the default Timestamp library.
Here I create CustomTimestamp.java to extend Timestamp and override its toString() method. I modified the changes according to your requirement.
public class CustomTimestamp extends Timestamp {
private int nanos;
public CustomTimestamp(long time) {
super(time);
}
#Override
public String toString () {
int year = super.getYear() + 1900;
int month = super.getMonth() + 1;
int day = super.getDate();
int hour = super.getHours();
int minute = super.getMinutes();
int second = super.getSeconds();
String yearString;
String monthString;
String dayString;
String hourString;
String minuteString;
String secondString;
String nanosString;
String zeros = "000000000";
String yearZeros = "0000";
StringBuffer timestampBuf;
if (year < 1000) {
// Add leading zeros
yearString = "" + year;
yearString = yearZeros.substring(0, (4-yearString.length())) +
yearString;
} else {
yearString = "" + year;
}
if (month < 10) {
monthString = "0" + month;
} else {
monthString = Integer.toString(month);
}
if (day < 10) {
dayString = "0" + day;
} else {
dayString = Integer.toString(day);
}
if (hour < 10) {
hourString = "0" + hour;
} else {
hourString = Integer.toString(hour);
}
if (minute < 10) {
minuteString = "0" + minute;
} else {
minuteString = Integer.toString(minute);
}
if (second < 10) {
secondString = "0" + second;
} else {
secondString = Integer.toString(second);
}
if (nanos == 0) {
nanosString = "";
} else {
nanosString = Integer.toString(nanos);
// Add leading zeros
nanosString = zeros.substring(0, (9-nanosString.length())) +
nanosString;
// Truncate trailing zeros
char[] nanosChar = new char[nanosString.length()];
nanosString.getChars(0, nanosString.length(), nanosChar, 0);
int truncIndex = 8;
while (nanosChar[truncIndex] == '0') {
truncIndex--;
}
nanosString = new String(nanosChar, 0, truncIndex + 1);
}
// do a string buffer here instead.
timestampBuf = new StringBuffer(20+nanosString.length());
timestampBuf.append(yearString);
timestampBuf.append(monthString);
timestampBuf.append(dayString);
timestampBuf.append(hourString);
timestampBuf.append(minuteString);
timestampBuf.append(secondString);
timestampBuf.append(nanosString);
return (timestampBuf.toString());
}
}
Your main class should use CustomTimestamp to get the output
try {
String format = "yyyyMMddHHmmss";
String value = "20050225144824";
SimpleDateFormat dformat = new SimpleDateFormat(format);
java.util.Date date;
date = dformat.parse(value);
Timestamp result = new CustomTimestamp(date.getTime());
System.out.println("Result::" + result);
} catch (ParseException e) {
e.printStackTrace();
}

MysqlDataTruncation : Java evaluates SQL Date object as an expression

I need to insert Date to my database, so I convert date received from user to SQL Date and then try to insert date to database. But in process SQL date object is evaluate to integer and thus insertion fails .
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
String enrollNo = request.getParameter("enroll_no");
String name = request.getParameter("name");
String age = request.getParameter("age");
java.util.Date date = null;
try {
date = new SimpleDateFormat("yyyy-MM-dd").parse(request.getParameter("dob"));
} catch (ParseException e) {
e.printStackTrace();
}
Date SqlDate = new Date(date.getTime());
process(enrollNo, name, age, SqlDate);
response.sendRedirect("listing.jsp");
}
void process(String enrollNo, String name, String age, Date date) {
DatabaseOperations db = new DatabaseOperations("java", "root", "", "jsp");
db.insert(enrollNo, name, age, date);
}
Insert method of DatabaseOperations class
public int insert(Object...objects) {
int rowsAffected = 0;
String query = "INSERT INTO " + tableName + " VALUES (";
for(int i=0; i<objects.length-1; i++)
query += "'" + objects[i] + "',";
query += objects[objects.length-1] + ")";
try {
rowsAffected = statement.executeUpdate(query);
} catch (SQLException e) {
printSQLExceptions(e);
}
return rowsAffected;
}
While inserting date 11/02/2000 following error shows up
com.mysql.jdbc.MysqlDataTruncation: Data truncation: Incorrect date value: '1987' for column 'Dob' at row 1

How to Disable date ranges in java calendar

In my java project i want to disable a range of dates in the java calendar and could not be successful. I'm using Netbeans as my IDE and JCalendar. Below is my code. Any help would be appreciated.
ArrayList<JSONObject> arrays = new ArrayList<JSONObject>();
for (int i = 0; i < n; i++) {
JSONObject another_json_object = vacation_home_booking_data.getJSONObject(i);
JSONObject[] jsons = new JSONObject[arrays.size()];
arrays.toArray(jsons);
String id = another_json_object.getString("id");
String vh_id = another_json_object.getString("vh_id");
String check_in = another_json_object.getString("check_in");
String check_out = another_json_object.getString("check_out");
String status = another_json_object.getString("status");
//creating two arrays of checking and checkout
//check_in_arr[i] = another_json_object.getString("check_in");
//check_out_arr[i] = another_json_object.getString("check_out");
System.out.println("ID is " + id + "vh id is " + vh_id + "check in is " + check_in + "check out is " + check_out);
DateFormat formatter = new SimpleDateFormat("yyyy-mm-dd");
try {
Date date1 = formatter.parse(check_in);
Date date2 = formatter.parse(check_out);
jCalendar1.setSelectableDateRange(date1, date2);
jCalendar1.setBackground(Color.yellow);
//jCalendar1.setSelectedDate();
} catch (ParseException ex) {
Logger.getLogger(Calender.class.getName()).log(Level.SEVERE, null, ex);
ex.printStackTrace();
}
}

Please see, if the below methods works for you:
private DateChooserCombochooser; // Initialize this somewhere
public void setMaxDate(Calendar aDate) {
chooser.setMaxDate(aDate);
}
public void setMinDate(Calendar aDate) {
chooser.setMinDate(aDate);
}
Alternatively, try using setDefaultPeriods(PeriodSet periods) method in the API.

Extract dates from web page

I want to extract dates with different formats out of web pages. I am using the Selenium2 Java API to interact with the browser. Also i use jQuery to further interact with the document. So, solutions for both layers are welcome.
Dates can have very different formats in different locales. Also, month names can be written as text or as number. I need to match as much dates as possible, and I am aware of the fact that there are many combinations.
For example if I have a HTML element like this:
<div class="tag_view">
Last update: May,22,2011
View :40
</div>
I want that the relevant part of the date is extracted and recognized:
May,22,2011
This should now be converted to a regular Java Date object.
Update
This should work with the HTML from any web page, the date can be contained in any element in any format. For example here on Stackoverflow the source code looks like this:
<span class="relativetime" title="2011-05-13 14:45:06Z">May 13 at 14:45</span>
I want it to be done the most effective way and i guess this would be a jQuery selector or filter which returns a standardized date representation. But I am open to your suggestions.

Since we can't limit ourselves to any specific element type or children of any element, you're basically talking about searching the whole page's text for dates. The only way to do this with any kind of efficiency is to use regular expressions. Since you're looking for dates in any format, you need a regex for each acceptable format. Once you define what those are, just compile the regexes and run something like:
var datePatterns = new Array();
datePatterns.push(/\d\d\/\d\d\/\d\d\d\d/g);
datePatterns.push(/\d\d\d\d\/\d\d\/\d\d/g);
...
var stringToSearch = $('body').html(); // change this to be more specific if at all possible
var allMatches = new Array();
for (datePatternIndex in datePatterns){
allMatches.push(stringToSearch.match(datePatterns[datePatternIndex]));
}
You can find more date regexes by googling around, or make them yourself, they're pretty easy. One thing to note: You could probably combine some regexes above to create a more efficient program. I'd be very careful with that, it could cause your code to become hard to read very quickly. Doing one regex per date format seems much cleaner.

You could consider using getText to get element text and then split the String, like -
String s = selenium.getText("css=span.relativetime");
String date = s.split("Last update:")[1].split("View :")[0];

I will answer this myself because i came up with a working solution. I appreciate comments though.
/**
* Extract date
*
* #return Date object
* #throws ParseException
*/
public Date extractDate(String text) throws ParseException {
Date date = null;
boolean dateFound = false;
String year = null;
String month = null;
String monthName = null;
String day = null;
String hour = null;
String minute = null;
String second = null;
String ampm = null;
String regexDelimiter = "[-:\\/.,]";
String regexDay = "((?:[0-2]?\\d{1})|(?:[3][01]{1}))";
String regexMonth = "(?:([0]?[1-9]|[1][012])|(Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Sept|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?))";
String regexYear = "((?:[1]{1}\\d{1}\\d{1}\\d{1})|(?:[2]{1}\\d{3}))";
String regexHourMinuteSecond = "(?:(?:\\s)((?:[0-1][0-9])|(?:[2][0-3])|(?:[0-9])):([0-5][0-9])(?::([0-5][0-9]))?(?:\\s?(am|AM|pm|PM))?)?";
String regexEndswith = "(?![\\d])";
// DD/MM/YYYY
String regexDateEuropean =
regexDay + regexDelimiter + regexMonth + regexDelimiter + regexYear + regexHourMinuteSecond + regexEndswith;
// MM/DD/YYYY
String regexDateAmerican =
regexMonth + regexDelimiter + regexDay + regexDelimiter + regexYear + regexHourMinuteSecond + regexEndswith;
// YYYY/MM/DD
String regexDateTechnical =
regexYear + regexDelimiter + regexMonth + regexDelimiter + regexDay + regexHourMinuteSecond + regexEndswith;
// see if there are any matches
Matcher m = checkDatePattern(regexDateEuropean, text);
if (m.find()) {
day = m.group(1);
month = m.group(2);
monthName = m.group(3);
year = m.group(4);
hour = m.group(5);
minute = m.group(6);
second = m.group(7);
ampm = m.group(8);
dateFound = true;
}
if(!dateFound) {
m = checkDatePattern(regexDateAmerican, text);
if (m.find()) {
month = m.group(1);
monthName = m.group(2);
day = m.group(3);
year = m.group(4);
hour = m.group(5);
minute = m.group(6);
second = m.group(7);
ampm = m.group(8);
dateFound = true;
}
}
if(!dateFound) {
m = checkDatePattern(regexDateTechnical, text);
if (m.find()) {
year = m.group(1);
month = m.group(2);
monthName = m.group(3);
day = m.group(3);
hour = m.group(5);
minute = m.group(6);
second = m.group(7);
ampm = m.group(8);
dateFound = true;
}
}
// construct date object if date was found
if(dateFound) {
String dateFormatPattern = "";
String dayPattern = "";
String dateString = "";
if(day != null) {
dayPattern = "d" + (day.length() == 2 ? "d" : "");
}
if(day != null && month != null && year != null) {
dateFormatPattern = "yyyy MM " + dayPattern;
dateString = year + " " + month + " " + day;
} else if(monthName != null) {
if(monthName.length() == 3) dateFormatPattern = "yyyy MMM " + dayPattern;
else dateFormatPattern = "yyyy MMMM " + dayPattern;
dateString = year + " " + monthName + " " + day;
}
if(hour != null && minute != null) {
//TODO ampm
dateFormatPattern += " hh:mm";
dateString += " " + hour + ":" + minute;
if(second != null) {
dateFormatPattern += ":ss";
dateString += ":" + second;
}
}
if(!dateFormatPattern.equals("") && !dateString.equals("")) {
//TODO support different locales
SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormatPattern.trim(), Locale.US);
date = dateFormat.parse(dateString.trim());
}
}
return date;
}
private Matcher checkDatePattern(String regex, String text) {
Pattern p = Pattern.compile(regex, Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
return p.matcher(text);
}