We Keep Coding

Ideas to optimize this data comparison from base to base?

I wrote a little java (jdk1.8) class to compare data between two databases : one request get rows from source DB, then multitasks search each row in target DB (each thread get its own connection and preparedstatement).
The code below is ok (I just remove some logs, passwords, and some catch exceptions), for example I compare 28.700.000 rows, for a table with 140 columns, from DB2 to same table in Oracle, in 3 hours. Not incredible, but correct, I think ;).
So if you have some ideas to optimize this code, I take ideas :)
public class CompareDB {
public static AtomicInteger nbRowEqual = new AtomicInteger();
public static AtomicInteger nbRowNotFound = new AtomicInteger();
public static AtomicInteger nbRowDiff = new AtomicInteger();
public static AtomicInteger nbRowErr = new AtomicInteger();
public static List<PreparedStatement> targetPstmtPool = new ArrayList<PreparedStatement>();
public static final String[] columns = {"id", "col1", "col2", "col3", "col4"};
#SuppressWarnings({ "unchecked", "rawtypes" })
public static void main(String[] args) throws Exception {
int MAX_THREAD = 200, position = 0;
Integer targetPstmtNumber;
List<Connection> targetConnectionPool = new ArrayList<Connection>();
long beginExecTime = System.currentTimeMillis(), startReqTime;
DateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
// Create connection + preparedStatement pools to target DB
System.out.println("Start opening connections");
for (int i=0; i<MAX_THREAD; i++) {
Connection targetConn = DriverManager.getConnection("jdbc:oracle:thin:#<host>:<port>:<dbname>", "<user>", "<password>");
targetConn.setAutoCommit(false);
targetConn.setReadOnly(true);
targetConnectionPool.add(targetConn);
PreparedStatement targetPstmt = targetConn.prepareStatement("select "+String.join(",", columns)+" from mytable where id=?");
targetPstmtPool.add(targetPstmt);
}
// Connect + select from source DB
Connection sourceConn = DriverManager.getConnection("jdbc:db2://<IP>:<port>/<dbname>", "<user>", "<password>");
sourceConn.setAutoCommit(false);
sourceConn.setReadOnly(true);
Statement sourceStmt = sourceConn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
ResultSet sourceResult = sourceStmt.executeQuery("select "+String.join(",", columns)+ " from owner.mytable");
System.out.println("Connections and statements opened in "+(System.currentTimeMillis()-beginExecTime)+" ms");
// Open pool of threads
ThreadPoolExecutor executor = new ThreadPoolExecutor(10, 20, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue());
CompletionService<Integer> completion = new ExecutorCompletionService<Integer>(executor);
System.out.println("------------------------------------------------------------");
System.out.println("Start compare data ("+MAX_THREAD+" parallel threads)");
System.out.println();
startReqTime = System.currentTimeMillis();
// For each row of the source request
while (sourceResult.next()) {
// Set which pstmt to give to thread
if (position < MAX_THREAD) targetPstmtNumber = new Integer(position);
else targetPstmtNumber = completion.take().get();
// extract values from resultSet as parameter for next thread
List<Object> sourceRow = new ArrayList<Object>();
for (int i=1; i<=columns.length; i++) {
sourceRow.add(sourceResult.getObject(i));
}
// Call thread
completion.submit(new CompareDbThread(sourceRow, targetPstmtNumber));
position++;
if (position % 10000 == 0) System.out.println(" -- "+position+" rows --");
else if (position % 100 == 0) System.out.print(".");
}
// Await for last threads
System.out.println();
System.out.println("Waiting last threads...");
executor.awaitTermination(5, TimeUnit.SECONDS);
executor.shutdown();
// Close all Rs, Stmt, Conn
try {
System.out.println("------------------------------------------------------------");
System.out.println("Close all requests, statements, connections");
for (PreparedStatement targetPstmt : targetPstmtPool) targetPstmt.close();
for (Connection targetConn : targetConnectionPool) targetConn.close();
sourceResult.close();
sourceStmt.close();
sourceConn.close();
} catch (Exception e) {
System.out.println("[INFO] Error closing connections and requests : "+e.getMessage());
}
System.out.println("------------------------------------------------------------");
System.out.println("Data comparison done in "+(dateFormat.format(new Date(System.currentTimeMillis()-startReqTime)))
+" : "+nbRowEqual+" equals, "+nbRowNotFound+" not found, "+nbRowDiff+" diff, "+nbRowErr+" ERROR rows");
System.out.println("Threads : getCompletedTaskCount() = "+executor.getCompletedTaskCount()+", getTaskCount() = "+executor.getTaskCount());
System.out.println("Total time : "+(dateFormat.format(new Date(System.currentTimeMillis()-beginExecTime))));
}
}
public class CompareDbThread implements Callable<Integer> {
protected List<Object> sourceRow;
protected Integer targetPstmtNumber;
public CompareDbThread(List<Object> sourceRow, Integer targetPstmtNumber) {
this.sourceRow = sourceRow;
this.targetPstmtNumber = targetPstmtNumber;
}
public Integer call() {
ResultSet targetResult;
Object sourceColumnValue, targetColumnValue;
String sSourceColumnValue, sTargetColumnValue;
Double dSourceColumnValue, dTargetColumnValue;
boolean equalRow=true, equalColumn=true;
String message="", tempMessage="";
try {
PreparedStatement targetPstmt = CompareDB.targetPstmtPool.get(targetPstmtNumber.intValue());
targetPstmt.setObject(1, sourceRow.get(0));
targetResult = targetPstmt.executeQuery();
if (targetResult.next()) {
// Compare each column
for (int i=0; i<CompareDB.columns.length; i++) {
sourceColumnValue = sourceRow.get(i);
targetColumnValue = targetResult.getObject(i+1);
if ((sourceColumnValue!=null && targetColumnValue==null) || (sourceColumnValue==null && targetColumnValue!=null)) {
equalRow=false;
message += CompareDB.columns[i] + " : " + targetColumnValue + "(au lieu de " + sourceColumnValue + "), ";
}
else if (sourceColumnValue!=null && targetColumnValue!=null) {
// Compare as objects ...
if (!sourceColumnValue.equals(targetColumnValue)) {
sSourceColumnValue=sourceColumnValue.toString();
sTargetColumnValue=targetColumnValue.toString();
// if differents, compare as string ...
if (!sSourceColumnValue.equals(sTargetColumnValue)) {
tempMessage = CompareDB.columns[i] + " [String] : " + sTargetColumnValue + "(instead of " + sSourceColumnValue + "), ";
// if differents as string, compare as double
try {
dSourceColumnValue = new Double(sSourceColumnValue);
dTargetColumnValue = new Double(sTargetColumnValue);
if (!dSourceColumnValue.equals(dTargetColumnValue)) {
tempMessage = CompareDB.columns[i] + " [Number] : " + dTargetColumnValue + "(instead of " + dSourceColumnValue + "), ";
equalColumn=false;
}
}
catch (NumberFormatException e) {
equalColumn=false;
}
}
}
if (!equalColumn) {
message += tempMessage;
tempMessage = "";
equalColumn=true;
equalRow=false;
}
}
}
if (equalRow) {
CompareDB.nbRowEqual.incrementAndGet();
}
else {
CompareDB.nbRowDiff.incrementAndGet();
System.out.println(" [DIFFERENT] ["+CompareDB.columns[CompareDB.ID_COLUMN_POS]+"="+sourceRow.get(CompareDB.ID_COLUMN_POS)+"] => "+message);
}
}
else {
CompareDB.nbRowNotFound.incrementAndGet();
System.out.println(" [NOT FOUND] ["+CompareDB.columns[CompareDB.ID_COLUMN_POS]+"="+sourceRow.get(CompareDB.ID_COLUMN_POS)+"]");
}
}
catch (Exception e) {
CompareDB.nbRowErr.incrementAndGet();
System.err.println("[ERROR] "+CompareDB.columns[CompareDB.ID_COLUMN_POS]+"="+sourceRow.get(CompareDB.ID_COLUMN_POS)+" : "+e.getMessage());
}
return targetPstmtNumber;
}
}

How can I draw gantt chart with date from database in java

I have problem in taking from the database the startdate and the finaldate of task to draw the chart
public IntervalCategoryDataset getCategoryDataset() {
conn = ConnectDB.ConnectDB();
TaskSeriesCollection dataset = new TaskSeriesCollection();
String sql = "SELECT `TITRE`, `DATE DEBUT Prévi`, `DATE FIN prévi` FROM `projet`;";
try {
pst = conn.prepareStatement(sql);
rs = pst.executeQuery(sql);
while (rs.next()) {
String a = rs.getString("TITRE");
Date StartDate = rs.getDate("DATE DEBUT Prévi");
Date EndDate = rs.getDate("DATE FIN prévi");
Names.add(a);
Dates.add(StartDate);
Dates.add(EndDate);
++count;
}
} catch (Exception e) {
JOptionPane.showMessageDialog(null, e.getMessage());
}
int j = 0;
int k = 1;
TaskSeries series1 = new TaskSeries("Estimated Date");
for (int i = 0; i < count; i++) {
series1.add(new Task(Names.get(i),
Date.from(LocalDate.of(Dates.get(j).getYear(), Dates.get(j).getMonth(), Dates.get(j).getDay())
.atStartOfDay()
.toInstant(ZoneOffset.UTC)),
Date.from(LocalDate.of(Dates.get(k).getYear(), Dates.get(k).getMonth(), Dates.get(k).getDay())
.atStartOfDay()
.toInstant(ZoneOffset.UTC))));
}
dataset.add(series1);
return dataset;
}
I expect to have for every tasks its chart but I have the same chart for all the tasks.

You should apply a single loop, and in addition either use getObject(index, LocalDate.class) or - if your driver doesn't support the previous option - java.sql.Date.toLocalDate():
public IntervalCategoryDataset getCategoryDataset() {
conn = ConnectDB.ConnectDB();
TaskSeriesCollection dataset = new TaskSeriesCollection();
TaskSeries series1 = new TaskSeries("Estimated Date");
String sql = "SELECT `TITRE`, `DATE DEBUT Prévi`, `DATE FIN prévi` FROM `projet`;";
try {
pst = conn.prepareStatement(sql);
rs = pst.executeQuery(sql);
while (rs.next()) {
String name = rs.getString("TITRE");
LocalDate startDate = rs.getObject("DATE DEBUT Prévi", LocalDate.class);
LocalDate endDate = rs.getObject("DATE FIN prévi", LocalDate.class);
series1.add(new Task(name,
Date.from(startDate.atStartOfDay().toInstant(ZoneOffset.UTC)),
Date.from(endDate.atStartOfDay().toInstant(ZoneOffset.UTC)));
}
} catch (Exception e) {
JOptionPane.showMessageDialog(null, e.getMessage());
}
dataset.add(series1);
return dataset;
}

You always get the same result because the j and k values are stacked in their default values and not incrementing along with the i counter in your for loop. Referring to the code am seeing, i advise you to get rid of the j and k counters, replace the j with i and the k with i+1.
Your for loop should be like the following:
for (int i =0; i<count; i++) {
series1.add(new Task(Names.get(i),
Date.from(LocalDate.of(Dates.get(i).getYear(), Dates.get(i).getMonth(),Dates.get(i).getDay()).atStartOfDay().toInstant(ZoneOffset.UTC)),
Date.from(LocalDate.of(Dates.get(i+1).getYear(),Dates.get(i+1).getMonth(),Dates.get(i+1).getDay()).atStartOfDay().toInstant(ZoneOffset.UTC))
)
);
}

Strings can not be converted to DAO Receiver

I am trying to perform batch insertion operation with a list object but while inserting I am getting String cannot be converted to DAO.The receiver in the iterator loop.
I have tried to list the list object, at that time it is printing values from the list. but, when I use generics are normal list it is showing error and I don't find any solution to insert
From this method I am reading the excel file and storing into list
public List collect(Receiver rec)
{
//ReadFromExcel rd = new ReadFromExcel();
List<String> up = new ArrayList<String>();
//List<String> details = rd.reader();
//System.out.println(details);
try( InputStream fileToRead = new FileInputStream(new File(rec.getFilePath())))
{
XSSFWorkbook wb = new XSSFWorkbook(fileToRead);
wb.setMissingCellPolicy(Row.MissingCellPolicy.RETURN_BLANK_AS_NULL);
XSSFSheet sheet = wb.getSheetAt(0);
DataFormatter fmt = new DataFormatter();
String data ="";
for(int sn = 0;sn<wb.getNumberOfSheets()-2;sn++)
{
sheet = wb.getSheetAt(sn);
for(int rn =sheet.getFirstRowNum();rn<=sheet.getLastRowNum();rn++)
{
Row row = sheet.getRow(rn);
if(row == null)
{
System.out.println("no data in row ");
}
else
{
for(int cn=0;cn<row.getLastCellNum();cn++)
{
Cell cell = row.getCell(cn);
if(cell == null)
{
// System.out.println("no data in cell ");
// data = data + " " + "|";
}
else
{
String cellStr = fmt.formatCellValue(cell);
data = data + cellStr + "|";
}
}
}
}
}
up = Arrays.asList(data.split("\\|"));
// System.out.println(details);
}
catch (FileNotFoundException ex)
{
Logger.getLogger(BImplementation.class.getName()).log(Level.SEVERE, null, ex);
}
catch (IOException ex)
{
Logger.getLogger(BImplementation.class.getName()).log(Level.SEVERE, null, ex);
}
Iterator iter = up.iterator();
while(iter.hasNext())
{
System.out.println(iter.next());
}
String row="";
Receiver info = null;
String cid = "";
String cname = "";
String address = "";
String mid = "";
boolean b = false;
List<Receiver> res = new ArrayList<Receiver>();
int c = 0;
try
{
String str = Arrays.toString(up.toArray());
//System.out.println(str);
String s = "";
s = s + str.substring(1,str.length());
// System.out.println("S:"+s);
StringTokenizer sttoken = new StringTokenizer(s,"|");
int count = sttoken.countTokens();
while(sttoken.hasMoreTokens())
{
if(sttoken.nextToken() != null)
{
// System.out.print(sttoken.nextToken());
cid = sttoken.nextToken();
cname = sttoken.nextToken();
address = sttoken.nextToken();
mid = sttoken.nextToken();
info = new Receiver(cid,cname,address,mid);
res.add(info);
System.out.println("cid :"+cid+ " cname : "+cname +" address : "+address+" mid : "+mid);
c = res.size();
// System.out.println(c);
}
else
{
break;
}
}
System.out.println(count);
// System.out.println("s");
}
catch(NoSuchElementException ex)
{
System.out.println("No Such Element Found Exception" +ex);
}
return up;
}
with this method I'm trying to insert into database
public boolean insert(List res)
{
String sqlQuery = "insert into records(c_id) values (?)";
DBConnection connector = new DBConnection();
boolean flag = false;
// Iterator itr=res.iterator();
// while(it.hasNext())
// {
// System.out.println(it.next());
// }
try( Connection con = connector.getConnection();)
{
con.setAutoCommit(false);
PreparedStatement pstmt = con.prepareStatement(sqlQuery);
Iterator it = res.iterator();
while(it.hasNext())
{
Receiver rs =(Receiver) it.next();
pstmt.setString(1,rs.getcID());
pstmt.setString(2,rs.getcName());
pstmt.setString(3,rs.getAddress());
pstmt.setString(4,rs.getMailID());
pstmt.addBatch();
}
int [] numUpdates=pstmt.executeBatch();
for (int i=0; i < numUpdates.length; i++)
{
if (numUpdates[i] == -2)
{
System.out.println("Execution " + i +": unknown number of rows updated");
flag=false;
}
else
{
System.out.println("Execution " + i + "successful: " + numUpdates[i] + " rows updated");
flag=true;
}
}
con.commit();
} catch(BatchUpdateException b)
{
System.out.println(b);
flag=false;
}
catch (SQLException ex)
{
Logger.getLogger(BImplementation.class.getName()).log(Level.SEVERE, null, ex);
System.out.println(ex);
flag=false;
}
return flag;
}
I want to insert list object using JDBC batch insertion to the database.

Your method collect(Receiver rec) returns the List of strings called up.
return up;
However (if you are really using the method collect to pass the List into insert(List res) method), you are expecting this list to contain Receiver objects. Which is incorrect, since it collect(..) returns the list of Strings.
And that causes an error when you try to cast Receiver rs =(Receiver) it.next();
You need to review and fix your code, so you will pass the list of Receiver objects instead of strings.
And I really recommend you to start using Generics wherever you use List class. In this case compiler will show you all data-type errors immediately.

list.isEmpty() or list.size() not working in an if loop in java

I am using java for sending streams at given rate(say 100 events/sec) via UDP to another receiver program. The receiver program has 2 threads. Thread 1 appends the values to List and another thread is checking is the list has some element and perform some action on it.
Earlier, I have been using a queue instead of a list. I was having issues with Iterator thread while checking if the queue has some element or not. It's a wired problem, I may be making some silly mistake. For this reason, I decided to use List, but I am having same issues now.
Can someone please tell me what am I doing wrong?
Code of Sender Program is
public class simpleGen {
public static void main(String[] args) throws IOException, InterruptedException {
Integer arrival_rate = 1;
Integer sleep_time = 1000/arrival_rate;
Long currentTime;
Integer value = null;
Integer Sensor_id;
Integer Patient_id;
Integer uid = 0;
Long count = 0L;
Integer time_in_sec = 60*2 ;
Integer lower_bound = 10;
Integer upper_bound = 20;
Long start_time = System.currentTimeMillis();
Long end_time = start_time + (1000 * time_in_sec);
int server_port = 8000;
DatagramSocket s = new DatagramSocket();
s.setSendBufferSize(2147483647);
InetAddress local = InetAddress.getByName("172.17.195.107");
while (System.currentTimeMillis() < end_time) {
uid = 1;
count += 1;
Random random = new Random(System.currentTimeMillis());
Patient_id = 1;
Sensor_id = 1;
currentTime = System.nanoTime();
value = lower_bound + random.nextInt((upper_bound - lower_bound) + 1);
Event event = new Event(Patient_id, Sensor_id, uid, currentTime, value);
String messageStr = event.toString();
// System.out.println(messageStr);
int msg_length = messageStr.length();
byte[] message = messageStr.getBytes();
DatagramPacket p = new DatagramPacket(message, msg_length, local, server_port);
s.send(p);
System.out.print(" \r Sensor 1 count = " + count );
System.out.flush();
Thread.sleep(sleep_time );
}
Float inpt_rate = Float.valueOf(count)/time_in_sec;
System.out.println(" \n Average output rate = " + inpt_rate + " events/second" );
}
}
Code of Receiver Program is
public class Simplereceiver {
public static final String ANSI_RED = "\u001B[31m";
public static final String ANSI_BLUE = "\u001B[34m";
public static final String ANSI_PURPLE = "\u001B[35m";
public static void main(String args[]) throws Exception
{
Queue<String> queue = new LinkedList<String>();
List<String> list = new ArrayList<String>();
Thread iterating_thread = new Thread(){
#Override
public void run() {
System.out.println( ANSI_BLUE + " iterating_thread started");
Boolean running = true ;
while(running){
if(list.size() > 0){
System.out.println("has element ----");
System.out.println(list.size());
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}};
Thread receiving_thread = new Thread(){
#Override
public void run() {
Integer total_re_ECG = 0;
String[] sensor_value = new String[10];
String[] parsed_tuple = new String[10];
boolean run = true;
DatagramSocket udpSocket = null;
DatagramPacket packet = null;
try {
udpSocket = new DatagramSocket(8000);
udpSocket.setReceiveBufferSize(2147483647);
} catch (SocketException e) {
e.printStackTrace();
}
byte[] message = new byte[8000];
packet = new DatagramPacket(message, message.length);
System.out.println( ANSI_RED + " receiving_thread started");
while (run) {
try {
// Block until the host receives a UDP packet.
udpSocket.receive(packet);
String text = new String(message, 0, packet.getLength());
String[] tupleData = text.split(",");
int i = 0;
for (String tuple : tupleData) {
String[] tupleValue = tuple.split("=");
sensor_value[i] = tupleValue[1];
i += 1;
}// key pair of tuple ends
total_re_ECG += 1;
Integer patient_id = Integer.valueOf(sensor_value[0]);
Integer sensor_id = Integer.valueOf(sensor_value[1]);
Integer tuple_id = Integer.valueOf(sensor_value[2]);
Long generation_time = Long.valueOf(sensor_value[3]);
Float sensor1_data = Float.valueOf(sensor_value[4]);
Long event_arrival_time = System.nanoTime();
// System.out.println(event_arrival_time);
parsed_tuple[0] = total_re_ECG + "," + event_arrival_time ;
// queue.add(String.valueOf(sensor1_data));
list.add(String.valueOf(sensor1_data));
System.out.println("packet added is = " + parsed_tuple[0]);
} catch (IOException e) {
run = false;
}//catch
}//while
}
};
receiving_thread.start();
iterating_thread.start();
}//main
}//class

Using LinkedBlockingQueue solved my issue
Updated code for Iterator Thread is
Thread iterating_thread = new Thread(){
#Override
public void run() {
System.out.println( ANSI_BLUE + " iterating_thread started");
Boolean running = true ;
while(running){
if(!linkedBlockingQueue.isEmpty()){
System.out.println(linkedBlockingQueue.element());
linkedBlockingQueue.remove();
}
}
}};

J2ME , Quizz using choiceGroups

I am working on a driving licence project on j2Me wich is including Tests like quizz , well and i am having a problem after parsing the questions and moving them into choiceGroups just like that :
if (questions.length > 0) {
for (int i = 0; i < questions.length; i++) {
ChoiceGroup reponses = new ChoiceGroup("Reponses" + i, Choice.EXCLUSIVE);
reponses.append(questions[i].getReponse1(), null);
reponses.append(questions[i].getReponse2(), null);
reponses.append(questions[i].getReponse3(), null);
pass.append(questions[i].getContenu());
pass.append(reponses);
}
}
} catch (Exception e) {
System.out.println("Exception:" + e.toString());
}
disp.setCurrent(pass);
and the next step is the command who's controlling the choiceGroups to test them if they are like the true answer or not .
so i am blocked here .
if (c == valider) {
int result = 0;
for (int i = 0; i < pass.size(); i++) {
String ch = pass.get(i).getLabel();
System.out.println(ch);
}
}
I don't know how to get the choice from the choicegroup
any help

Actually, I am not sure what totally you want for:
This code will help you get selected items from choicegroup that i did long time before:
//get a selected array in choicegroup
private String[] choiceGroupSelected(ChoiceGroup cg) {
String selectedArray[] = new String[cg.size()];
int k = 0;
for (int i = 0; i < cg.size(); i++) {
if (cg.isSelected(i)) {
selectedArray[k] = cg.getString(i);
k++;
}
}
return selectedArray;
}
That function will help me get all selected items for deleting action below:
private void deleteSpecificItem() {
try {
String temp = null;
int index;
//get ChoiceGroup size
int numbers = cgTrip.size();
String selectedItems[] = choiceGroupSelected(cgTrip);
//
rs = services.RecordStoreManager.openRecordStoreByName("TripRS");
re = rs.enumerateRecords(null, null, true);
String[] tripList = new String[2];
for (int i = 0; i < numbers; i++) {
temp = selectedItems[i];
if (temp != null) {
while (re.hasNextElement()) {
try {
index = re.nextRecordId();
System.out.println("RecordID: " + index);
byte[] byteBuff = rs.getRecord(index);
String source = new String(byteBuff);
tripList = services.StringManager.getItems(source, ";", 2);
String strProcess = tripList[0] + "-" + tripList[1];
//inspect all of items in choicegroup and if they are selecting then compare with record
//If comparison is true then delete this record
if (temp.equals(strProcess)) {
System.out.println("Delete RecordID: " + index);
rs.deleteRecord(index);
re.keepUpdated(true);
break;
}
} catch (RecordStoreException ex) {
ex.printStackTrace();
}
}
}
}
try {
rs.closeRecordStore();
} catch (RecordStoreException ex) {
ex.printStackTrace();
}
rs = null;
re.destroy();
this.LoadTripItem();
} catch (RecordStoreNotOpenException ex) {
ex.printStackTrace();
}
}