Ideas to optimize this data comparison from base to base? - java

I wrote a little java (jdk1.8) class to compare data between two databases : one request get rows from source DB, then multitasks search each row in target DB (each thread get its own connection and preparedstatement).
The code below is ok (I just remove some logs, passwords, and some catch exceptions), for example I compare 28.700.000 rows, for a table with 140 columns, from DB2 to same table in Oracle, in 3 hours. Not incredible, but correct, I think ;).
So if you have some ideas to optimize this code, I take ideas :)
public class CompareDB {
public static AtomicInteger nbRowEqual = new AtomicInteger();
public static AtomicInteger nbRowNotFound = new AtomicInteger();
public static AtomicInteger nbRowDiff = new AtomicInteger();
public static AtomicInteger nbRowErr = new AtomicInteger();
public static List<PreparedStatement> targetPstmtPool = new ArrayList<PreparedStatement>();
public static final String[] columns = {"id", "col1", "col2", "col3", "col4"};
#SuppressWarnings({ "unchecked", "rawtypes" })
public static void main(String[] args) throws Exception {
int MAX_THREAD = 200, position = 0;
Integer targetPstmtNumber;
List<Connection> targetConnectionPool = new ArrayList<Connection>();
long beginExecTime = System.currentTimeMillis(), startReqTime;
DateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
// Create connection + preparedStatement pools to target DB
System.out.println("Start opening connections");
for (int i=0; i<MAX_THREAD; i++) {
Connection targetConn = DriverManager.getConnection("jdbc:oracle:thin:#<host>:<port>:<dbname>", "<user>", "<password>");
targetConn.setAutoCommit(false);
targetConn.setReadOnly(true);
targetConnectionPool.add(targetConn);
PreparedStatement targetPstmt = targetConn.prepareStatement("select "+String.join(",", columns)+" from mytable where id=?");
targetPstmtPool.add(targetPstmt);
}
// Connect + select from source DB
Connection sourceConn = DriverManager.getConnection("jdbc:db2://<IP>:<port>/<dbname>", "<user>", "<password>");
sourceConn.setAutoCommit(false);
sourceConn.setReadOnly(true);
Statement sourceStmt = sourceConn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
ResultSet sourceResult = sourceStmt.executeQuery("select "+String.join(",", columns)+ " from owner.mytable");
System.out.println("Connections and statements opened in "+(System.currentTimeMillis()-beginExecTime)+" ms");
// Open pool of threads
ThreadPoolExecutor executor = new ThreadPoolExecutor(10, 20, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue());
CompletionService<Integer> completion = new ExecutorCompletionService<Integer>(executor);
System.out.println("------------------------------------------------------------");
System.out.println("Start compare data ("+MAX_THREAD+" parallel threads)");
System.out.println();
startReqTime = System.currentTimeMillis();
// For each row of the source request
while (sourceResult.next()) {
// Set which pstmt to give to thread
if (position < MAX_THREAD) targetPstmtNumber = new Integer(position);
else targetPstmtNumber = completion.take().get();
// extract values from resultSet as parameter for next thread
List<Object> sourceRow = new ArrayList<Object>();
for (int i=1; i<=columns.length; i++) {
sourceRow.add(sourceResult.getObject(i));
}
// Call thread
completion.submit(new CompareDbThread(sourceRow, targetPstmtNumber));
position++;
if (position % 10000 == 0) System.out.println(" -- "+position+" rows --");
else if (position % 100 == 0) System.out.print(".");
}
// Await for last threads
System.out.println();
System.out.println("Waiting last threads...");
executor.awaitTermination(5, TimeUnit.SECONDS);
executor.shutdown();
// Close all Rs, Stmt, Conn
try {
System.out.println("------------------------------------------------------------");
System.out.println("Close all requests, statements, connections");
for (PreparedStatement targetPstmt : targetPstmtPool) targetPstmt.close();
for (Connection targetConn : targetConnectionPool) targetConn.close();
sourceResult.close();
sourceStmt.close();
sourceConn.close();
} catch (Exception e) {
System.out.println("[INFO] Error closing connections and requests : "+e.getMessage());
}
System.out.println("------------------------------------------------------------");
System.out.println("Data comparison done in "+(dateFormat.format(new Date(System.currentTimeMillis()-startReqTime)))
+" : "+nbRowEqual+" equals, "+nbRowNotFound+" not found, "+nbRowDiff+" diff, "+nbRowErr+" ERROR rows");
System.out.println("Threads : getCompletedTaskCount() = "+executor.getCompletedTaskCount()+", getTaskCount() = "+executor.getTaskCount());
System.out.println("Total time : "+(dateFormat.format(new Date(System.currentTimeMillis()-beginExecTime))));
}
}
public class CompareDbThread implements Callable<Integer> {
protected List<Object> sourceRow;
protected Integer targetPstmtNumber;
public CompareDbThread(List<Object> sourceRow, Integer targetPstmtNumber) {
this.sourceRow = sourceRow;
this.targetPstmtNumber = targetPstmtNumber;
}
public Integer call() {
ResultSet targetResult;
Object sourceColumnValue, targetColumnValue;
String sSourceColumnValue, sTargetColumnValue;
Double dSourceColumnValue, dTargetColumnValue;
boolean equalRow=true, equalColumn=true;
String message="", tempMessage="";
try {
PreparedStatement targetPstmt = CompareDB.targetPstmtPool.get(targetPstmtNumber.intValue());
targetPstmt.setObject(1, sourceRow.get(0));
targetResult = targetPstmt.executeQuery();
if (targetResult.next()) {
// Compare each column
for (int i=0; i<CompareDB.columns.length; i++) {
sourceColumnValue = sourceRow.get(i);
targetColumnValue = targetResult.getObject(i+1);
if ((sourceColumnValue!=null && targetColumnValue==null) || (sourceColumnValue==null && targetColumnValue!=null)) {
equalRow=false;
message += CompareDB.columns[i] + " : " + targetColumnValue + "(au lieu de " + sourceColumnValue + "), ";
}
else if (sourceColumnValue!=null && targetColumnValue!=null) {
// Compare as objects ...
if (!sourceColumnValue.equals(targetColumnValue)) {
sSourceColumnValue=sourceColumnValue.toString();
sTargetColumnValue=targetColumnValue.toString();
// if differents, compare as string ...
if (!sSourceColumnValue.equals(sTargetColumnValue)) {
tempMessage = CompareDB.columns[i] + " [String] : " + sTargetColumnValue + "(instead of " + sSourceColumnValue + "), ";
// if differents as string, compare as double
try {
dSourceColumnValue = new Double(sSourceColumnValue);
dTargetColumnValue = new Double(sTargetColumnValue);
if (!dSourceColumnValue.equals(dTargetColumnValue)) {
tempMessage = CompareDB.columns[i] + " [Number] : " + dTargetColumnValue + "(instead of " + dSourceColumnValue + "), ";
equalColumn=false;
}
}
catch (NumberFormatException e) {
equalColumn=false;
}
}
}
if (!equalColumn) {
message += tempMessage;
tempMessage = "";
equalColumn=true;
equalRow=false;
}
}
}
if (equalRow) {
CompareDB.nbRowEqual.incrementAndGet();
}
else {
CompareDB.nbRowDiff.incrementAndGet();
System.out.println(" [DIFFERENT] ["+CompareDB.columns[CompareDB.ID_COLUMN_POS]+"="+sourceRow.get(CompareDB.ID_COLUMN_POS)+"] => "+message);
}
}
else {
CompareDB.nbRowNotFound.incrementAndGet();
System.out.println(" [NOT FOUND] ["+CompareDB.columns[CompareDB.ID_COLUMN_POS]+"="+sourceRow.get(CompareDB.ID_COLUMN_POS)+"]");
}
}
catch (Exception e) {
CompareDB.nbRowErr.incrementAndGet();
System.err.println("[ERROR] "+CompareDB.columns[CompareDB.ID_COLUMN_POS]+"="+sourceRow.get(CompareDB.ID_COLUMN_POS)+" : "+e.getMessage());
}
return targetPstmtNumber;
}
}

Related

Is there a way to create a player specific countdown, that stops when the player leaves in Java?

I'm looking for a way to create a player-specific countdown for my BankSystem Plugin in Java.
Currently everybody gets their interest at the same time, because I'm using a Bukkit scheduler.
Bukkit.getScheduler().scheduleSyncRepeatingTask(Main.getPlugin(), new Runnable() {
#Override
public void run() {
try {
Statement stmt = DatabaseManager.getCon().createStatement();
String sql = ("SELECT uuid, money FROM Accounts");
stmt.executeUpdate("USE " + ConfigManager.getConf().getString("Database.DBName"));
ResultSet rs = stmt.executeQuery(sql);
while (rs.next()) {
uids.add(rs.getString(1));
money.put(rs.getString(1), rs.getInt(2));
}
if (!ConfigManager.getConf().getBoolean("Settings.PayInterestOffline")) {
try {
for (String uid : uids) {
Player pl = Bukkit.getPlayer(UUID.fromString(uid));
if (pl == null) {
uids.remove(IndexIdentifier.getIndex(uid, uids));
money.remove(uid);
}
}
} catch (Exception e) {
}
}
for (int i = 0; i < uids.size(); i++) {
try {
String puid = uids.get(i);
double doubleMoney = money.get(puid);
if (doubleMoney > ConfigManager.getConf().getInt("Settings.MaximumMoney")) {
continue;
} else {
doubleMoney = (((doubleMoney / 100) * percent) + doubleMoney);
int intMoney = (int) Math.ceil(doubleMoney);
stmt.executeUpdate("UPDATE Accounts SET money = " + intMoney + " WHERE uuid = '" + puid + "';");
Player p = Bukkit.getPlayer(UUID.fromString(puid));
if (p.isOnline() && p != null) {
p.sendMessage(
"§aYou've credited an interest of §6" + (int) Math.ceil((intMoney / 100) * percent)
+ ".0 " + ConfigManager.getConf().getString("Settings.currency"));
}
}
money.remove(puid);
uids.remove(i);
} catch (NullPointerException e) {
}
}
} catch (SQLException e) {
e.printStackTrace();
}
}
}, 0, period);
Is there a way, to create a countdown for every online player. That means that the countdown stops, when the player leaves the server and resumes after rejoining.
You can associate an integer with a player in a hashmap:
HashMap<UUID, Integer> playersAndTimes = new HashMap<>();
To add players to the HashMap when you want to start the countdown:
playersAndTimes.put(player.getUniqueId(), time)
Now you just need to run this function when the plugin enables which loops through every player online, if they are in the HashMap (have a countdown on them) it will remove 1 every second from their value:
Bukkit.getScheduler().scheduleSyncRepeatingTask(Main.getPlugin(Main.class), new Runnable() {
#Override
public void run() {
for (Player player : Bukkit.getOnlinePlayers()) {
if (playersAndTimes.containsKey(player.getUniqueId())) {
if (playersAndTimes.get(player.getUniqueId()) >= 1) {
playersAndTimes.put(player.getUniqueId(), playersAndTimes.get(player.getUniqueId()) - 1);
} else {
//The Player's Time Has Expired As The Number Associated With Their UUID In The Hashmap Is Now Equal To 0.
//DO SOMETHING
}
}
}
}
}, 0, 20);

Why the place of the option always change?

I'm doing a phone book project in Java, using MySql for school.
I wanted to print the methods using the Class.getDeclaredMethods();
adding them to a Vector of type String.
and invoke a menu() method that prints and accepts the option from the user using Scanner
the problem is that it always changes the methods places.
for example it can print
0.addPerson
1.deleteContact
2.searchByChar
and the next time
0.deleteContact
1.addPerson
2.searchByChar.
the problem is that i have a Switch case depend on it.
the menu function:
public static int menu(Vector<?> options){
System.out.println("The Options: ");
for (int i = 0; i < options.size(); i++) {
System.out.println(i + ". " + options.get(i));
}
Scanner scanner = new Scanner(System.in);
System.out.println("Your Choice: ");
String optionString = scanner.nextLine();
int option = 0;
if(isNumber(optionString)){
option = Integer.valueOf(optionString);
}else{
System.out.println("Please Choose Valid Option");
return menu(options);
}
return option;
}
the methods that get my methods:
public static Vector<String> getClassMethods(Class whichClass){
Method[] methods = whichClass.getDeclaredMethods();
Vector<String> stringMethods = new Vector<>();
for (Method method : methods) {
if(Modifier.toString(method.getModifiers()).equals("protected")){
stringMethods.add(method.getName());
}
}
return stringMethods;
}
my class the connects to the data base:
private boolean getData(Person person){
String sql = "SELECT * FROM " + DB_NAME + " WHERE name = '" + person.getName() + "' and phone_number = '" + person.getPhoneNumber() + "'";
try {
ResultSet resultSet = db.prepareStatement(sql).executeQuery();
if (resultSet.next()) {
return true;
}
} catch (SQLException e) {
System.out.println(e.getMessage());
}
return false;
}
protected void addPerson(){
Person person = MyUtills.createPerson();
if(getData(person)){
System.out.println(person.getName() + ", " + person.getPhoneNumber() + ": Already in Contacts" );
}else{
add(person);
}
}
private void add(Person person) {
String pName = person.getName();
String pPhone = person.getPhoneNumber();
String pAddress = person.getAddress();
String sql = "INSERT INTO " + DB_NAME + " (name,phone_number,address)" +
"VALUES (?,?,?)";
try {
statement = db.prepareStatement(sql);
statement.setString(1,pName);
statement.setString(2,pPhone);
statement.setString(3,pAddress);
statement.execute();
System.out.println("Added Successfully");
} catch (SQLException e) {
e.printStackTrace();
}
}
//delete contact by name
protected void deleteContact(){
System.out.println("Enter Name Please");
String name = MyUtills.readStringFromUser();
Vector<Person> vector = checkMoreThanOne(name);
if(vector.size() > 1){
System.out.println("Choose One To Delete: ");
int option = menu(vector);
delete(vector.get(option));
}
System.out.println("Deleted");
}
private Vector<Person> checkMoreThanOne(String name) {
Vector<Person> vector = new Vector<>();
String sql = "SELECT * FROM " + DB_NAME;
try {
ResultSet resultSet = db.prepareStatement(sql).executeQuery();
while(resultSet.next()){
String pName = resultSet.getString("name");
String pPhone = resultSet.getString("phone_number");
String pAddress = resultSet.getString("address");
if(pName.equals(name)){
vector.add(new Person(pName,pPhone,pAddress));
}
}
return vector;
} catch (SQLException e) {
e.printStackTrace();
}
return null;
}
//deleting and existing contact;
private void delete(Person person){
String sql = "DELETE FROM " + DB_NAME + " WHERE name = '" + person.getName() + "' and phone_number = '" + person.getPhoneNumber() + "'";
try {
statement = db.prepareStatement(sql);
statement.execute();
System.out.println("Deleted Successfully");
} catch (SQLException e) {
e.printStackTrace();
}
}
//creating a new table for empty data base!
private void createTable() {
try {
statement = db.prepareStatement(SQL_TABLE_STRING);
statement.execute();
} catch (SQLException e) {
e.printStackTrace();
}
}
protected void searchByFirstChar(Character character){
Vector<Person> personVector = new Vector<>();
String sql = "SELECT * FROM newphonebook";
try {
ResultSet resultSet = db.prepareStatement(sql).executeQuery();
while(resultSet.next()){
String name = resultSet.getString("name");
String phoneNum = resultSet.getString("phone_number");
String address = resultSet.getString("address");
if(character.equals(name.charAt(0))){
personVector.add(new Person(name,phoneNum,address));
}
}
System.out.println(personVector);
} catch (SQLException e) {
System.out.println(e.getMessage());
}
}
public void getOptions(){
Vector<String> options = MyUtills.getClassMethods(DBWriterReader.class);
int option = MyUtills.menu(options);
switch (option){
case 0:
addPerson();
break;
case 1:
deleteContact();
break;
case 2:
// searchByFirstChar();
break;
}
}
}
I know it's not best written but I'm working on it to make it better
The Writing and Reading from the data base works fine, its the way it prints my methods that makes the problem..
If you need to guarantee the order of elements in a data structure, you don't use Vector -- it's not 1999 anymore. Look at the documentation for Vector. You get elements in the order determined by an iterator, not as they are stored.
Change your data structure to an ArrayList, which guarantees order. ArrayLists are also more performant in a single threaded application like yours, because unlike Vector, an ArrayList skips the overhead associated with being synchronized. Using the index of the ArrayList elements may also simplify the way you construct your switch statement.

Strings can not be converted to DAO Receiver

I am trying to perform batch insertion operation with a list object but while inserting I am getting String cannot be converted to DAO.The receiver in the iterator loop.
I have tried to list the list object, at that time it is printing values from the list. but, when I use generics are normal list it is showing error and I don't find any solution to insert
From this method I am reading the excel file and storing into list
public List collect(Receiver rec)
{
//ReadFromExcel rd = new ReadFromExcel();
List<String> up = new ArrayList<String>();
//List<String> details = rd.reader();
//System.out.println(details);
try( InputStream fileToRead = new FileInputStream(new File(rec.getFilePath())))
{
XSSFWorkbook wb = new XSSFWorkbook(fileToRead);
wb.setMissingCellPolicy(Row.MissingCellPolicy.RETURN_BLANK_AS_NULL);
XSSFSheet sheet = wb.getSheetAt(0);
DataFormatter fmt = new DataFormatter();
String data ="";
for(int sn = 0;sn<wb.getNumberOfSheets()-2;sn++)
{
sheet = wb.getSheetAt(sn);
for(int rn =sheet.getFirstRowNum();rn<=sheet.getLastRowNum();rn++)
{
Row row = sheet.getRow(rn);
if(row == null)
{
System.out.println("no data in row ");
}
else
{
for(int cn=0;cn<row.getLastCellNum();cn++)
{
Cell cell = row.getCell(cn);
if(cell == null)
{
// System.out.println("no data in cell ");
// data = data + " " + "|";
}
else
{
String cellStr = fmt.formatCellValue(cell);
data = data + cellStr + "|";
}
}
}
}
}
up = Arrays.asList(data.split("\\|"));
// System.out.println(details);
}
catch (FileNotFoundException ex)
{
Logger.getLogger(BImplementation.class.getName()).log(Level.SEVERE, null, ex);
}
catch (IOException ex)
{
Logger.getLogger(BImplementation.class.getName()).log(Level.SEVERE, null, ex);
}
Iterator iter = up.iterator();
while(iter.hasNext())
{
System.out.println(iter.next());
}
String row="";
Receiver info = null;
String cid = "";
String cname = "";
String address = "";
String mid = "";
boolean b = false;
List<Receiver> res = new ArrayList<Receiver>();
int c = 0;
try
{
String str = Arrays.toString(up.toArray());
//System.out.println(str);
String s = "";
s = s + str.substring(1,str.length());
// System.out.println("S:"+s);
StringTokenizer sttoken = new StringTokenizer(s,"|");
int count = sttoken.countTokens();
while(sttoken.hasMoreTokens())
{
if(sttoken.nextToken() != null)
{
// System.out.print(sttoken.nextToken());
cid = sttoken.nextToken();
cname = sttoken.nextToken();
address = sttoken.nextToken();
mid = sttoken.nextToken();
info = new Receiver(cid,cname,address,mid);
res.add(info);
System.out.println("cid :"+cid+ " cname : "+cname +" address : "+address+" mid : "+mid);
c = res.size();
// System.out.println(c);
}
else
{
break;
}
}
System.out.println(count);
// System.out.println("s");
}
catch(NoSuchElementException ex)
{
System.out.println("No Such Element Found Exception" +ex);
}
return up;
}
with this method I'm trying to insert into database
public boolean insert(List res)
{
String sqlQuery = "insert into records(c_id) values (?)";
DBConnection connector = new DBConnection();
boolean flag = false;
// Iterator itr=res.iterator();
// while(it.hasNext())
// {
// System.out.println(it.next());
// }
try( Connection con = connector.getConnection();)
{
con.setAutoCommit(false);
PreparedStatement pstmt = con.prepareStatement(sqlQuery);
Iterator it = res.iterator();
while(it.hasNext())
{
Receiver rs =(Receiver) it.next();
pstmt.setString(1,rs.getcID());
pstmt.setString(2,rs.getcName());
pstmt.setString(3,rs.getAddress());
pstmt.setString(4,rs.getMailID());
pstmt.addBatch();
}
int [] numUpdates=pstmt.executeBatch();
for (int i=0; i < numUpdates.length; i++)
{
if (numUpdates[i] == -2)
{
System.out.println("Execution " + i +": unknown number of rows updated");
flag=false;
}
else
{
System.out.println("Execution " + i + "successful: " + numUpdates[i] + " rows updated");
flag=true;
}
}
con.commit();
} catch(BatchUpdateException b)
{
System.out.println(b);
flag=false;
}
catch (SQLException ex)
{
Logger.getLogger(BImplementation.class.getName()).log(Level.SEVERE, null, ex);
System.out.println(ex);
flag=false;
}
return flag;
}
I want to insert list object using JDBC batch insertion to the database.
Your method collect(Receiver rec) returns the List of strings called up.
return up;
However (if you are really using the method collect to pass the List into insert(List res) method), you are expecting this list to contain Receiver objects. Which is incorrect, since it collect(..) returns the list of Strings.
And that causes an error when you try to cast Receiver rs =(Receiver) it.next();
You need to review and fix your code, so you will pass the list of Receiver objects instead of strings.
And I really recommend you to start using Generics wherever you use List class. In this case compiler will show you all data-type errors immediately.

Not able to get lock on synchronized method when accessed using more than one thread

I have below code where I am testing thread synchronization for elastic search, but somehow I am not able to success in it, can any one let me know where I am going wrong?
If I enable Thread sleep inside 'startThreadProcess' method then everything works fine because it sleeps for certain amount of time period. Which I don't want to, I want to get proper lock for thread without using thread sleep.
but what is happening in above code I have used Executor for pooling threads. There I am running for loop for count 4 so as to initiate 4 threads from pool. so when my thread submitted using executor submit which calls synchronized method and inside that synchronized method I am giving call to other synchronized method from where I am getting total number of count in particular node and proceeding ahead with that count by incrementing to insert new document where my first thread is yet not completed 2nd thread enters and try to get total number of count from the method which is being called from synchronized method so there I am getting wrong count for thread 2 as my first thread will insert 10000 json documents inside node so I am expecting Thread 2 should get count 10000 and then it should process for insert but here my Thread 2 enters in between and get count any random number and start inserting by incrementing from that number which is not expected scenario
package com.acn.adt.main;
public class ESTest {
private static final String dataPath = "C:\\Elastic Search\\Data.json";
static ESTest esTest = new ESTest();
private static TransportClient client = null;
private Properties elasticPro = null;
private InputStream input = null;
ElasticSearchCrud esCRUD = null;
private final Object lock = new Object();
public static void main(String[] args) {
String strArray[] = new String[] {"1"};
esTest.startProcess(strArray);
}
public void startProcess(String strArray[]) {
try {
input = new FileInputStream(ElasticSearchConstants.ELASTIC_PROPERTIES);
elasticPro = new Properties();
//elasticPro.load(ElasticSearchClient.class.getResourceAsStream(ElasticSearchConstants.ELASTIC_PROPERTIES));
elasticPro.load(input);
System.out.println(elasticPro.getProperty("homeDir"));
long startTime = System.currentTimeMillis();
Settings setting = Settings.builder()
//.put("client.transport.ping_timeout", "100s")
.put("cluster.name", elasticPro.getProperty("cluster"))
//.put("node.name", elasticPro.getProperty("node"))
//.put("client.transport.sniff", Boolean.valueOf(elasticPro.getProperty("transport.sniff")))
.put("client.transport.sniff", false)
.put("cluster.routing.allocation.enable", "all")
.put("cluster.routing.allocation.allow_rebalance", "always")
//.put("client.transport.ignore_cluster_name", true)
.build();
client = new PreBuiltTransportClient(setting)
.addTransportAddress(new TransportAddress(InetAddress.getByName("localhost"),
Integer.valueOf("9300")));
long endTime = System.currentTimeMillis();
System.out.println("Time taken for connecting " + TimeUnit.MILLISECONDS.toSeconds((endTime - startTime)));
ExecutorService executorService = Executors.newFixedThreadPool(10);
for(int i = 1; i <=4; i++) {
if(i==1) {
strArray = new String [] {"1"};
}else if(i == 2) {
strArray = new String [] {"1"};
}else if(i == 3) {
strArray = new String [] {"1"};
}else if(i == 4) {
strArray = new String [] {"1"};
}
executorService.execute(new ESThread(esTest,strArray,i));
}
}catch(Exception e) {
}
}
public class ESThread implements Runnable {
private final Object lock = new Object();
ESTest esTester = null;
String strArr [] = null;
int i =0;
public ESThread(ESTest esTester,String[] strArr,int i) {
this.esTester = esTester;
this.strArr = strArr;
this.i = i;
}
#Override
public void run() {
System.out.println("Name of Current thread is Thread_"+i);
synchronized(lock) {
esTester.startCRUDProcess(strArr);
}
System.out.println("Thread_"+i+" done.");
}
}
public void startCRUDProcess(String [] strArr) {
SearchAPI esSearch = new SearchAPIImpl();
boolean caseFlg = false;
String _indexName = "gcindex";
String _indexType = "gctype";
String _ids = "501,602,702###1,10000,10001";
String _id = "10000";
String[] _strIds = new String[] {"10000","9999"};
System.out.println("Insert Multiple Process is started...");
System.out.println("--------------------------------------");
try {
caseFlg = insertMultipleDocument(dataPath,client,_indexName,_indexType);
} catch (IOException | ParseException e) {
e.printStackTrace();
caseFlg = false;
}
}
public synchronized boolean insertMultipleDocument(String dataPath,TransportClient client,String _indexName,String _indexType) throws FileNotFoundException, ParseException {
try {
JSONParser parser = new JSONParser();
// we know we get an array from the example data
JSONArray jsonArray = (JSONArray) parser.parse( new FileReader( dataPath ) );
BulkRequestBuilder bulkDocument = client.prepareBulk();
#SuppressWarnings("unchecked")
Iterator<JSONObject> it = jsonArray.iterator();
int i = 0;
i = _getTotalHits(client,_indexName,_indexType);
System.out.println("Total number of hits inside index = "+_indexName+" of type = "+_indexType+" are : "+i);
System.out.println("-------------------------------------------------------------------------------------");
while( it.hasNext() ) {
i++;
JSONObject json = it.next();
System.out.println("Insert document for "+i+": " + json.toJSONString() );
// either use client#prepare, or use Requests# to directly build index/delete requests
bulkDocument.add(client.prepareIndex(_indexName, _indexType, i+"")
.setSource(json.toJSONString(), XContentType.JSON )
);
}
BulkResponse bulkResponse = bulkDocument.get();
if (bulkResponse.hasFailures()) {
System.out.println("process failures by iterating through each bulk response item : "+bulkResponse.buildFailureMessage());
return false;
} else {
System.out.println("All Documents inserted successfully...");
/*if(bulkResponse.getItems()!=null) {
for(BulkItemResponse response:bulkResponse.getItems()) {
System.out.println(response.toString());
System.out.println(response.getResponse());
}
}*/
return true;
}
} catch (IOException ex) {
System.out.println("Exception occurred while get Multiple Document : " + ex/*, ex*/);
return false;
}
}
public synchronized int _getTotalHits(TransportClient client,String _indexName,String _indexType){
SearchHits hits = null;
int recCount = 0;
long totalCount = 0;
try {
SearchResponse seacrhResponse = client.prepareSearch(_indexName)
.setTypes(_indexType)
.setSearchType(SearchType.QUERY_THEN_FETCH)
.get();
if (seacrhResponse != null) {
hits = seacrhResponse.getHits();
totalCount = hits.getTotalHits();
System.out.println("count = "+totalCount);
}
recCount = Integer.parseInt(totalCount+"");
}catch(Exception ex) {
System.out.println("Exception occurred while search Index : " + ex/*, ex*/);
}
return recCount;
}
}

Thread automatically stop (freeze) in Eclipse

I am running a thread in eclipse to get data from mysql server. Thread works fine. The problem is after bit of time thread stop running(withing 6 to 8 hours). Thread get freeze. After that I have to manually close and re-run the program. Eclipse runs in a windows server 2012 r2 machine. No error or exception is shown.
Main class.
public class Main {
private final static int fONE_DAY = 1;
private final static int fZERO_MINUTES = 0;
public static void main(String[] argv) {
Timer timer1 = new Timer();
Timer timer4 = new Timer();
try {
timer1.scheduleAtFixedRate(new CearteSDQuatation(),500 , 1000*60*4);// 4min
System.out.println("timer 1 : createSDQuotation");
} catch (Exception e) {
e.printStackTrace();
}
try {
timer4.schedule(new GarbageCol(), 5000, 1000 * 60 * 60);// 60mins
System.out.println("timer 2 : garbageCollector");
} catch (Exception e) {
e.printStackTrace();
}
}
private static Date getTomorrowRunningTime(int Ftime){
Calendar tomorrow = new GregorianCalendar();
tomorrow.add(Calendar.DATE, fONE_DAY);
Calendar result = new GregorianCalendar(
tomorrow.get(Calendar.YEAR),
tomorrow.get(Calendar.MONTH),
tomorrow.get(Calendar.DATE),
Ftime,
fZERO_MINUTES
);
return result.getTime();
}
}
CearteSDQuatation class.
public class CearteSDQuatation extends TimerTask {
DateFormat formatter = new SimpleDateFormat("dd.MM.yyyy");;
DateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy");
Date date = new Date();
DBPool_SF pooler;
DataSource dataSource;
DataSource dataSource1;
DBTableQueryExcecutre qex = null;
DBTableQueryExcecutre qex3 = null;
DBTableQueryExcecutre qex1 = null;
DBTableQueryExcecutre qex2 = null;
ArrayList<String> dates = null;
HashSet<String> dateSet = null;
Iterator<String> itr = null;
StringBuilder inClause;
int rcount = 0;
StringBuilder sbDel = null;
int delStatus = 0;
ArrayList<String> elements = new ArrayList<String>();// for update query
int queryStatus = 0;
String delete_query1 = " ";
String strDate = "";
String strMatnr = "";
String strkunnr = "";
// Object for table data
private Object[][] itemData;
RFCHandler handler;
public CearteSDQuatation(){
handler = new RFCHandler();
}
#Override
public void run() {
// TODO Auto-generated method stub
try{
CallItem_ListCreate();
CallRFC_CreateSDQuata();
System.out.println("Thread Run");
}
catch (Exception e){
e.printStackTrace();
}
}
private void CallItem_ListCreate() {
// TODO Auto-generated method stub
DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
Date date = new Date();
System.out.println(dateFormat.format(date));
System.out.println("Create Quotation : Started sub excecutions List create...");
String sbQuery3 = "SELECT d.plant, h.dis_channel, h.order_no, h.Rep_no," +
"h.dealer_no,h.order_date,h.last_date, d.items_no," +
"d.quantity " +
"FROM tbl_item_list as d, " +
"tbl_items_header_t as h " +
"where d.order_no = h.order_no " +
"and h.status <> 'X'" ;
try {
pooler = DBPool_SF.getInstance();
dataSource1 = pooler.getDataSource();
System.out.println("pooler");
} catch (Exception e1)
{
e1.printStackTrace();
}
try {
Connection con3 = dataSource1.getConnection();
con3.setAutoCommit(false);
Statement st = con3.createStatement();
ResultSet rs = st.executeQuery(sbQuery3);
int lineitem = 0;
try {
rs.last();
rcount = rs.getRow();
rs.beforeFirst();
}
catch(Exception ex) {
ex.printStackTrace();
}
System.out.println("while loop");
itemData = new Object[9][rcount];
while(rs.next())
{
itemData[0][lineitem] = rs.getString("plant");
lineitem = lineitem + 1;
}
} catch (SQLException e)
{
e.printStackTrace();
}
}
public void CallRFC_CreateSDQuata()
{
System.out.println("Create Quotation : Started sub excecutions");
JCO.Table IT_LIST = null;
JCO.Table IT_LIST1 = null;
JCO.Table IT_REF = null;
JCO.Table IT_Msg = null;
try {
if(rcount > 0)
{
handler.createRFCFunction("ZSL");
IT_LIST = handler.getTablePara("IT_LIST");
IT_LIST1 = handler.getTablePara("IT_LIST1");
for(int x = 0; x < rcount; x++ )
{
IT_LIST.appendRow();
IT_LIST1.appendRow();
IT_LIST.setValue( itemData[0][x].toString().trim(), "SAL_ORG");
}
handler.excFunction();
IT_REF = handler.getTablePara("IT_REF");
IT_Msg = handler.getTablePara("IT_MSG");
handler.releaseClient();
int int_row = IT_REF.getNumRows();
if (int_row > 0) {
this.tableOparator(IT_REF , IT_Msg);
}
}
} catch (Exception ex) {
handler.releaseClient();
ex.printStackTrace();
}
finally {
// Release the client to the pool
//handler.releaseClient();
rcount = 0;
}
}
public String leadingZeros(String s, int length) {
if (s.length() >= length) return s;
else return String.format("%0" + (length-s.length()) + "d%s", 0, s);
}
public void tableOparator(JCO.Table table , JCO.Table table1 ) throws Exception {
pooler = DBPool_SF.getInstance();
dataSource = pooler.getDataSource();
Connection con = dataSource.getConnection();
con.setAutoCommit(false);
qex = new DBTableQueryExcecutre(con);
StringBuilder sbQuery = new StringBuilder(
"INSERT INTO tbl_RefQut (QuatationNo,RefOrderNumber) VALUES");
System.out.println("COL -->"+table.getNumRows()
+ " records to insert for tbl_RefQut");
// --- create query ---------------------------------
for (int i = 0; i < table.getNumRows(); i++) {
// table.setRow(i);
sbQuery.append("(?,?),");
}
sbQuery.deleteCharAt(sbQuery.length() - 1);
sbQuery.append(";");
qex.setUpdateQuery(sbQuery.toString());// *****************
elements.clear();
for (int i = 0; i < table.getNumRows(); i++) {
table.setRow(i);
elements.add(table.getString("SDOCUMENT").trim());
elements.add(table.getString("ONUMBER").trim());
}
qex.updateInsertQuery(elements);
con.commit();
StringBuilder sbQuery1 = new StringBuilder(
"INSERT INTO tbl_item_order_msg (order_no,msg) VALUES");
System.out.println("COL -->"+table1.getNumRows()
+ " records to insert for tbl_item_order_msg");
// --- create query ---------------------------------
for (int i = 0; i < table1.getNumRows(); i++) {
// table.setRow(i);
sbQuery1.append("(?,?),");
}
sbQuery1.deleteCharAt(sbQuery1.length() - 1);
sbQuery1.append(";");
qex.setUpdateQuery(sbQuery1.toString());// *****************
elements.clear();
for (int i = 0; i < table1.getNumRows(); i++) {
table1.setRow(i);
elements.add(table1.getString("ORDER_NUMBER").trim());
elements.add(table1.getString("MESSAGE").trim());
}
qex.updateInsertQuery(elements);
con.commit();
for (int i = 0; i < table.getNumRows(); i++) {
table.setRow(i);
String sbQuery2 = "update tbl_items_header_t set status = 'X' where order_no = '" + table.getString("ORDER_NUMBER").trim() + "';";
int rcount = qex.runQuery(sbQuery2);
System.out.println("tbl_items_header_t Rows -->"+rcount + "Status Updated");
con.commit();
}
qex.closeConnections();
System.out.println("COL --> Cycle Finished....");
}
}
a possible reason to this situation is, you are hitting mysql's max open connections error, because you are not closing connections in method CallItem_ListCreate. it seems mysql default max connection count is 151. you are openning 15 connections per hour and after 10 hour you will hit that mentiooned error.
p.s. : you should try running your code in debug mode (or using tools like visualvm, jstack) and view thread dumps as #saurav commented.

Categories