We Keep Coding

Processing huge File quickly in Java

I have a huge CSV file ~800 K records and using that data I have to form a POST request and make a rest call.
Initially I tried WITHOUT using threads but it is taking very long time to process it.
So I thought of using threads to speed up the process and followed below approach.
divided file into relatively smaller files (Tried with 3 files with approx ~5K data each file). (I Did this Manually Before passing to application)
Created 3 threads (traditional way by extending Thread class)
Reads each file with each thread and form HashMap with details required to form request and added it to ArrayList of POST Request
Send Post request to in loop
List item
Get the Response and add it to Response List
Both above approach takes too long to even complete half (>3 hrs). Might be one of them would not be good approach.
Could anyone please provide suggestion which helps me speeding up the process ? It is NOT mandatory to use threading.
Just to add that my code is at the consumer end and does not have much control on producer end.
MultiThreadedFileRead (Main class that Extends Thread)
class MultiThreadedFileRead extends Thread
{
public static final String EMPTY = "";
public static Logger logger = Logger.getLogger(MultiThreadedFileRead.class);
BufferedReader bReader = null;
String filename = null;
int Request_Num = 0;
HashMap<Integer, String> filteredSrcDataMap = null;
List<BrandRQ> brandRQList = null;
List<BrandRS> brandRSList = null;
ReadDLShipFile readDLShipFile = new ReadDLShipFile();
GenerateAndProcessPostBrandAPI generateAndProcessPostBrandAPI = new GenerateAndProcessPostBrandAPI();
MultiThreadedFileRead()
{
}
MultiThreadedFileRead(String fname) throws Exception
{
filename = fname;
Request_Num = 0;
List<BrandRQ> brandRQList = new ArrayList<BrandRQ>();
List<BrandRS> brandRSList = new ArrayList<BrandRS>();
this.start();
}
public void run()
{
logger.debug("File *** : " + this.filename);
//Generate Source Data Map
HashMap<Integer, String> filteredSrcDataMap = (HashMap<Integer, String>) readDLShipFile.readSourceFileData(this.filename);
generateAndProcessPostBrandAPI.generateAndProcessPostRequest(filteredSrcDataMap, this);
}
public static void main(String args[]) throws Exception
{
String environment = ""; // TO BE UNCOMMENTED
String outPutfileName = ""; // TO BE UNCOMMENTED
BrandRetrivalProperties brProps = BrandRetrivalProperties.getInstance();
if(args != null && args.length == 2 && DLPremiumUtil.isNotEmpty(args[0]) && DLPremiumUtil.isNotEmpty(args[1])) // TO BE UNCOMMENTED
{
environment = args[0].trim();
outPutfileName = args[1].trim();
ApplicationInitializer applicationInitializer = new ApplicationInitializer(environment);
applicationInitializer.initializeParams();
logger.debug("Environment : " + environment);
logger.debug("Filename : " + outPutfileName);
// TO BE UNCOMMENTED - START
}else{
System.out.println("Invalid parameters passed. Expected paramenters: Environment");
System.out.println("Sample Request: java -jar BrandRetrival.jar [prd|int] brand.dat");
System.exit(1);
}
MultiThreadedFileRead multiThreadedFileRead = new MultiThreadedFileRead();
List<String> listOfFileNames = multiThreadedFileRead.getFileNames(brProps);
logger.debug("List : " + listOfFileNames);
int totalFileSize = listOfFileNames.size();
logger.debug("totalFileSize : " + totalFileSize);
MultiThreadedFileRead fr[]=new MultiThreadedFileRead[totalFileSize];
logger.debug("Size of Array : " + fr.length);
for(int i = 0; i < totalFileSize; i++)
{
logger.debug("\nFile Getting Added to Array : " + brProps.getCountFilePath()+listOfFileNames.get(i));
fr[i]=new MultiThreadedFileRead(brProps.getCountFilePath()+listOfFileNames.get(i));
}
}
public List<String> getFileNames(BrandRetrivalProperties brProps)
{
MultiThreadedFileRead multiThreadedFileRead1 = new MultiThreadedFileRead();
BufferedReader br = null;
String line = "";
String filename = multiThreadedFileRead1.getCounterFileName(brProps.getCountFilePath(), brProps.getCountFilePathStartsWith());
List<String> fileNameList = null;
logger.debug("filename : " + filename);
if(filename != null)
{
fileNameList = new ArrayList<String>();
try{
br = new BufferedReader(new FileReader(brProps.getCountFilePath()+filename));
while ((line = br.readLine()) != null)
{
fileNameList.add(line);
}
} catch (FileNotFoundException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (IOException e2) {
// TODO Auto-generated catch block ;
e2.printStackTrace();
} catch (Exception e3) {
// TODO Auto-generated catch block
e3.printStackTrace();
} finally
{
if (br != null)
{
try
{
br.close();
} catch (IOException e4)
{
e4.printStackTrace();
} catch (Exception e5)
{
e5.printStackTrace();
}
}
}
}
return fileNameList;
}
// Get the Name of the file which has name of individual CSV files to read to form the request
public String getCounterFileName(String sourceFilePath, String sourceFileNameStartsWith)
{
String fileName = null;
if(sourceFilePath != null && !sourceFilePath.equals(EMPTY) &&
sourceFileNameStartsWith != null && !sourceFileNameStartsWith.equals(EMPTY))
{
File filePath = new File(sourceFilePath);
File[] listOfFiles = filePath.listFiles();
for (int i = 0; i < listOfFiles.length; i++) {
if (listOfFiles[i].isFile() && listOfFiles[i].getName().startsWith(sourceFileNameStartsWith)) {
fileName = listOfFiles[i].getName();
} else if (listOfFiles[i].isDirectory()) {
fileName = null;
}
}
}
return fileName;
}
}
GenerateAndProcessPostBrandAPI
public class GenerateAndProcessPostBrandAPI {
public static Logger logger = Logger.getLogger(GenerateAndProcessPostBrandAPI.class);
List<BrandRQ> brandRQList = new ArrayList<BrandRQ>();
List<BrandRS> brandRSList = new ArrayList<BrandRS>();
DLPremiumClientPost dlPremiumclientPost = new DLPremiumClientPost();
JSONRequestGenerator jsonRequestGenerator = new JSONRequestGenerator();
public List<BrandRQ> getBrandRequstList(Map<Integer, String> filteredSrcDataMap)
{
if(filteredSrcDataMap != null)
{
brandRQList = jsonRequestGenerator.getBrandRQList(filteredSrcDataMap, brandRQList);
}
return brandRQList;
}
public void generateAndProcessPostRequest(Map<Integer, String> filteredSrcDataMap, MultiThreadedFileRead multiThreadedFileRead)
{
if(filteredSrcDataMap != null)
{
brandRQList = jsonRequestGenerator.getBrandRQList(filteredSrcDataMap, brandRQList);
if(brandRQList != null && brandRQList.size() > 0)
{
BrandRetrivalProperties brProps = BrandRetrivalProperties.getInstance();
String postBrandsEndpoint = brProps.getPostBrandsEndPoint();
for (BrandRQ brandRQ : brandRQList)
{
multiThreadedFileRead.Request_Num = multiThreadedFileRead.Request_Num + 1;
logger.debug("Brand Request - " + multiThreadedFileRead.Request_Num + " : " + ObjectToJSONCovertor.converToJSON(brandRQ));
BrandRS brandResponse = dlPremiumclientPost.loadBrandApiResponseJSON(brandRQ, postBrandsEndpoint, DLPremiumUtil.getTransactionID(), BrandConstants.AUTH_TOKEN);
logger.debug("Response - " + multiThreadedFileRead.Request_Num + " : " + ObjectToJSONCovertor.converToJSON(brandResponse));
brandRSList.add(brandResponse);
}
}
}
}
}

run the startUp method at the deploy a web service

I try to execute the startUp method when i deploy my web services, but doesn`t work.
I'm using:
windows 7
tomcat 8.0.30
axis2 1.7.0
I've tryed to deploy my service as a pojo service also i try generating the *.aar and placing it in:
apache-tomcat-8.0.30\webapps\axis2\WEB-INF\services
but when i run the tomcat, and deploy this and other services, the startUp method dont launch.
this is my code:
import java.io.*;
import java.util.*;
import org.apache.axis2.context.ConfigurationContext;
import org.apache.axis2.description.AxisService;
import org.apache.axis2.engine.ServiceLifeCycle;
public class Login implements ServiceLifeCycle{
static String IPprop = "";
static String rutaDB = "C:/resources/users_DB.txt";
static String rutaUddiXml = "C:/resources/uddi.xml";
static String rutaIP = "C:/resources/ip.txt";
static boolean registrado=false;
static String comp ="";
public static void main(String[] args) {
IP();
String nombreServicio = "Login";
String rutaServicio = "http://"+ IPprop +":8080/axis2/services/Login";
SimplePublishPortable spp = new SimplePublishPortable(rutaUddiXml);
spp.publish(nombreServicio, rutaServicio);
System.out.println("te registraste");
}
public static void createUser(String user, String pass) {
interacFich("crea", user, pass);
}
public static String loginAccess(String user, String pass) {
return interacFich("login", user, pass);
}
public static String runComprobation(){
return "deployed service" + comp;
}
public static String regComprobation(){
if(registrado==true){
return "registered";
}
else{
return "failed";
}
}
private static String getToken() {
String cadenaAleatoria = "";
int i = 0;
while (i < 10) {
char c = (char) ((int) (Math.random() * 255 + 1));
if ((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z')) {
cadenaAleatoria += c;
i++;
}
}
return cadenaAleatoria;
}
private static String interacFich(String accion, String user, String pass) {
String usuario = "";
LinkedHashMap<String, String> usuarios = new LinkedHashMap<String, String>();
File archivo = new File(rutaDB);
// leer fichero y meterlo en el mapa
if (archivo.exists() == false) {
try {
archivo.createNewFile();
} catch (IOException e) {
e.printStackTrace();
}
} else {
try (BufferedReader br = new BufferedReader(new FileReader(archivo))) {
while (br.ready()) {
usuario = br.readLine();
String[] param = usuario.split("\\*");
usuarios.put(param[0], param[1]);
}
br.close();
} catch (IOException e) {
e.printStackTrace();
}
}
switch (accion) {
case "crea":
usuarios.put(user + "-" + pass, getToken());
try (BufferedWriter bw = new BufferedWriter(new FileWriter(archivo))) {
Set<String> keysUsuarios = usuarios.keySet();
for (String k : keysUsuarios) {
bw.write(k + "*" + usuarios.get(k).toString());
bw.write("\n");
}
System.out.println("todo escrito");
bw.close();
return "el fichero se crea";
} catch (IOException e) {
e.printStackTrace();
}
break;
case "login":
if (usuarios.containsKey(user + "-" + pass)) {
return usuarios.get(user + "-" + pass);
}
return "User o pass erroneos";
default:
break;
}
return null;
}
private static void IP() {
File archivo = new File(rutaIP);
try (BufferedReader br = new BufferedReader(new FileReader(archivo))) {
br.readLine();
IPprop = br.readLine();
br.close();
} catch (IOException e) {
e.printStackTrace();
}
return;
}
#Override
public void startUp(ConfigurationContext arg0, AxisService arg1) {
///////////////
//registrarse//
///////////////
comp="entramos";
IP();
String nombreServicio = "Login";
String rutaServicio = "http://"+ IPprop +":8080/axis2/services/Login";
SimplePublishPortable spp = new SimplePublishPortable(rutaUddiXml);
spp.publish(nombreServicio, rutaServicio);
registrado=true;
}
#Override
public void shutDown(ConfigurationContext arg0, AxisService arg1) {
// TODO Auto-generated method stub
}
}

Have you created the Service Definition (services.xml) file, on which you describe your services?
I am referring to a file similar to the example in Code Listing 3: The Service Definition File in the following link (axis2 quick start):
https://axis.apache.org/axis2/java/core/docs/quickstartguide.html#services

parse CSV using BaneUtilBean

I am trying to parse a csv and map the fields to a POJO class. However I can see that the mapping is not achieved correctly.
I am trying to map the header from a POJO file to the csv.
public class CarCSVFileInputBean {
private long Id;
private String shortName;
private String Name;
private String Type;
private String Environment;
//getter and setters
}
Can someone please take a look at my code:
public class carCSVUtil {
private static Log log = LogFactory.getLog(carCSVUtil.class);
private static final List<String> fileHeaderFields = new ArrayList<String>();
private static final String UTF8CHARSET = "UTF-8";
static {
for (Field f : carCSVFileInputBean.class.getDeclaredFields()) {
fileHeaderFields.add(f.getName());
}
}
public static List<carCSVFileInputBean> getCSVInputList(InputStream inputStream) {
CSVReader reader = null;
List<carCSVFileInputBean> csvList = null;
carCSVFileInputBean inputRecord = null;
String[] header = null;
String[] row = null;
try {
reader = new CSVReader(new InputStreamReader(inputStream, UTF8CHARSET));
csvList = new ArrayList<carCSVFileInputBean>();
header = reader.readNext();
boolean isEmptyLine = true;
while ((row = reader.readNext()) != null) {
isEmptyLine = true;
if (!(row.length == 1 && StringUtils.isBlank(row[0]))) { // not an empty line, not even containing ','
inputRecord = new carCSVFileInputBean();
isEmptyLine = populateFields(inputRecord, header, row);
if (!isEmptyLine)
csvList.add(inputRecord);
}
}
} catch (IOException e) {
log.debug("IOException while accessing carCSVFileInputBean: " + e);
return null;
} catch (IllegalAccessException e) {
log.debug("IllegalAccessException while accessing carCSVFileInputBean: " + e);
return null;
} catch (InvocationTargetException e) {
log.debug("InvocationTargetException while copying carCSVFileInputBean properties: " + e);
return null;
} catch (Exception e) {
log.debug("Exception while parsing CSV file: " + e);
return null;
} finally {
try {
if (reader != null)
reader.close();
} catch (IOException ioe) {}
}
return csvList;
}
protected static boolean populateFields(carCSVFileInputBean inputRecord, String[] header, String[] row) throws IllegalAccessException, InvocationTargetException {
boolean isEmptyLine = true;
for (int i = 0; i < row.length; i++) {
String val = row[i];
if (!StringUtils.isBlank(val)) {
BeanUtilsBean.getInstance().copyProperty(inputRecord, header[i], val);
isEmptyLine = false;
}
}
return isEmptyLine;
}
}

I found the solution - the headers in the csv file are expected to begin with a lowercase.

return string array in place of string

i have a class which is returning a string type value and i want to return an String array, so please tell how can i able to do that
i have an xml file like resource.xml
<prompts>
<prompt id="p1">welcome to</prompt>
<prompt id ="p2">stack overflow</prompt>
<prompt id="p3">You entered</prompt>
<prompt id="p4">the correct number</prompt>
<prompts>
i am parsing it using sax parser
public class XmlReaderPrompt {
public List<PromptBean> load(String langMode)
{
String fileName="resource.xml";
DocumentBuilderFactory dbf=DocumentBuilderFactory.newInstance();
InputStream prompt_configfile=Thread.currentThread().getContextClassLoader().getResourceAsStream(fileName);
DocumentBuilder db = null;
List<PromptBean> promptMap = new ArrayList<PromptBean>();
try {
try {
db = dbf.newDocumentBuilder();
} catch (ParserConfigurationException e) {
e.printStackTrace();
}
Document doc = null;
try {
doc = db.parse(prompt_configfile);
}
catch (SAXException e) {
e.printStackTrace();
}
NodeList nodeList=doc.getElementsByTagName("prompt");
for(int i=0;i<nodeList.getLength();i++)
{
Node node=nodeList.item(i);
if(node.getNodeType()==Node.ELEMENT_NODE)
{
Element element=(Element)node;
String id = element.getAttribute("id");
String name = element.getAttribute("name");
String prompt=getTextValue(element);
promptMap.add(new PromptBean(id,name,prompt));
}
}
}
catch(Exception io)
{
io.printStackTrace();
}
finally
{
db=null;
dbf=null;
}
return promptMap;
}
private String getTextValue(Element element) {
String textValue=element.getFirstChild().getTextContent().toString();
return textValue;
}
}
and a UserFunction class to return the text from the xml file
public class UserFunction{
List<PromptBean> promptObject = new ArrayList<PromptBean>();
public String getPromptFunction(String promptTag,String langMode )
{
List<PromptBean> promptObject=xrpObject.load(langMode);
for (Iterator<PromptBean> iterator = promptObject.iterator(); iterator.hasNext();){
PromptBean promptBean= (PromptBean)iterator.next();
if(promptBean.getId().equalsIgnoreCase(promptTag)){
return StringEscapeUtils.escapeXml(promptBean.getPrompt());
}
}
return null;
}
The problem is that I have to call the method getPromptFunction of UserFunction class every time I need to get text from the sub element like
String pr1 = UserFunction.getPromptFunction("p1" "resource");
String pr1 = UserFunction.getPromptFunction("p2" "resource");
String pr1 = UserFunction.getPromptFunction("p3" "resource");
and using it in jsp page as <%=pr1%>
So I want to use array like
String[] pr = UserFunction.getPromptFunction('"p1","p2","p3"' "resource")
So how I am able to do that and also tell how to use it in jsp page .

You can do it like this
public String[] getPromptFunction(String promptTag,String langMode )
{
String temp[] = new String[promptObject.size()];
List<PromptBean> promptObject=xrpObject.load(langMode);
int i = 0;
for (Iterator<PromptBean> iterator = promptObject.iterator(); iterator.hasNext();) {
PromptBean promptBean= (PromptBean)iterator.next();
if(promptBean.getId().equalsIgnoreCase(promptTag)){
temp[i] = StringEscapeUtils.escapeXml(promptBean.getPrompt());
}
i++;
}
return temp;
}

Google Data Api library for Blackberry

I want to get google contacts in my Blackberry Application. Is there any public libraries availabile for blackberry to do this?
I try to use Oauth-SignPost. But the libraies used in it not supported by blackberry.Then I try the following code
public static String requestToken(){
String url = C.REQUEST_URL;
String header = oauth_header(url, HttpProtocolConstants.HTTP_METHOD_GET);
String requestTokenUrl = concatURL(url, header);
HttpConnection httpConn = null;
InputStream input = null;
try{
HttpConnectionFactory factory = new HttpConnectionFactory( requestTokenUrl,
HttpConnectionFactory.TRANSPORT_WIFI |
HttpConnectionFactory.TRANSPORT_WAP2 |
HttpConnectionFactory.TRANSPORT_BIS |
HttpConnectionFactory.TRANSPORT_BES |
HttpConnectionFactory.TRANSPORT_DIRECT_TCP);
httpConn = factory.getNextConnection();
httpConn.setRequestMethod(HttpProtocolConstants.HTTP_METHOD_GET);
httpConn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
input = httpConn.openDataInputStream();
int resp = httpConn.getResponseCode();
if (resp == HttpConnection.HTTP_OK) {
StringBuffer buffer = new StringBuffer();
int ch;
while ( (ch = input.read()) != -1){
buffer.append( (char) ch);
}
String content = buffer.toString();
System.out.println("Response"+content);
}
return "";
} catch (IOException e) {
return "exception";
} catch (NoMoreTransportsException nc) {
return "noConnection";
} finally {
try {
httpConn.close();
input.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
The oauth_header() which create the appending parameters
public static String oauth_header(String url, String method) {
String nonce = nonce();
long timestamp = timestamp();
Hashtable pairs = new Hashtable();
pairs.put(C.OAUTH_CONSUMER_KEY, C.CONSUMER_KEY);
pairs.put(C.OAUTH_NONCE, nonce);
pairs.put(C.OAUTH_SIGNATURE_METHOD, C.SIGNATURE_METHOD);
pairs.put(C.OAUTH_TIMESTAMP, Long.toString(timestamp));
pairs.put(C.OAUTH_SCOPE,C.SCOPE);
pairs.put(C.OAUTH_VERSION, "1.0");
String sig = signature(method, url, pairs);
StringBuffer header_sb = new StringBuffer();
header_sb.append(C.OAUTH_CONSUMER_KEY).append("=").append(C.CONSUMER_KEY).append(",");
header_sb.append(C.OAUTH_NONCE).append("=").append(nonce).append(",");
header_sb.append(C.OAUTH_SIGNATURE).append("=").append(URLUTF8Encoder.encode(sig)).append(",");
header_sb.append(C.OAUTH_SIGNATURE_METHOD).append("=").append(C.SIGNATURE_METHOD).append(",");
header_sb.append(C.OAUTH_TIMESTAMP).append("=").append(Long.toString(timestamp)).append(",");
header_sb.append(C.OAUTH_SCOPE).append("=").append(C.SCOPE);
header_sb.append(C.OAUTH_VERSION).append("=").append("1.0");
return header_sb.toString();
}
Signature() and concatUrl() here
private static String signature(String method, String requestURL, Hashtable pairs) {
StringBuffer sb = new StringBuffer();
String[] keys = new String[pairs.size()];
Enumeration e = pairs.keys();
int i = 0;
while(e.hasMoreElements()) {
String k = (String)e.nextElement();
keys[i++] = k + "=" + URLUTF8Encoder.encode((String)pairs.get(k));
}
Arrays.sort(keys, new Comparator() {
public int compare(Object arg0, Object arg1) {
return ((String)arg0).compareTo((String)arg1);
}
});
for(i = 0; i < keys.length; i++) {
sb.append(keys[i]).append('&');
}
sb.deleteCharAt(sb.length()-1);
String msg = method.toUpperCase() +"&" + URLUTF8Encoder.encode(requestURL) + "&" + URLUTF8Encoder.encode(sb.toString());
System.out.println(msg);
StringBuffer key = new StringBuffer();
if(C.CONSUMER_SECRET != null) key.append(URLUTF8Encoder.encode(C.CONSUMER_SECRET));
key.append('&');
/* if(Const.tokenSecret != null){
key.append(URLUTF8Encoder.encode(Const.tokenSecret));
}*/
try {
return hmacsha1(key.toString(), msg);
} catch (Exception ex) {
return null;
}
}
private static String hmacsha1(String key, String message)
throws CryptoTokenException, CryptoUnsupportedOperationException, IOException {
HMACKey k = new HMACKey(key.getBytes());
HMAC hmac = new HMAC(k, new SHA1Digest());
hmac.update(message.getBytes());
byte[] mac = hmac.getMAC();
return Base64OutputStream.encodeAsString(mac, 0, mac.length, false, false);
}
public static String concatURL(String url, String header){
String newurl=url;
header = header.replace(',', '&');
newurl = newurl+"?"+header;
return newurl;
}
Then I get the signature_invalid Message. please Help me to find out the error.