We Keep Coding

JSchException Cannot find message

I want to know some of the reasons that can cause below exception. I am unable to find this message Cannot find message in jsch-0.1.54.jar. There exists some straight forward messages like file not found and others that make sense. But I need more information about this one so that I can reach to the root cause.
SftpException while running get ---> 2: Cannot find message [/destination/file.txt]
at com.jcraft.jsch.ChannelSftp.throwStatusError(ChannelSftp.java:2289)
at com.jcraft.jsch.ChannelSftp._stat(ChannelSftp.java:1741)
at com.jcraft.jsch.ChannelSftp._stat(ChannelSftp.java:1758)
at com.jcraft.jsch.ChannelSftp.get(ChannelSftp.java:786)
at com.jcraft.jsch.ChannelSftp.get(ChannelSftp.java:750)
at com.iyi.ftp.SFTP.get(SFTP.java:99)
Here is my calling method.
public boolean get(final String remoteFile, final String localFile) throws JSchException {
Vector connection = null;
Session session = null;
ChannelSftp c = null;
boolean status = false;
try {
connection = this.connect();
session = connection.get(0);
c = connection.get(1);
c.get(remoteFile, localFile);
status = true;
}
catch (JSchException e) {
SFTP.LGR.warn((Object)("JSchException in SFTP::get() ---> " + FTPFactory.getStackTrace((Throwable)e)));
throw e;
}
catch (SftpException e2) {
SFTP.LGR.warn((Object)("SftpException while running get ---> " + FTPFactory.getStackTrace((Throwable)e2)));
throw new JSchException(e2.getMessage());
}
catch (CredentialDecryptionException e3) {
SFTP.LGR.error((Object)"##CredentialDecryptionException##", (Throwable)e3);
throw new JSchException(e3.getMessage(), (Throwable)e3);
}
finally {
if (c != null) {
c.quit();
}
if (session != null) {
session.disconnect();
}
}
if (c != null) {
c.quit();
}
if (session != null) {
session.disconnect();
}
return status;
}
These methods are fetched from jsch-0.1.54.jar which is an open source utility.
public void get(String src, String dst, final SftpProgressMonitor monitor, final int mode) throws SftpException {
boolean _dstExist = false;
String _dst = null;
try {
((MyPipedInputStream)this.io_in).updateReadSide();
src = this.remoteAbsolutePath(src);
dst = this.localAbsolutePath(dst);
final Vector v = this.glob_remote(src);
final int vsize = v.size();
if (vsize == 0) {
throw new SftpException(2, "No such file");
}
final File dstFile = new File(dst);
final boolean isDstDir = dstFile.isDirectory();
StringBuffer dstsb = null;
if (isDstDir) {
if (!dst.endsWith(ChannelSftp.file_separator)) {
dst += ChannelSftp.file_separator;
}
dstsb = new StringBuffer(dst);
}
else if (vsize > 1) {
throw new SftpException(4, "Copying multiple files, but destination is missing or a file.");
}
for (int j = 0; j < vsize; ++j) {
final String _src = v.elementAt(j);
final SftpATTRS attr = this._stat(_src);
if (attr.isDir()) {
throw new SftpException(4, "not supported to get directory " + _src);
}
_dst = null;
if (isDstDir) {
final int i = _src.lastIndexOf(47);
if (i == -1) {
dstsb.append(_src);
}
else {
dstsb.append(_src.substring(i + 1));
}
_dst = dstsb.toString();
if (_dst.indexOf("..") != -1) {
final String dstc = new File(dst).getCanonicalPath();
final String _dstc = new File(_dst).getCanonicalPath();
if (_dstc.length() <= dstc.length() || !_dstc.substring(0, dstc.length() + 1).equals(dstc + ChannelSftp.file_separator)) {
throw new SftpException(4, "writing to an unexpected file " + _src);
}
}
dstsb.delete(dst.length(), _dst.length());
}
else {
_dst = dst;
}
final File _dstFile = new File(_dst);
if (mode == 1) {
final long size_of_src = attr.getSize();
final long size_of_dst = _dstFile.length();
if (size_of_dst > size_of_src) {
throw new SftpException(4, "failed to resume for " + _dst);
}
if (size_of_dst == size_of_src) {
return;
}
}
if (monitor != null) {
monitor.init(1, _src, _dst, attr.getSize());
if (mode == 1) {
monitor.count(_dstFile.length());
}
}
FileOutputStream fos = null;
_dstExist = _dstFile.exists();
try {
if (mode == 0) {
fos = new FileOutputStream(_dst);
}
else {
fos = new FileOutputStream(_dst, true);
}
this._get(_src, fos, monitor, mode, new File(_dst).length());
}
finally {
if (fos != null) {
fos.close();
}
}
}
}
catch (Exception e) {
if (!_dstExist && _dst != null) {
final File _dstFile2 = new File(_dst);
if (_dstFile2.exists() && _dstFile2.length() == 0L) {
_dstFile2.delete();
}
}
if (e instanceof SftpException) {
throw (SftpException)e;
}
if (e instanceof Throwable) {
throw new SftpException(4, "", e);
}
throw new SftpException(4, "");
}
}
private SftpATTRS _stat(final byte[] path) throws SftpException {
try {
this.sendSTAT(path);
Header header = new Header();
header = this.header(this.buf, header);
final int length = header.length;
final int type = header.type;
this.fill(this.buf, length);
if (type != 105) {
if (type == 101) {
final int i = this.buf.getInt();
this.throwStatusError(this.buf, i);
}
throw new SftpException(4, "");
}
final SftpATTRS attr = SftpATTRS.getATTR(this.buf);
return attr;
}
catch (Exception e) {
if (e instanceof SftpException) {
throw (SftpException)e;
}
if (e instanceof Throwable) {
throw new SftpException(4, "", e);
}
throw new SftpException(4, "");
}
}

The error message comes from your server. It's indeed quite strange message, but I assume that it's some custom SFTP server that deals with some "messages" rather than plain files.
So the message basically translates to "Cannot find file" error of a traditional SFTP server. Even the error code 2 (SSH_FX_NO_SUCH_FILE) supports that.
Your path in remoteFile is probably wrong.

Unable to upload a file through multipart form data from java as well as postman

I have been trying to upload a file from java as well as postman. But I am unable to upload. The server is giving back the response as 200 Ok. But, the file is not being uploaded.
API Details:
I have an API for uploading file as "FileExplorerController". This API has a method "upload()" to upload the files. Url to access this method is"/fileupload". The API is working fine if I upload a file through HTML UI.
But I am trying to upload using Java. I have tried using Postman as well.
I have passed the multipart form data in several ways. But unable to resolve the issue. The code is as follows.
API - Upload - Function
public Result upload() {
String fileName="";
String folderPath="";
String fileDescription="";
String userName = "";
StopWatch stopWatch = null;
List<FileUploadStatusVo> fileStatus = new ArrayList<>();
try {
stopWatch = LoggerUtil.startTime("FileExplorerController -->
upload() : File Upload");
StringBuilder exceptionBuilder = new StringBuilder();
Http.MultipartFormData body =
play.mvc.Controller.request().body().asMultipartFormData();
Http.Context ctx = Http.Context.current();
userName = ctx.session().get(SessionUtil.USER_NAME);
String password = "";
if(StringUtils.isBlank(userName)) {
Map<String, String[]> formValues = play.mvc.Controller.
request().body().asMultipartFormData().asFormUrlEncoded();
if(formValues != null) {
if(formValues.get("userName") != null &&
formValues.get("userName").length > 0) {
userName = formValues.get("userName")[0];
}
if(formValues.get("password") != null &&
formValues.get("password").length > 0) {
password = formValues.get("password")[0];
}
}
if(StringUtils.isBlank(userName) ||
StringUtils.isBlank(password)) {
return Envelope.ok();
}
UserVo userVo = userService.findUserByEmail(userName);
boolean success = BCrypt.checkpw(password, userVo.password);
if(!success) {
return badRequest("Password doesn't match for the given user
name: "+userName);
}
if(userVo == null) {
return Envelope.ok();
}
}
boolean override = false;
String fileTags="";
boolean isPublicView = false;
boolean isPublicDownload = false;
boolean isPublicDelete = false;
boolean isEmailNotification = false;
boolean isEmailWithS3Link = false;
List<String> viewerGroupNames = new ArrayList<>();
List<String> downloaderGroupNames = new ArrayList<>();
List<String> deleterGroupNames = new ArrayList<>();
List<String> viewerUserNames = new ArrayList<>();
List<String> downloaderUserNames = new ArrayList<>();
List<String> deleterUserNames = new ArrayList<>();
List<String> emailIds = new ArrayList<>();
Map<String, String[]> formValues =
play.mvc.Controller.request().body().
asMultipartFormData().asFormUrlEncoded();
JSONObject obj = new JSONObject(formValues.get("model")[0]);
Set<String> groupNames = new HashSet<>();
Set<String> userNames = new HashSet<>();
if(obj != null) {
if(obj.get("override") != null) {
override = Boolean.valueOf(obj.get("override").toString());
}
if(obj.get("description") != null) {
fileDescription = obj.get("description").toString();
}
if(obj.get("tags") != null) {
fileTags = obj.get("tags").toString();
}
if(obj.get("folderPath") != null){
folderPath = obj.get("folderPath").toString();
} else {
folderPath =
ctx.session().get(SessionUtil.LOCAL_STORAGE_PATH);
}
if(obj.get("isPublicView") != null) {
isPublicView =
Boolean.parseBoolean(obj.get("isPublicView").toString());
}
if(obj.get("isPublicDownload") != null) {
isPublicDownload =
Boolean.parseBoolean(obj.get("isPublicDownload").toString());
}
if(obj.get("isPublicDelete") != null) {
isPublicDelete = Boolean.parseBoolean(
obj.get("isPublicDelete").toString());
}
if(obj.get("emailNotification") != null) {
isEmailNotification =
Boolean.parseBoolean(obj.get("emailNotification").toString());
}
if(obj.get("emailWithFileAttachement") != null) {
isEmailWithS3Link =
Boolean.parseBoolean(obj.get(
"emailWithFileAttachement").toString());
}
if(obj.get("viewerGroupNames") != null) {
//TODO
if(!isPublicView) {
String[] namesArr =
(obj.get("viewerGroupNames").toString()).split(",");
for(String name:namesArr) {
if(StringUtils.isNotEmpty(name)) {
viewerGroupNames.add(name);
groupNames.add(name);
}
}
}
}
if(obj.get("downloaderGroupNames") != null) {
//TODO
if(!isPublicDownload) {
String[] namesArr =
(obj.get("downloaderGroupNames").toString().split(","));
for(String name:namesArr){
if(StringUtils.isNotEmpty(name)) {
downloaderGroupNames.add(name);
groupNames.add(name);
}
}
}
}
if(obj.get("deleteGroupNames") != null) {
//TODO
if(!isPublicDelete){
String[] namesArr =
(obj.get("deleteGroupNames").toString().split(","));
for(String name:namesArr){
if(StringUtils.isNotEmpty(name)) {
deleterGroupNames.add(name);
groupNames.add(name);
}
}
}
}
if(obj.get("viewerUserNames") != null) {
//TODO
if(!isPublicView) {
String[] namesArr =
(obj.get("viewerUserNames").toString()).split(",");
for(String name:namesArr) {
if(StringUtils.isNotEmpty(name)) {
viewerUserNames.add(name);
userNames.add(name);
}
}
}
}
if(obj.get("downloaderUserNames") != null) {
//TODO
if(!isPublicDownload) {
String[] namesArr =
(obj.get("downloaderUserNames").toString().split(","));
for(String name:namesArr){
if(StringUtils.isNotEmpty(name)) {
downloaderUserNames.add(name);
userNames.add(name);
}
}
}
}
if(obj.get("deleteUserNames") != null) {
//TODO
if(!isPublicDelete){
String[] namesArr =
(obj.get("deleteUserNames").toString().split(","));
for(String name:namesArr){
if(StringUtils.isNotEmpty(name)) {
deleterUserNames.add(name);
userNames.add(name);
}
}
}
}
if(obj.get("emailIds") != null) {
if(isEmailWithS3Link) {
String[] emailIdsArr =
(obj.get("emailIds").toString()).split(",");
for(String emailId:emailIdsArr){
if(StringUtils.isNotEmpty(emailId)){
emailIds.add(emailId);
}
}
}
}
}
if(groupNames.size() == 0 && userNames.size() == 0){
isEmailNotification = false;
}
List<Http.MultipartFormData.FilePart> files = body.getFiles();
boolean multiUpload = false;
if(files != null && files.size() > 1) {
multiUpload = true;
}
Logger.info("Total Number of files is to be uploaded:"+ files.size()
+" by user: " + userName);
int uploadCount = 0;
List<String> fileNames = new ArrayList<>();
List<String> fileMasters = new ArrayList<>();
FileMasterVo fileMasterVo = null;
UserVo userVo = userService.findUserByEmail(userName);
for(Http.MultipartFormData.FilePart uploadedFile: files) {
if (uploadedFile == null) {
return badRequest("File upload error for file " +
uploadedFile + " for file path: " + fileName);
}
uploadCount++;
String contentType = uploadedFile.getContentType();
String name = uploadedFile.getFile().getName();
Logger.info("Content Type: " + contentType);
Logger.info("File Name: " + fileName);
Logger.info("Name: " + name);
Logger.info("Files Processed : "+uploadCount+"/"+files.size()+"
for user: "+userName);
try {
String extension =
FileUtil.getExtension(uploadedFile.getFilename()).toLowerCase();
File renamedUploadFile =
FileUtil.moveTemporaryFile(System.getProperty("java.io.tmpdir"),
System.currentTimeMillis() + "_" +
uploadedFile.getFilename(), uploadedFile.getFile());
FileInputStream fis = new
FileInputStream(renamedUploadFile);
String errorMsg = "";
fileName = folderPath + uploadedFile.getFilename();
fileNames.add(uploadedFile.getFilename());
if(multiUpload) {
Logger.info("Attempting to upload file " + folderPath +
"/" + uploadedFile.getFilename());
fileMasterVo = fileService.upload(folderPath,fileName,
fileDescription, new Date(), fis, fis.available(),
extension, override,
fileTags, isPublicView, isPublicDownload,
isPublicDelete, viewerGroupNames, downloaderGroupNames,
deleterGroupNames, viewerUserNames,
downloaderUserNames,
deleterUserNames,userName,isEmailNotification);
} else if(fileName != null) {
Logger.info("Attempting to upload file " + fileName);
int index = fileName.lastIndexOf("/");
if (index > 1) {
fileMasterVo =
fileService.upload(folderPath,fileName, fileDescription,
new Date(), fis, fis.available(), extension, override,
fileTags, isPublicView, isPublicDownload,
isPublicDelete, viewerGroupNames, downloaderGroupNames,
deleterGroupNames, viewerUserNames,
downloaderUserNames,
deleterUserNames,userName,isEmailNotification);
} else {
errorMsg = "Root Folder MUST exist to upload any
file";
return badRequest(errorMsg);
}
} else {
errorMsg = "File Name is incorrect";
return badRequest(errorMsg);
}
createFileActivityLog(
fileMasterVo,userVo,ViewConstants.UPLOADED);
if (fileMasterVo != null && fileMasterVo.getId() != null) {
fileMasters.add(fileMasterVo.getId().toString());
}
} catch (Exception inEx) {
createErrorLog(userName,fileName,inEx);
exceptionBuilder.append("Exception occured in uploading
file: ");
exceptionBuilder.append(name);
exceptionBuilder.append(" are as follows ");
exceptionBuilder.append(ExceptionUtils.getStackTrace(inEx));
}
fileStatus.add(new
FileUploadStatusVo(uploadedFile.getFilename(),
fileMasterVo.getStatus()));
}
if(isEmailNotification){
fileService.sendNotificationForFile(folderPath,fileNames,
userName, groupNames,
userNames, ViewConstants.UPLOADED);
}
if (isEmailWithS3Link) {
//fileService.sendFileS3Link(folderPath, emailIds, fileMasters);
// Replacing sending S3 link with sending cdi specific link
fileService.sendFilesLink(emailIds, fileMasters);
}
String exceptions = exceptionBuilder.toString();
LoggerUtil.endTime(stopWatch);
if(!StringUtils.isBlank(exceptions)) {
Logger.error("Exception occured while uploading file: " +
fileName + " are as follows " + exceptions);
}
return Envelope.ok(fileStatus);
} catch (Exception inEx) {
createErrorLog(userName,fileName,inEx);
return badRequest("There is a system error please contact
support/administrator" );
} }
Client
**Client - Program**
multipart.addFormField("fileName",file.getAbsolutePath());
multipart.addFormField("folderPath","D/");
multipart.addFormField("fileDescription","Desc");
multipart.addFormField("userName","superadmin");
multipart.addFormField("password","admin");
multipart.addFormField("override","false");
multipart.addFormField("fileTags","tag");
multipart.addFormField("isPublicView","true");
multipart.addFormField("isPublicDownload","true");
multipart.addFormField("isPublicDelete","false");
multipart.addFormField("isEmailNotification","false");
multipart.addFormField("isEmailWithS3Link","true");*/
multipart.addFormField("file", input);
System.out.print("SERVER REPLIED: ");
for (String line : response)
{
System.out.print(line);
}
// synchronize(clientFolder, uploadFolder, true);
}
catch (MalformedURLException e)
{
e.printStackTrace();
}
catch (IOException e)
{
e.printStackTrace();
}

I am able to upload using the following code snippet.
Here "model" is a json object which contain all parameters.
DefaultHttpClient client = new DefaultHttpClient();
HttpEntity entity = MultipartEntityBuilder
.create()
.addTextBody("userName", userName)
.addTextBody("password", passWord)
.addBinaryBody("upload_file", new File(sourceFolder + "/" + fileName), ContentType.create("application/octet-stream"), fileName)
.addTextBody("model", object.toString())
.build();
HttpPost post = new HttpPost(uploadURL);
post.setEntity(entity);
HttpResponse response = null;
try {
response = client.execute(post);
if (response.getStatusLine().getStatusCode() == 200) {
logger.info("File " + file.getName() + " Successfully Uploaded At: " + destination);
} else {
logger.info("File Upload Unsuccessful");
}
logger.info("Response from server:" + response.getStatusLine());
} catch (ClientProtocolException e) {
logger.error("Client Protocol Exception");
logger.error(e.getMessage());

Realm findfirst() is returning null

realm newbie here and facing a problem in my project. So what I want to achieve is that, in my app when I click a photo I am saving a realm object inside my database as well as caching the image locally to be used later. Here is the code for both:
#Override
public void onPhotoSelected(PXImage photo) {
PXComposition composition = null;
PXPhoto photoParent = null;
for (PXComposition pxComposition : mSession.getCompositions()) {
if (pxComposition.getItems() == null || pxComposition.getItems().size() < 1)
continue;
for (PXPhoto item : pxComposition.getItems()) {
if (item.getImage().getOriginalPath().equals(photo.getOriginalPath())) {
composition = pxComposition;
photoParent = item;
break;
}
}
}
if (composition == null && photo.isSelected()) {
mRealm.beginTransaction();
String uuid = mSession.addPhoto(photo).getUuid();
mRealm.commitTransaction(); // IMPORTANT to commit transaction. ContentDownloadTask requires PXImage to be written in Realm.
ThreadUtils.getDefaultExecutorService().submit(new ContentDownloadTask(this, photo.getOriginalPath(), uuid));
} else if (composition != null && !photo.isSelected()) {
mRealm.beginTransaction();
if (photoParent.getImage().isDownloaded()) // FIXME What if not copied yet ???
FileUtils.deleteFile(photoParent.getImage().getApplicationPath());
mSession.removePhoto(photoParent);
mRealm.commitTransaction();
}
onCheckChanged(photo);
App.log().v("SESSION", mSession.toString());
}
This is where click events on the photo are handled. After the selection click I am calling ThreadUtils.getDefaultExecutorService().submit(new ContentDownloadTask(this, photo.getOriginalPath(), uuid)); to cache the image locally. Here is the code for the ContentDownloadTask:
public class ContentDownloadTask implements Callable<Void> {
private String id;
private String path;
private Context context;
private Realm mRealm;
public static final String TAG = "ContentDownloadTask";
private static final String DIR_EXTERNAL_NAME = "Local";
private static final String FILE_EXTENSION = ".jpg";
public ContentDownloadTask(Context context, String path, String id) {
this.path = path;
this.id = id;
this.context = context;
}
#Override
public Void call() throws Exception {
try {
Log.d(TAG, "Copying From " + path);
//create output directory if it doesn't exist
final File file = context.getExternalFilesDir(Environment.DIRECTORY_PICTURES);
if (file == null) {
throw new IllegalStateException("Failed to get external storage files directory");
} else if (file.exists()) {
if (!file.isDirectory()) {
throw new IllegalStateException(file.getAbsolutePath() +
" already exists and is not a directory");
}
} else {
if (!file.mkdirs()) {
throw new IllegalStateException("Unable to create directory: " +
file.getAbsolutePath());
}
}
File to = new File(file, DIR_EXTERNAL_NAME);
if (to.exists()) {
if (!to.isDirectory()) {
throw new IllegalStateException(file.getAbsolutePath() +
" already exists and is not a directory");
}
} else if (!to.mkdirs()) {
throw new IllegalStateException("Unable to create directory: " +
file.getAbsolutePath());
}
to = new File(to, fileName(this.id));
if (!to.exists())
to.createNewFile();
InputStream in = null;
if (PackageUtils.isContentUri(path)) {
Uri uri = PackageUtils.toUri(path);
in = context.getContentResolver().openInputStream(uri);
} else {
File from = new File(this.path);
in = new FileInputStream(from);
}
OutputStream out = new FileOutputStream(to);
// Transfer bytes from in to out
byte[] buf = new byte[1024];
int len;
while ((len = in.read(buf)) > 0) {
out.write(buf, 0, len);
}
in.close();
out.close();
mRealm = Realm.getDefaultInstance();
PXImage image = mRealm.where(PXImage.class).equalTo("uuid", this.id).findFirst();
if (image == null) {
to.delete();
throw new Exception("Photo Not Found ID: " + this.id);
}
mRealm.beginTransaction();
image.setApplicationPath(to.getPath());
mRealm.commitTransaction();
mRealm.close();
Log.d(TAG, "Complete Copied From " + path + " To " + to.getAbsolutePath());
} catch (Exception e) {
Log.e(TAG, e.getMessage());
}
return null;
}
Now the null is returned at this statement: PXImage image = mRealm.where(PXImage.class).equalTo("uuid", this.id).findFirst();. Basically here I need to save the application path of the image that I cached locally inside the realm object. But it's returning null. Also this not happening every time I click the photos. This only happens sometimes and the error is not reproducible. Any kind of help will be appreciated. I have already checked the following duplicate questions:
first
second
third
fourth
public PXPhoto addPhoto(PXImage image) {
PXComposition composition = null;
PXPhoto photo = null;
boolean isLandscape = false;
int photosPerItem = getProduct().getSelectedPhotosPerItem();
switch (getProduct().getRootShortCode()) {
case Product.CategoryType.SQUARES:
case Product.CategoryType.CLASSIC:
case Product.CategoryType.WALLOGRAPHS:
case Product.CategoryType.SIGNATURES:
case Product.CategoryType.MOSAIC:
setLayoutType(image.isLandscape() ? C.LayoutType.LANDSCAPE.ordinal() : C.LayoutType.PORTRAIT.ordinal());
composition = PXComposition.initializeNewComposition(this);
photo = PXPhoto.initializePhoto(image);
break;
case Product.CategoryType.PANORAMA:
composition = PXComposition.initializeNewComposition(this);
photo = PXPhoto.initializePhoto(image);
break;
case Product.CategoryType.STRIPS:
case Product.CategoryType.POSTERS:
if (getCompositions() == null || getCompositions().size() == 0
|| (getProduct().getRootShortCode().equals(Product.CategoryType.STRIPS) &&
getCompositions().last().getItems().size() % photosPerItem == 0))
composition = PXComposition.initializeNewComposition(this);
else
composition = getCompositions().last();
photo = PXPhoto.initializePhoto(image);
break;
}
composition.addItem(photo);
composition.updateComposition();
if (!composition.isManaged()) {
Realm realm = Realm.getDefaultInstance();
composition = realm.copyToRealmOrUpdate(composition);
realm.close();
photo.setComposition(composition);
addComposition(composition);
} else
photo.setComposition(composition);
return photo;
}
EDIT:
uuid generation:
public PXPhoto() {
this.uuid = UUID.randomUUID().toString();
this.autoEnhance = false;
this.zoom = 1;
}

Processing huge File quickly in Java

I have a huge CSV file ~800 K records and using that data I have to form a POST request and make a rest call.
Initially I tried WITHOUT using threads but it is taking very long time to process it.
So I thought of using threads to speed up the process and followed below approach.
divided file into relatively smaller files (Tried with 3 files with approx ~5K data each file). (I Did this Manually Before passing to application)
Created 3 threads (traditional way by extending Thread class)
Reads each file with each thread and form HashMap with details required to form request and added it to ArrayList of POST Request
Send Post request to in loop
List item
Get the Response and add it to Response List
Both above approach takes too long to even complete half (>3 hrs). Might be one of them would not be good approach.
Could anyone please provide suggestion which helps me speeding up the process ? It is NOT mandatory to use threading.
Just to add that my code is at the consumer end and does not have much control on producer end.
MultiThreadedFileRead (Main class that Extends Thread)
class MultiThreadedFileRead extends Thread
{
public static final String EMPTY = "";
public static Logger logger = Logger.getLogger(MultiThreadedFileRead.class);
BufferedReader bReader = null;
String filename = null;
int Request_Num = 0;
HashMap<Integer, String> filteredSrcDataMap = null;
List<BrandRQ> brandRQList = null;
List<BrandRS> brandRSList = null;
ReadDLShipFile readDLShipFile = new ReadDLShipFile();
GenerateAndProcessPostBrandAPI generateAndProcessPostBrandAPI = new GenerateAndProcessPostBrandAPI();
MultiThreadedFileRead()
{
}
MultiThreadedFileRead(String fname) throws Exception
{
filename = fname;
Request_Num = 0;
List<BrandRQ> brandRQList = new ArrayList<BrandRQ>();
List<BrandRS> brandRSList = new ArrayList<BrandRS>();
this.start();
}
public void run()
{
logger.debug("File *** : " + this.filename);
//Generate Source Data Map
HashMap<Integer, String> filteredSrcDataMap = (HashMap<Integer, String>) readDLShipFile.readSourceFileData(this.filename);
generateAndProcessPostBrandAPI.generateAndProcessPostRequest(filteredSrcDataMap, this);
}
public static void main(String args[]) throws Exception
{
String environment = ""; // TO BE UNCOMMENTED
String outPutfileName = ""; // TO BE UNCOMMENTED
BrandRetrivalProperties brProps = BrandRetrivalProperties.getInstance();
if(args != null && args.length == 2 && DLPremiumUtil.isNotEmpty(args[0]) && DLPremiumUtil.isNotEmpty(args[1])) // TO BE UNCOMMENTED
{
environment = args[0].trim();
outPutfileName = args[1].trim();
ApplicationInitializer applicationInitializer = new ApplicationInitializer(environment);
applicationInitializer.initializeParams();
logger.debug("Environment : " + environment);
logger.debug("Filename : " + outPutfileName);
// TO BE UNCOMMENTED - START
}else{
System.out.println("Invalid parameters passed. Expected paramenters: Environment");
System.out.println("Sample Request: java -jar BrandRetrival.jar [prd|int] brand.dat");
System.exit(1);
}
MultiThreadedFileRead multiThreadedFileRead = new MultiThreadedFileRead();
List<String> listOfFileNames = multiThreadedFileRead.getFileNames(brProps);
logger.debug("List : " + listOfFileNames);
int totalFileSize = listOfFileNames.size();
logger.debug("totalFileSize : " + totalFileSize);
MultiThreadedFileRead fr[]=new MultiThreadedFileRead[totalFileSize];
logger.debug("Size of Array : " + fr.length);
for(int i = 0; i < totalFileSize; i++)
{
logger.debug("\nFile Getting Added to Array : " + brProps.getCountFilePath()+listOfFileNames.get(i));
fr[i]=new MultiThreadedFileRead(brProps.getCountFilePath()+listOfFileNames.get(i));
}
}
public List<String> getFileNames(BrandRetrivalProperties brProps)
{
MultiThreadedFileRead multiThreadedFileRead1 = new MultiThreadedFileRead();
BufferedReader br = null;
String line = "";
String filename = multiThreadedFileRead1.getCounterFileName(brProps.getCountFilePath(), brProps.getCountFilePathStartsWith());
List<String> fileNameList = null;
logger.debug("filename : " + filename);
if(filename != null)
{
fileNameList = new ArrayList<String>();
try{
br = new BufferedReader(new FileReader(brProps.getCountFilePath()+filename));
while ((line = br.readLine()) != null)
{
fileNameList.add(line);
}
} catch (FileNotFoundException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (IOException e2) {
// TODO Auto-generated catch block ;
e2.printStackTrace();
} catch (Exception e3) {
// TODO Auto-generated catch block
e3.printStackTrace();
} finally
{
if (br != null)
{
try
{
br.close();
} catch (IOException e4)
{
e4.printStackTrace();
} catch (Exception e5)
{
e5.printStackTrace();
}
}
}
}
return fileNameList;
}
// Get the Name of the file which has name of individual CSV files to read to form the request
public String getCounterFileName(String sourceFilePath, String sourceFileNameStartsWith)
{
String fileName = null;
if(sourceFilePath != null && !sourceFilePath.equals(EMPTY) &&
sourceFileNameStartsWith != null && !sourceFileNameStartsWith.equals(EMPTY))
{
File filePath = new File(sourceFilePath);
File[] listOfFiles = filePath.listFiles();
for (int i = 0; i < listOfFiles.length; i++) {
if (listOfFiles[i].isFile() && listOfFiles[i].getName().startsWith(sourceFileNameStartsWith)) {
fileName = listOfFiles[i].getName();
} else if (listOfFiles[i].isDirectory()) {
fileName = null;
}
}
}
return fileName;
}
}
GenerateAndProcessPostBrandAPI
public class GenerateAndProcessPostBrandAPI {
public static Logger logger = Logger.getLogger(GenerateAndProcessPostBrandAPI.class);
List<BrandRQ> brandRQList = new ArrayList<BrandRQ>();
List<BrandRS> brandRSList = new ArrayList<BrandRS>();
DLPremiumClientPost dlPremiumclientPost = new DLPremiumClientPost();
JSONRequestGenerator jsonRequestGenerator = new JSONRequestGenerator();
public List<BrandRQ> getBrandRequstList(Map<Integer, String> filteredSrcDataMap)
{
if(filteredSrcDataMap != null)
{
brandRQList = jsonRequestGenerator.getBrandRQList(filteredSrcDataMap, brandRQList);
}
return brandRQList;
}
public void generateAndProcessPostRequest(Map<Integer, String> filteredSrcDataMap, MultiThreadedFileRead multiThreadedFileRead)
{
if(filteredSrcDataMap != null)
{
brandRQList = jsonRequestGenerator.getBrandRQList(filteredSrcDataMap, brandRQList);
if(brandRQList != null && brandRQList.size() > 0)
{
BrandRetrivalProperties brProps = BrandRetrivalProperties.getInstance();
String postBrandsEndpoint = brProps.getPostBrandsEndPoint();
for (BrandRQ brandRQ : brandRQList)
{
multiThreadedFileRead.Request_Num = multiThreadedFileRead.Request_Num + 1;
logger.debug("Brand Request - " + multiThreadedFileRead.Request_Num + " : " + ObjectToJSONCovertor.converToJSON(brandRQ));
BrandRS brandResponse = dlPremiumclientPost.loadBrandApiResponseJSON(brandRQ, postBrandsEndpoint, DLPremiumUtil.getTransactionID(), BrandConstants.AUTH_TOKEN);
logger.debug("Response - " + multiThreadedFileRead.Request_Num + " : " + ObjectToJSONCovertor.converToJSON(brandResponse));
brandRSList.add(brandResponse);
}
}
}
}
}

SNMP code that does not work with TCP

I got one example of SNMPV3 trap,it is working fine when i use DefaultUdpTransportMapping but when i use DefaultTcpTransportMapping ,it will return null response event.
I have also attached code below .
public class SnmpUtilSendTrapV3 {
private Snmp snmp = null;
private Address targetAddress = null;
// private TcpAddress targetAddress = null;
public void initComm() throws IOException {
targetAddress = new TcpAddress(Inet4Address.getLocalHost(), 162);
// targetAddress = GenericAddress.parse("127.0.0.1/162");
TransportMapping transport = new DefaultTcpTransportMapping();
snmp = new Snmp(transport);
snmp.listen();
}
/**
* send trap
*
* #throws IOException
*/
public void sendPDU() throws IOException {
UserTarget target = new UserTarget();
target.setAddress(targetAddress);
target.setRetries(2);
target.setTimeout(1500);
// snmp version
target.setVersion(SnmpConstants.version3);
// target.setSecurityLevel(SecurityLevel.NOAUTH_NOPRIV);
target.setSecurityLevel(SecurityLevel.AUTH_PRIV);
target.setSecurityName(new OctetString("MD5DES"));
USM usm = new USM(SecurityProtocols.getInstance(), new OctetString(
MPv3.createLocalEngineID()), 0);
usm.setEngineDiscoveryEnabled(true);
SecurityModels.getInstance().addSecurityModel(usm);
UsmUser user = new UsmUser(new OctetString("MD5DES"), AuthMD5.ID,
new OctetString("MD5DESUserAuthPassword"), PrivDES.ID,
new OctetString("MD5DESUserPrivPassword"));
snmp.getUSM().addUser(new OctetString("MD5DES"), user);
// create PDU
ScopedPDU pdu = new ScopedPDU();
pdu.add(new VariableBinding(new OID("1.3.6.1.2.1.1.3.0"),
new OctetString("DemoTrapv3")));
pdu.add(new VariableBinding(new OID("1.3.6.1.2.1.1.5.0"),
new OctetString("Demo")));
pdu.setType(PDU.TRAP);
// send PDU to Agent and recieve Response
ResponseEvent respEvnt = snmp.send(pdu, target);
// analyze Response
if (respEvnt != null && respEvnt.getResponse() != null) {
Vector<VariableBinding> variableBindings = (Vector<VariableBinding>) respEvnt
.getResponse().getVariableBindings();
Vector<VariableBinding> recVBs = variableBindings;
for (int i = 0; i < recVBs.size(); i++) {
VariableBinding recVB = recVBs.elementAt(i);
System.out
.println(recVB.getOid() + " : " + recVB.getVariable());
}
}
snmp.close();
}
public static void main(String[] args) {
try {
SnmpUtilSendTrapV3 util = new SnmpUtilSendTrapV3();
util.initComm();
util.sendPDU();
} catch (IOException e) {
e.printStackTrace();
}
}
}
public class MultiThreadedTrapReceiver implements CommandResponder {
private Address address = GenericAddress.parse("0.0.0.0/162");
private int numDispatcherThreads = 2;
private OID authProtocol = AuthMD5.ID;
private OID privProtocol = PrivDES.ID;
private OctetString securityName = new OctetString("MD5DES");
private OctetString privPassphrase = new OctetString(
"MD5DESUserPrivPassword");
private OctetString authPassphrase = new OctetString(
"MD5DESUserAuthPassword");
public MultiThreadedTrapReceiver() {
try {
listen();
} catch (IOException ex) {
System.out.println(ex);
}
}
public synchronized void listen() throws IOException {
AbstractTransportMapping transport;
if (address instanceof TcpAddress) {
transport = new DefaultTcpTransportMapping((TcpAddress) address);
} else {
transport = new DefaultUdpTransportMapping((UdpAddress) address);
}
ThreadPool threadPool = ThreadPool.create("DispatcherPool",
numDispatcherThreads);
MessageDispatcher mtDispatcher = new MultiThreadedMessageDispatcher(
threadPool, new MessageDispatcherImpl());
// add message processing models
mtDispatcher.addMessageProcessingModel(new MPv1());
mtDispatcher.addMessageProcessingModel(new MPv2c());
mtDispatcher.addMessageProcessingModel(new MPv3(new OctetString(MPv3
.createLocalEngineID()).getValue()));
// add all security protocols
SecurityProtocols.getInstance().addDefaultProtocols();
SecurityProtocols.getInstance().addPrivacyProtocol(new Priv3DES());
Snmp snmp = new Snmp(mtDispatcher, transport);
USM usm = new USM(SecurityProtocols.getInstance(), new OctetString(
MPv3.createLocalEngineID()), 0);
SecurityModels.getInstance().addSecurityModel(usm);
// Add the configured user to the USM
addUsmUser(snmp);
snmp.addCommandResponder(this);
transport.listen();
try {
this.wait();
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
private void addUsmUser(Snmp snmp) {
snmp.getUSM().addUser(
securityName,
new UsmUser(securityName, authProtocol, authPassphrase,
privProtocol, privPassphrase));
}
#Override
public void processPdu(CommandResponderEvent respEvnt) {
System.out.println(respEvnt.getPDU());
InetAddress pduAgentAddress = null;
// System.out.println(respEvnt.getPDU() + " recieved;");
// this.setPdu(respEvnt.getPDU());
OctetString community = new OctetString(respEvnt.getSecurityName());
System.out.println("community: " + community.toString());
// handle the SNMP v1
if (respEvnt.getPDU().getType() == PDU.V1TRAP) {
Address address = respEvnt.getPeerAddress();
String hostName = address.toString().split("/")[0];
int nPort = Integer.parseInt(address.toString().split("/")[1]);
try {
pduAgentAddress = InetAddress.getByName(hostName);
} catch (UnknownHostException ex) {
}
System.out.println("hostname: " + pduAgentAddress.getHostAddress()
+ "; port: " + nPort);
} else {
Address address = respEvnt.getPeerAddress();
String hostName = address.toString().split("/")[0];
int nPort = Integer.parseInt(address.toString().split("/")[1]);
try {
pduAgentAddress = InetAddress.getByName(hostName);
} catch (UnknownHostException ex) {
}
System.out.println("hostname: " + pduAgentAddress.getHostAddress()
+ "; port: " + nPort);
}
}
public static void main(String[] args) {
MultiThreadedTrapReceiver trap = new MultiThreadedTrapReceiver();
}
}

I have changed in SnmpUtilSendTrapV3 class,
.use TcpAddress targetAddress instead of Address targetAddress
use targetAddress = new TcpAddress(Inet4Address.getLocalHost(),
162); instead of targetAddress =
GenericAddress.parse("127.0.0.1/164");
TransportMapping transport = new DefaultTcpTransportMapping();
instead of TransportMapping transport = new
DefaultUdpTransportMapping();
after this implementation error is solved but , after sending pdu using snmp.send(pdu, target) everytime ResponseEvent have null value.
Anyone have a solution of this problem ,please help me