We Keep Coding

How to retrieve tables which exists in a pdf using AWS Textract in java

I found article below to do in python.
https://docs.aws.amazon.com/textract/latest/dg/examples-export-table-csv.html
also I used article below to extract text.
https://docs.aws.amazon.com/textract/latest/dg/detecting-document-text.html
but above article helped to get only text, I also used function "block.getBlockType()"
of Block but none of block returned its type as "CELL" even tables are there in image/pdf.
Help me found java library similar to "boto3" to extract all tables.

What I did, I created models of each dataset in the json response and can use this models to build a table view in jsf.
public static List<TableModel> getTablesFromTextract(TextractModel textractModel) {
List<TableModel> tables = null;
try {
if (textractModel != null) {
tables = new ArrayList<>();
List<BlockModel> tableBlocks = new ArrayList<>();
Map<String, BlockModel> blockMap = new HashMap<>();
for (BlockModel block : textractModel.getBlocks()) {
if (block.getBlockType().equals("TABLE")) {
tableBlocks.add(block);
}
blockMap.put(block.getId(), block);
}
for (BlockModel blockModel : tableBlocks) {
Map<Long, Map<Long, String>> rowMap = new HashMap<>();
for (RelationshipModel relationship : blockModel.getRelationships()) {
if (relationship.getType().equals("CHILD")) {
for (String id : relationship.getIds()) {
BlockModel cell = blockMap.get(id);
if (cell.getBlockType().equals("CELL")) {
long rowIndex = cell.getRowIndex();
long columnIndex = cell.getColumnIndex();
if (!rowMap.containsKey(rowIndex)) {
rowMap.put(rowIndex, new HashMap<>());
}
Map<Long, String> columnMap = rowMap.get(rowIndex);
columnMap.put(columnIndex, getCellText(cell, blockMap));
}
}
}
}
tables.add(new TableModel(blockModel, rowMap));
}
System.out.println("row Map " + tables.toString());
}
} catch (Exception e) {
LOG.error("Could not get table from textract model", e);
}
return tables;
}
private static String getCellText(BlockModel cell, Map<String, BlockModel> blockMap) {
String text = "";
try {
if (cell != null
&& CollectionUtils.isNotEmpty(cell.getRelationships())) {
for (RelationshipModel relationship : cell.getRelationships()) {
if (relationship.getType().equals("CHILD")) {
for (String id : relationship.getIds()) {
BlockModel word = blockMap.get(id);
if (word.getBlockType().equals("WORD")) {
text += word.getText() + " ";
} else if (word.getBlockType().equals("SELECTION_ELEMENT")) {
if (word.getSelectionStatus().equals("SELECTED")) {
text += "X ";
}
}
}
}
}
}
} catch (Exception e) {
LOG.error("Could not get cell text of table", e);
}
return text;
}
TableModel to create the view from:
public class TableModel {
private BlockModel table;
private Map<Long, Map<Long, String>> rowMap;
public TableModel(BlockModel table, Map<Long, Map<Long, String>> rowMap) {
this.table = table;
this.rowMap = rowMap;
}
public BlockModel getTable() {
return table;
}
public void setTable(BlockModel table) {
this.table = table;
}
public Map<Long, Map<Long, String>> getRowMap() {
return rowMap;
}
public void setRowMap(Map<Long, Map<Long, String>> rowMap) {
this.rowMap = rowMap;
}
#Override
public String toString() {
return table.getId() + " - " + rowMap.toString();
}

I have something similar:
public class AnalyzeDocument {
public DocumentModel startProcess(byte[] content) {
Region region = Region.EU_WEST_2;
TextractClient textractClient = TextractClient.builder().region(region)
.credentialsProvider(EnvironmentVariableCredentialsProvider.create()).build();
return analyzeDoc(textractClient, content);
}
public DocumentModel analyzeDoc(TextractClient textractClient, byte[] content) {
try {
SdkBytes sourceBytes = SdkBytes.fromByteArray(content);
Util util = new Util();
Document myDoc = Document.builder().bytes(sourceBytes).build();
List<FeatureType> featureTypes = new ArrayList<FeatureType>();
featureTypes.add(FeatureType.FORMS);
featureTypes.add(FeatureType.TABLES);
AnalyzeDocumentRequest analyzeDocumentRequest = AnalyzeDocumentRequest.builder().featureTypes(featureTypes)
.document(myDoc).build();
AnalyzeDocumentResponse analyzeDocument = textractClient.analyzeDocument(analyzeDocumentRequest);
List<Block> docInfo = analyzeDocument.blocks();
// util.displayBlockInfo(docInfo);
PageModel pageModel = util.getTableResults(docInfo);
DocumentModel documentModel = new DocumentModel();
documentModel.getPages().add(pageModel);
Iterator<Block> blockIterator = docInfo.iterator();
while (blockIterator.hasNext()) {
Block block = blockIterator.next();
log.debug("The block type is " + block.blockType().toString());
}
return documentModel;
} catch (TextractException e) {
System.err.println(e.getMessage());
}
return null;
}
and this is the util file:
public PageModel getTableResults(List<Block> blocks) {
List<Block> tableBlocks = new ArrayList<>();
Map<String, Block> blockMap = new HashMap<>();
for (Block block : blocks) {
blockMap.put(block.id(), block);
if (block.blockType().equals(BlockType.TABLE)) {
tableBlocks.add(block);
log.debug("added table: " + block.text());
}
}
PageModel page = new PageModel();
if (tableBlocks.size() == 0) {
return null;
}
int i = 0;
for (Block table : tableBlocks) {
page.getTables().add(generateTable(table, blockMap, i++));
}
return page;
}
private TableModel generateTable(Block table, Map<String, Block> blockMap, int index) {
TableModel model = new TableModel();
Map<Integer, Map<Integer, String>> rows = getRowsColumnsMap(table, blockMap);
model.setTableId("Table_" + index);
for (Map.Entry<Integer, Map<Integer, String>> entry : rows.entrySet()) {
RowModel rowModel = new RowModel();
Map<Integer, String> value = entry.getValue();
for (int i = 0; i < value.size(); i++) {
rowModel.getCells().add(value.get(i));
}
model.getRows().add(rowModel);
}
return model;
}
private Map<Integer, Map<Integer, String>> getRowsColumnsMap(Block block, Map<String, Block> blockMap) {
Map<Integer, Map<Integer, String>> rows = new HashMap<>();
for (Relationship relationship : block.relationships()) {
if (relationship.type().equals(RelationshipType.CHILD)) {
for (String childId : relationship.ids()) {
Block cell = blockMap.get(childId);
if (cell != null) {
int rowIndex = cell.rowIndex();
int colIndex = cell.columnIndex();
if (rows.get(rowIndex) == null) {
Map<Integer, String> row = new HashMap<>();
rows.put(rowIndex, row);
}
rows.get(rowIndex).put(colIndex, getText(cell, blockMap));
}
}
}
}
return rows;
}
public String getText(Block block, Map<String, Block> blockMap) {
String text = "";
if (block.relationships() != null && block.relationships().size() > 0) {
for (Relationship relationship : block.relationships()) {
if (relationship.type().equals(RelationshipType.CHILD)) {
for (String childId : relationship.ids()) {
Block wordBlock = blockMap.get(childId);
if (wordBlock != null && wordBlock.blockType() != null) {
if (wordBlock.blockType().equals(BlockType.WORD))) {
text += wordBlock.text() + " ";
}
}
}
}
}
}
return text;
}

Incompatible Type Error in Android Project

This is an Android project. I'm completely new to Java (just started learning). As stated in the title, I'm getting an Incompatible Type Error
I've attached the respective method here :
public void init(Map map) {
this.productIds = new ArrayList();
try {
if (map.containsKey("products")) {
for (Entry<String, Object> "//Error Lies here" entry : ((HashMap) map.get("products")).entrySet()) {
InAppProduct productId = new InAppProduct();
productId.productId = ((String) entry.getKey()).toLowerCase();
HashMap<String, Object> extraValues = (HashMap) entry.getValue();
if (extraValues.containsKey(ShareConstants.MEDIA_TYPE)) {
productId.productType = (String) extraValues.get(ShareConstants.MEDIA_TYPE);
}
if (extraValues.containsKey("days")) {
productId.days = ((Integer) extraValues.get("days")).intValue();
}
this.productIds.add(productId);
}
return;
}
this.productIds = new ArrayList(ConfigurationFetcher.this.mDefaultsDelegate.getDefaultsInAppPackages());
} catch (Exception e) {
e.printStackTrace();
}
}
The Error is :
Required Object but found Entry <String, Object>
Let me know if you need additional code or any details. Thank You.

Set is a generic type. It is a container that can contain any kind of object.
In your case, it seems that your Set contains Map.Entry<String, Object> objects but since you don't specify that anywhere, Java assumes your Set contains Objects (the Java class that all other classes derive from) and produces an Incompatible Type Error.
Here's a slightly altered version of your code that should work.
public void init(Map map) {
this.productIds = new ArrayList();
try {
if (map.containsKey("products")) {
// ***** We now specify the type of object that the Set contains.
Set<Map.Entry<String, Object>> entrySet = ((HashMap) hm.get("products")).entrySet();
for (Entry<String, Object> entry : entrySet) {
InAppProduct productId = new InAppProduct();
productId.productId = ((String) entry.getKey()).toLowerCase();
HashMap<String, Object> extraValues = (HashMap) entry.getValue();
if (extraValues.containsKey(ShareConstants.MEDIA_TYPE)) {
productId.productType = (String) extraValues.get(ShareConstants.MEDIA_TYPE);
}
if (extraValues.containsKey("days")) {
productId.days = ((Integer) extraValues.get("days")).intValue();
}
this.productIds.add(productId);
}
return;
}
this.productIds = new ArrayList(ConfigurationFetcher.this.mDefaultsDelegate.getDefaultsInAppPackages());
} catch (Exception e) {
e.printStackTrace();
}
}

map.get("products")).entrySet() is a set of products, each product is a Object, not Entry <String, Object>.
This should work:
public void init(Map map) {
this.productIds = new ArrayList();
try {
if (map.containsKey("products")) {
for (Object entry : ((HashMap) map.get("products")).entrySet()) {
InAppProduct productId = new InAppProduct();
productId.productId = ((String) entry.getKey()).toLowerCase();
HashMap<String, Object> extraValues = (HashMap) entry.getValue();
if (extraValues.containsKey(ShareConstants.MEDIA_TYPE)) {
productId.productType = (String) extraValues.get(ShareConstants.MEDIA_TYPE);
}
if (extraValues.containsKey("days")) {
productId.days = ((Integer) extraValues.get("days")).intValue();
}
this.productIds.add(productId);
}
return;
}
this.productIds = new ArrayList(ConfigurationFetcher.this.mDefaultsDelegate.getDefaultsInAppPackages());
} catch (Exception e) {
e.printStackTrace();
}
}

Kafka Streams not sending records to output topics

I have a Kafka Streams Application that is receiving records to an input topic doing some stream processing on it and sending processed records to multiple output topics. It was running perfectly fine until I stopped it.
After stopping the first Streams app, I created a different streams application that has different input and output topic.
Next, I started both applications and my second application worked perfect but my first application has stopped doing any processing at all.
When I start a console-consumer on input topics I can see records being produced, but when I start console-consumer on output topics I am not receiving any records. I don't what could have gone wrong. I thought may be its because of the second streams application. So I stopped it and recreated all the topics and restarted the first application again but its still not doing anything.
Note: I am doing this on a local server.
Next I build the streams application on my personal machine and its working as expected.
What could have gone wrong that I am seeing this unexpectedly strange behavior?
Here is my code:
Main Class:
public class Pipe {
static Logger log = Logger.getLogger(Pipe.class.getName());
public static void main(String[] args) throws Exception {
PropertyConfigurator.configure("log4j.properties");
log.info("Starting application");
Map<String, String> env = System.getenv();
Properties props = new Properties();
String BROKER_URL = env.get("BROKER_URL");
String appId = "98aff1c5-7a69-46b7-899c-186851054b43";
String appSecret = "zVyS/V694ffWe99QpCvYqE1sqeqLo36uuvTL8gmZV0A=";
String appTenant = "2f6cb1a6-ecb8-4578-b680-bf84ded07ff4";
props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pipe");
props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); // pass from env localhost:9092 | BROKER_URL + ":9092"
props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
final StreamsBuilder builder = new StreamsBuilder();
log.info("Creating stream: o365_storage");
KStream<String, String> source_o365_storage = builder.stream("o365_storage");
log.info("Creating stream: scan_result_dlp");
KStream<String, String> source_scan_result_dlp = builder.stream("scan_result_dlp");
log.info("Creating stream: scan_result_malware");
KStream<String, String> source_scan_result_malware = builder.stream("scan_result_malware");
log.info("Creating stream: source_o365_user_contenturl");
//KStream<String, String> source_o365_contenturl = builder.stream("o365_activity_contenturl");
KStream<String, String> source_o365_user_contenturl = builder.stream("o365_activity_contenturl");
log.info("Creating stream: source_o365_contenturl_result");
KStream<String, String> source_o365_contenturl_result = source_o365_user_contenturl.flatMapValues(new ValueMapper<String, Iterable<String>>() {
#Override
public Iterable<String> apply(String value) {
ArrayList<String> keywords = new ArrayList<String>();
ExecutorService executor = new ThreadPoolExecutor(4, 4, 1, TimeUnit.MINUTES, new LinkedBlockingQueue<Runnable>());
try {
String accessToken = O365Util.getAccessToken(appId, appSecret, appTenant);
System.out.println("accessToken : " + accessToken);
System.out.println("Creating futures..");
List<Future<?>> futures = new ArrayList<Future<?>>();
JSONArray contentUrlList = new JSONArray(value);
for (int i = 0; i < contentUrlList.length(); i++) {
JSONObject contentUri = contentUrlList.getJSONObject(i);
//futures.add(executor.submit(new FetchLogService(accessToken, contentUri.getString("contentUri"))));
futures.add(executor.submit(new FetchLogService(accessToken, contentUri, appTenant)));
}
System.out.println("futures size is : " + futures.size());
for (Future<?> f : futures) {
if (f.get() != null) {
//System.out.println("Executing contentUri parallel....................... ");
String futureResult = f.get().toString();
if (String.valueOf(futureResult.charAt(0)).equalsIgnoreCase("[")) {
//System.out.println("futureResult is JSONArray");
JSONArray logList = new JSONArray(futureResult);
for (int k = 0; k < logList.length(); k++) {
JSONObject log = logList.getJSONObject(k);
//System.out.println("Added logs into Events for action : " + log.getString("Operation"));
keywords.add(log.toString());
}
} else {
System.out.println("futureResult is JSONObject");
JSONObject contentUrlObj = new JSONObject(futureResult);
keywords.add(contentUrlObj.toString());
}
} else {
System.out.println("future result is nullllllllllllllllllllllllllllllllllllllll");
}
}
} catch (Exception e) {
System.err.println("Unable to convert to json");
e.printStackTrace();
} finally {
executor.shutdownNow();
}
return keywords;
}
});
log.info("Creating stream: source_o365_user_activity_intermediate");
KStream<String, String> source_o365_user_activity_intermediate = source_o365_contenturl_result.flatMapValues(new ValueMapper<String, Iterable<String>>() {
#Override
public Iterable<String> apply(String value) {
ArrayList<String> keywords = new ArrayList<String>();
try {
if (value.contains("Operation\":")) {
keywords.add(value);
}
} catch (Exception e) {
System.err.println("Unable to convert to json");
e.printStackTrace();
}
return keywords;
}
});
source_o365_user_activity_intermediate.to("o365_user_activity");
log.info("Creating stream: o365_contenturls");
KStream<String, String> o365_contenturls = source_o365_contenturl_result.flatMapValues(new ValueMapper<String, Iterable<String>>() {
#Override
public Iterable<String> apply(String value) {
ArrayList<String> keywords = new ArrayList<String>();
try {
if (value.contains("contentUri\":")) {
keywords.add("["+value+"]");
}
} catch (Exception e) {
System.err.println("Unable to convert to json");
e.printStackTrace();
}
return keywords;
}
});
o365_contenturls.to("o365_activity_contenturl");
log.info("Creating stream: o365_user_activity");
KStream<String, String> source_o365_user_activity = builder.stream("o365_user_activity");
log.info("Creating branch: branches_source_o365_user_activity");
#SuppressWarnings("unchecked")
KStream<String, String>[] branches_source_o365_user_activity = source_o365_user_activity.branch(
(key, value) -> (value.contains("Operation\":\"SharingSet") && value.contains("ItemType\":\"File")), // Sharing Set by Date
(key, value) -> (value.contains("Operation\":\"AddedToSecureLink") && value.contains("ItemType\":\"File")), // Added to secure link
(key, value) -> (value.contains("Operation\":\"AddedToGroup")), // Added to group
(key, value) -> (value.contains("Operation\":\"Add member to role.") || value.contains("Operation\":\"Remove member from role.")),//Role update by date
(key, value) -> (value.contains("Operation\":\"FileUploaded") || value.contains("Operation\":\"FileDeleted")
|| value.contains("Operation\":\"FileRenamed") || value.contains("Operation\":\"FileMoved")), // Upload file by date
(key, value) -> (value.contains("Operation\":\"UserLoggedIn")), // User logged in by date
(key, value) -> (value.contains("Operation\":\"Delete user.") || value.contains("Operation\":\"Add user.")
&& value.contains("ResultStatus\":\"success")) // Manage user by date
);
log.info("Creating branch: branches1_source_o365_user_activity");
#SuppressWarnings("unchecked")
KStream<String, String>[] branches1_source_o365_user_activity = source_o365_user_activity.branch(
(key, value) -> (value.contains("Operation\":\"FileUploaded") || value.contains("Operation\":\"FileModified")
|| value.contains("Operation\":\"FileDeleted")), // File update by date
(key, value) -> (value.contains("Operation\":\"FileAccessed")) // File access by date
);
log.info("Creating branch: branches2_source_o365_user_activity");
#SuppressWarnings("unchecked")
KStream<String, String>[] branches2_source_o365_user_activity = source_o365_user_activity.branch(
(key, value) -> (value.contains("Operation\":\"FileUploaded") || value.contains("Operation\":\"FileModified")
|| value.contains("Operation\":\"FileDeleted") || value.contains("Operation\":\"SharingSet")
&& value.contains("ItemType\":\"File")) // File operation by date
);
log.info("Creating branch: branches3_source_o365_user_activity");
#SuppressWarnings("unchecked")
KStream<String, String>[] branches3_source_o365_user_activity = source_o365_user_activity.branch(
(key, value) -> (value.contains("Workload\":\"AzureActiveDirectory") || value.contains("Workload\":\"OneDrive") || value.contains("Workload\":\"SharePoint")) // Activity log by date
);
log.info("Creating branch: branches4_source_o365_user_activity");
#SuppressWarnings("unchecked")
KStream<String, String>[] branches4_source_o365_user_activity = source_o365_user_activity.branch(
(key, value) -> (value.contains("Operation\":\"FileUploaded") || value.contains("Operation\":\"FileModified")) // Download file for scanning
);
/////////////////////////////////========================= DLP LOGS ========================/////////////////////////////////////////////////////////
AppUtil.pushToTopic(source_scan_result_dlp, Constant.O365_GTB_BREACHED_POLICY_BY_DATE, "o365_gtb_dlp_breached_policy_by_date");
//////////////////////////////////////==================== MALWARE LOGS ================================////////////////////////////////////////////
AppUtil.pushToTopic(source_scan_result_malware, Constant.O365_LAST_LINE_MALWARE, "o365_last_line_malware");
//////////////////////////////////////==================== ALL LOGS ====================================////////////////////////////////////////////
AppUtil.pushToTopic(source_o365_user_activity, Constant.O365_USER_ACTIVITY_BY_DATE, "o365_user_activity_by_date");
////////////////////////////////////====================== STORAGE LOGS ====================================////////////////////////////////////////////
AppUtil.pushToTopic(source_o365_storage, Constant.O365_STORAGE_BY_DATE, "o365_storage_by_date");
//////////////////////////////////////==================== BRANCH LOGS ====================================////////////////////////////////////////////
AppUtil.pushToTopic(branches_source_o365_user_activity[0], Constant.O365_SHARING_SET_BY_DATE, "o365_sharing_set_by_date", Constant.O365_SHARING_SET_BY_DATE_EXCEP_KEYS);
AppUtil.pushToTopic(branches_source_o365_user_activity[1], Constant.O365_ADDED_TO_SECURE_LINK_BY_DATE, "o365_added_to_secure_link_by_date");
AppUtil.pushToTopic(branches_source_o365_user_activity[2], Constant.O365_ADDED_TO_GROUP_BY_DATE, "o365_added_to_group_by_date");
AppUtil.pushToTopic(branches_source_o365_user_activity[3], Constant.O365_ROLE_UPDATE_BY_DATE, "o365_role_update_by_date");
AppUtil.pushToTopic(branches_source_o365_user_activity[4], Constant.O365_UPLOAD_FILE_BY_DATE, "o365_upload_file_by_date", Constant.O365_UPLOAD_FILE_BY_DATE_EXCEP_KEYS);
AppUtil.pushToTopic(branches_source_o365_user_activity[5], Constant.O365_USER_LOGGED_IN_BY_DATE, "o365_user_logged_in_by_date");
AppUtil.pushToTopic(branches_source_o365_user_activity[6], Constant.O365_MANAGE_USER_BY_DATE, "o365_manage_user_by_date");
////////////////////////////////////====================== BRANCH 1 LOGS ====================================////////////////////////////////////////////
AppUtil.pushToTopic(branches1_source_o365_user_activity[0], Constant.O365_FILE_UPDATE_BY_DATE, "o365_file_update_by_date");
AppUtil.pushToTopic(branches1_source_o365_user_activity[1], Constant.O365_FILE_ACCESS_BY_DATE, "o365_file_access_by_date");
////////////////////////////////////====================== BRANCH 2 LOGS ====================================////////////////////////////////////////////
AppUtil.pushToTopic(branches2_source_o365_user_activity[0], Constant.O365_FILE_OPERATION_BY_DATE, "o365_file_operation_by_date");
////////////////////////////////////====================== BRANCH 3 LOGS ====================================////////////////////////////////////////////
AppUtil.pushToTopic(branches3_source_o365_user_activity[0], Constant.O365_ACTIVITY_LOG_BY_DATE, "o365_activity_log_by_date");
////////////////////////////////////====================== BRANCH 4 LOGS ====================================////////////////////////////////////////////
branches4_source_o365_user_activity[0].to("download_file_for_scanning");
final Topology topology = builder.build();
final KafkaStreams streams = new KafkaStreams(topology, props);
final CountDownLatch latch = new CountDownLatch(1);
// attach shutdown handler to catch control-c
Runtime.getRuntime().addShutdownHook(new Thread("streams-shutdown-hook") {
#Override
public void run() {
log.trace("Exiting application.");
streams.close();
latch.countDown();
}
});
try {
streams.start();
latch.await();
} catch (Throwable e) {
System.exit(1);
}
System.exit(0);
}
}
AppUtil:
public final class AppUtil {
static Logger log = Logger.getLogger(Pipe.class.getName());
public static HashMap createHashMap(String[] keys, String[] values) {
HashMap<String, String> hmap = new HashMap<String, String>();
for (int i = 0; i < values.length; i++) {
hmap.put(keys[i], values[i]);
}
return hmap;
}
public static void pushToTopic(KStream<String, String> sourceTopic, HashMap<String, String> hmap, String destTopicName) {
log.info(destTopicName+ " inside function");
System.out.println(destTopicName + " inside function");
sourceTopic.flatMapValues(new ValueMapper<String, Iterable<String>>() {
#Override
public Iterable<String> apply(String value) {
log.info("================================================================================================================================================================================");
log.info("========> " + destTopicName + " Log:\n \n" + value);
System.out.println("================================================================================================================================================================================");
System.out.println("========> " + destTopicName + " Log:\n \n" + value);
ArrayList<String> keywords = new ArrayList<String>();
try {
JSONObject send = new JSONObject();
JSONObject received = processJSON(new JSONObject(value), destTopicName);
send.put("current_date", getCurrentDate().toString());
if (!destTopicName.equals("o365_storage_by_date")) {
send.put("insertion_time", getCurrentDateTime().toString());
}
boolean valid_json = true;
for(String key: hmap.keySet()) {
if (received.has(hmap.get(key))) {
send.put(key, received.get(hmap.get(key)));
}
else {
System.out.println("\n \n Missing Key in JSON: Cannot send log to destination topic = " + destTopicName + " | " + hmap.get(key) + " Key is missing.");
log.error("\n \n Missing Key in JSON: Cannot send log to destination topic = " + destTopicName + " | " + hmap.get(key) + " Key is missing.");
valid_json = false;
}
}
if (valid_json) {
keywords.add(send.toString());
}
// apply regex to value and for each match add it to keywords
} catch (Exception e) {
// TODO: handle exception
log.error("Unable to convert to json");
System.err.println("Unable to convert to json");
e.printStackTrace();
}
return keywords;
}
}).to(destTopicName);
}
//////////////////////////////////////
public static void pushToTopic(KStream<String, String> sourceTopic, HashMap<String, String> hmap, String destTopicName, String[] exceptionalKeys) {
sourceTopic.flatMapValues(new ValueMapper<String, Iterable<String>>() {
#Override
public Iterable<String> apply(String value) {
log.info("================================================================================================================================================================================");
log.info("========> " + destTopicName + " Log:\n \n" + value);
System.out.println("================================================================================================================================================================================");
System.out.println("========> " + destTopicName + " Log:\n \n" + value);
ArrayList<String> keywords = new ArrayList<String>();
try {
JSONObject send = new JSONObject();
JSONObject received = processJSON(new JSONObject(value), destTopicName);
send.put("current_date", getCurrentDate().toString());
if (!destTopicName.equals("o365_storage_by_date")) {
send.put("insertion_time", getCurrentDateTime().toString());
}
boolean valid_json = true;
for(String key: hmap.keySet()) {
if (received.has(hmap.get(key))) {
send.put(key, received.get(hmap.get(key)));
}
else {
System.out.println("\n \n Missing Key in JSON: Sending log to destination topic = " + destTopicName + " with null value | " + hmap.get(key) + " Key is missing.");
log.warn("\n \n Missing Key in JSON: Sending log to destination topic = " + destTopicName + " with null value | " + hmap.get(key) + " Key is missing.");
if(!isExceptionalKey(exceptionalKeys, hmap.get(key))) {
valid_json = false;
}
}
}
if (valid_json) {
keywords.add(send.toString());
}
// apply regex to value and for each match add it to keywords
} catch (Exception e) {
// TODO: handle exception
log.error("Unable to convert to json");
System.err.println("Unable to convert to json");
e.printStackTrace();
}
return keywords;
}
}).to(destTopicName);
}
//////////////////////////////////////
private static boolean isExceptionalKey(String[] exceptionalKeys, String currKey) {
// TODO Auto-generated method stub
boolean isExceptionalKey = false;
for (String string : exceptionalKeys) {
if (string.equals(currKey)) {
isExceptionalKey = true;
break;
}
}
return isExceptionalKey;
}
public static JSONObject processJSON(JSONObject jsonObj, String destTopicName) {
if (jsonObj.has("UserId")) {
String val = jsonObj.get("UserId").toString().toLowerCase();
jsonObj.remove("UserId");
jsonObj.put("UserId", val);
}
if (jsonObj.has("TargetUserOrGroupName")) {
String val = jsonObj.get("TargetUserOrGroupName").toString().toLowerCase();
jsonObj.remove("TargetUserOrGroupName");
jsonObj.put("TargetUserOrGroupName", val);
}
if (jsonObj.has("ObjectId")) {
String val = jsonObj.get("ObjectId").toString().toLowerCase();
jsonObj.remove("ObjectId");
jsonObj.put("ObjectId", val);
}
if (jsonObj.has("EventData")) {
String val = jsonObj.get("EventData").toString().toLowerCase();
jsonObj.remove("EventData");
jsonObj.put("EventData", val);
}
if (destTopicName.equals("o365_last_line_malware")) {
jsonObj.put("MaliciousScore", "-1");
}
if (destTopicName.equals("o365_activity_log_by_date") || destTopicName.equals("o365_gtb_dlp_breached_policy_by_date")) {
jsonObj.put("ActivityDetail", jsonObj.toString());
}
return jsonObj;
}
public static String getCurrentDate() {
Date date = new Date();
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
String UTCdate = dateFormat.format(date);
return UTCdate;
}
private static String getCurrentDateTime() {
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date date = new Date();
String datetime = dateFormat.format(date);
return datetime;
}
}

Getting exception on PostUrlBuilder when build post URL for Volley Custom post request with array parameter with normal param

I am trying to build a post URL for Volley custom request to post some data to server using post method with normal param and array param. But I get some exception when build the post url for request. I am getting exception before make a request with volley custom request.
Map<String, String> params = new HashMap<>();
params.put(API.Parameter.ANDROID_DEVICE_ID, appManager.getDeviceId());
params.put(API.Parameter.ANDROID_APP_VERSION, appManager.getAppVersion());
params.put("trip_no", MainActivity.temp_trip_id);
Map<String, List<String>> arrayParams = new HashMap<>();
arrayParams.put("ticket_id",ticket_id);
arrayParams.put("ticket_name",ticket_name);
arrayParams.put("ticket_price_each",ticket_price_each);
PostUrlBuilder urlBuilder = new PostUrlBuilder(API.BOOK_TRIP_API_URL, params, arrayParams);
String Url = urlBuilder.getQueryUrl();
ObjectRequest<OnlineUserData> onlineUserDataObjectRequest = new ObjectRequest<OnlineUserData>(Request.Method.POST, Url, null,
new Response.Listener<OnlineUserData>() {
#Override
public void onResponse(OnlineUserData response) {
}
}, new Response.ErrorListener() {
#Override
public void onErrorResponse(VolleyError error) {
}
}, OnlineUserData.class);
AppController.getInstance().addToRequestQueue(onlineUserDataObjectRequest);
My PostUrlBuilder.java class is:
public class PostUrlBuilder {
private static final String TAG = PostUrlBuilder.class.getSimpleName();
private String url;
private Map<String, String> parameters;
private Map<String, List<String>> arrayParameters;
private String queryUrl;
public PostUrlBuilder() {
}
public PostUrlBuilder(String url, Map<String, String> parameters, Map<String, List<String>> arrayParameters) {
this.url = url;
this.parameters = parameters;
this.arrayParameters = arrayParameters;
buildQueryUrl();
}
public PostUrlBuilder(Map<String, String> parameters, String url) {
this.parameters = parameters;
this.url = url;
}
private void buildQueryUrl() {
StringBuilder urlBuilder = new StringBuilder(url);
if (parameters != null && parameters.size() != 0) {
urlBuilder.append("?");
int i = 0;
for (String key : parameters.keySet()) {
String value = parameters.get(key);
Log.d(TAG, "key= " + key + " & " + "value=" + value);
try {
urlBuilder.append(String.format("%s=%s", key, URLEncoder.encode(value, "UTf-8")));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
i++;
if (i != parameters.size()) {
urlBuilder.append("&");
}
}
}
if (arrayParameters != null && arrayParameters.size() != 0) {
if (parameters != null && parameters.size() != 0) {
urlBuilder.append("&");
} else {
urlBuilder.append("?");
}
int i = 0;
for (String key : arrayParameters.keySet()) {
List<String> values = arrayParameters.get(key);
int j = 0;
for (String value : values) {
Log.d(TAG, "size = " + values.size() + " j = " + j + " key= " + key + " & " + "value=" + value);
try {
urlBuilder.append(String.format("%s[]=%s", key, URLEncoder.encode(value, "UTf-8")));
j++;
if (j != values.size()) {
urlBuilder.append("&");
}
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
i++;
if (i != arrayParameters.size()) {
urlBuilder.append("&");
}
}
}
queryUrl = urlBuilder.toString();
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public Map<String, String> getParameters() {
return parameters;
}
public void setParameters(Map<String, String> parameters) {
this.parameters = parameters;
}
public String getQueryUrl() {
return queryUrl;
}
public void setQueryUrl(String queryUrl) {
this.queryUrl = queryUrl;
}
public Map<String, List<String>> getArrayParameters() {
return arrayParameters;
}
public void setArrayParameters(Map<String, List<String>> arrayParameters) {
this.arrayParameters = arrayParameters;
}
#Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof PostUrlBuilder)) return false;
PostUrlBuilder that = (PostUrlBuilder) o;
return this.getUrl().equals(that.getUrl())
&& this.getParameters().equals(that.getParameters())
&& this.getArrayParameters().equals(that.getArrayParameters())
&& this.getQueryUrl().equals(that.getQueryUrl());
}
#Override
public int hashCode() {
int result = getUrl().hashCode();
result = 31 * result + getParameters().hashCode();
result = 31 * result + getArrayParameters().hashCode();
result = 31 * result + getQueryUrl().hashCode();
return result;
}
#Override
public String toString() {
return "GetUrlBuilder{" +
"url='" + url + '\'' +
", parameters=" + parameters +
", queryUrl='" + queryUrl + '\'' +
'}';
}
}
I am getting exception in line below:
urlBuilder.append(String.format("%s[]=%s", key, URLEncoder.encode(value, "UTf-8")));
Error Log-cat shows errors below :
FATAL EXCEPTION: main
Process: com.shohoz.launch.cabin, PID: 12230
java.lang.NullPointerException
at libcore.net.UriCodec.encode(UriCodec.java:132)
at java.net.URLEncoder.encode(URLEncoder.java:57)
at com.shohoz.launch.cabin.toolbox.PostUrlBuilder.buildQueryUrl(PostUrlBuilder.java:70)
at com.shohoz.launch.cabin.toolbox.PostUrlBuilder.<init>(PostUrlBuilder.java:29)
at com.shohoz.launch.cabin.fragment.RightFullSeatLayoutFragment.onlineCreateAndInsertIntoConfirm(RightFullSeatLayoutFragment.java:1105)
at com.shohoz.launch.cabin.fragment.RightFullSeatLayoutFragment$15.onClick(RightFullSeatLayoutFragment.java:857)
at android.support.v7.app.AlertController$ButtonHandler.handleMessage(AlertController.java:157)
at android.os.Handler.dispatchMessage(Handler.java:102)
at android.os.Looper.loop(Looper.java:136)
at android.app.ActivityThread.main(ActivityThread.java:5398)
at java.lang.reflect.Method.invokeNative(Native Method)
at java.lang.reflect.Method.invoke(Method.java:515)
at com.android.internal.os.ZygoteInit$MethodAndArgsCaller.run(ZygoteInit.java:864)
at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:680)
at dalvik.system.NativeStart.main(Native Method)
Thanks in advance.
I've Figured out arrayParams.put("ticket_price_each",ticket_price_each); sending null value. That's why app was crashed.

Java output JSON to CSV file

I'm not too familiar with how to output files back to the client with Java. I am trying to create a CSV file to be sent back to the client and opened in Excel.
I found this tool for the server side creation. I'm not sure exactly how to use it to return the actual file though. Here is a sample of code I have used to return a txt file that I think I can use parts of the response for, but I'm not fetching a file anymore since I'm creating this CSV so I'm not sure what I can use.
In the code below my biggest question is what do I have to return with the controller and how do I accomplish that? I'm not sure what I need to be returning between that and also from the CSV writer to the controller. Any help would be appreciated.
Here's my code so far:
Controller:
#RequestMapping(value = "/web/csvexport", method = RequestMethod.POST)
protected void processCSV(HttpServletRequest request, HttpServletResponse response, #RequestBody String jsonRequest)
throws ServletException, IOException {
response.setContentType("text/html;charset=UTF-8");
try {
CSVWriter csvWriter = new CSVWriter();
JsonFlattener jsonFlattener = new JsonFlattener();
String fileName = "StandardQuery";
csvWriter.writeAsCSV(jsonFlattener.parseJson(jsonRequest), fileName);
} catch (Exception e) {
System.out.println("Exception: " + e);
}
}
CVS Writer:
public class CSVWriter {
public void writeAsCSV(List<Map<String, String>> flatJson, String fileName) throws FileNotFoundException {
Set<String> headers = collectHeaders(flatJson);
String output = StringUtils.join(headers.toArray(), ",") + "\n";
for (Map<String, String> map : flatJson) {
output = output + getCommaSeperatedRow(headers, map) + "\n";
}
writeToFile(output, fileName);
}
private void writeToFile(String output, String fileName) throws FileNotFoundException {
BufferedWriter writer = null;
try {
writer = new BufferedWriter(new FileWriter(fileName));
writer.write(output);
} catch (IOException e) {
e.printStackTrace();
} finally {
close(writer);
}
}
private void close(BufferedWriter writer) {
try {
if (writer != null) {
writer.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
private String getCommaSeperatedRow(Set<String> headers, Map<String, String> map) {
List<String> items = new ArrayList<String>();
for (String header : headers) {
String value = map.get(header) == null ? "" : map.get(header).replace(",", "");
items.add(value);
}
return StringUtils.join(items.toArray(), ",");
}
private Set<String> collectHeaders(List<Map<String, String>> flatJson) {
Set<String> headers = new TreeSet<String>();
for (Map<String, String> map : flatJson) {
headers.addAll(map.keySet());
}
return headers;
}
}
Json Flattener:
public class JsonFlattener {
public Map<String, String> parse(JSONObject jsonObject) {
Map<String, String> flatJson = new HashMap<String, String>();
flatten(jsonObject, flatJson, "");
return flatJson;
}
public List<Map<String, String>> parse(JSONArray jsonArray) {
List<Map<String, String>> flatJson = new ArrayList<Map<String, String>>();
int length = jsonArray.length();
for (int i = 0; i < length; i++) {
JSONObject jsonObject = jsonArray.getJSONObject(i);
Map<String, String> stringMap = parse(jsonObject);
flatJson.add(stringMap);
}
return flatJson;
}
public List<Map<String, String>> parseJson(String json) throws Exception {
List<Map<String, String>> flatJson = null;
try {
JSONObject jsonObject = new JSONObject(json);
flatJson = new ArrayList<Map<String, String>>();
flatJson.add(parse(jsonObject));
} catch (JSONException je) {
flatJson = handleAsArray(json);
}
return flatJson;
}
private List<Map<String, String>> handleAsArray(String json) throws Exception {
List<Map<String, String>> flatJson = null;
try {
JSONArray jsonArray = new JSONArray(json);
flatJson = parse(jsonArray);
} catch (Exception e) {
throw new Exception("Json might be malformed");
}
return flatJson;
}
private void flatten(JSONArray obj, Map<String, String> flatJson, String prefix) {
int length = obj.length();
for (int i = 0; i < length; i++) {
if (obj.get(i).getClass() == JSONArray.class) {
JSONArray jsonArray = (JSONArray) obj.get(i);
if (jsonArray.length() < 1) continue;
flatten(jsonArray, flatJson, prefix + i);
} else if (obj.get(i).getClass() == JSONObject.class) {
JSONObject jsonObject = (JSONObject) obj.get(i);
flatten(jsonObject, flatJson, prefix + (i + 1));
} else {
String value = obj.getString(i);
if (value != null)
flatJson.put(prefix + (i + 1), value);
}
}
}
private void flatten(JSONObject obj, Map<String, String> flatJson, String prefix) {
Iterator iterator = obj.keys();
while (iterator.hasNext()) {
String key = iterator.next().toString();
if (obj.get(key).getClass() == JSONObject.class) {
JSONObject jsonObject = (JSONObject) obj.get(key);
flatten(jsonObject, flatJson, prefix);
} else if (obj.get(key).getClass() == JSONArray.class) {
JSONArray jsonArray = (JSONArray) obj.get(key);
if (jsonArray.length() < 1) continue;
flatten(jsonArray, flatJson, key);
} else {
String value = obj.getString(key);
if (value != null && !value.equals("null"))
flatJson.put(prefix + key, value);
}
}
}
}
Here's the service that I'm calling the controller from. I used this to return a .txt file before so I'm not sure how usable it is, but I think if I stream the file back it will handle it...:
getFile: function(jsonObj, fileName) {
var _defer = $q.defer();
$http.post("/web/csvexport/", jsonObj).success(function(data, status, headers) {
var octetStreamMime = "application/octet-stream";
// Get the headers
headers = headers();
// Get the filename from the x-filename header or default to "download.bin"
//var filename = headers["x-filename"] || "logfile.log";
var filename = fileName;
// Determine the content type from the header or default to "application/octet-stream"
var contentType = headers["content-type"] || octetStreamMime;
if(navigator.msSaveBlob)
{
// Save blob is supported, so get the blob as it's contentType and call save.
var blob = new Blob([data], { type: contentType });
navigator.msSaveBlob(blob, filename);
console.log("SaveBlob Success");
}
else
{
// Get the blob url creator
var urlCreator = window.URL || window.webkitURL || window.mozURL || window.msURL;
if(urlCreator)
{
// Try to use a download link
var link = document.createElement("a");
if("download" in link)
{
// Prepare a blob URL
var blob = new Blob([data], { type: contentType });
var url = urlCreator.createObjectURL(blob);
link.setAttribute("href", url);
// Set the download attribute (Supported in Chrome 14+ / Firefox 20+)
link.setAttribute("download", filename);
// Simulate clicking the download link
var event = document.createEvent('MouseEvents');
event.initMouseEvent('click', true, true, window, 1, 0, 0, 0, 0, false, false, false, false, 0, null);
link.dispatchEvent(event);
console.log("Download link Success");
} else {
// Prepare a blob URL
// Use application/octet-stream when using window.location to force download
var blob = new Blob([data], { type: octetStreamMime });
var url = urlCreator.createObjectURL(blob);
window.location = url;
console.log("window.location Success");
}
} else {
console.log("Not supported");
}
}

Firstly, why don't use CSV mime type instead of html ?
replace
response.setContentType("text/html;charset=UTF-8");
by
response.setContentType("text/csv");
And do you know that Jackson, Java JSON API handle CSV ? see
https://github.com/FasterXML/jackson-dataformat-csv
Finaly, in the controler you need to use the printWriter from the response to write the CSV.
Dont forget, to prefer Stream or BufferedString to handle large file and have better performances.