I found article below to do in python.
https://docs.aws.amazon.com/textract/latest/dg/examples-export-table-csv.html
also I used article below to extract text.
https://docs.aws.amazon.com/textract/latest/dg/detecting-document-text.html
but above article helped to get only text, I also used function "block.getBlockType()"
of Block but none of block returned its type as "CELL" even tables are there in image/pdf.
Help me found java library similar to "boto3" to extract all tables.
What I did, I created models of each dataset in the json response and can use this models to build a table view in jsf.
public static List<TableModel> getTablesFromTextract(TextractModel textractModel) {
List<TableModel> tables = null;
try {
if (textractModel != null) {
tables = new ArrayList<>();
List<BlockModel> tableBlocks = new ArrayList<>();
Map<String, BlockModel> blockMap = new HashMap<>();
for (BlockModel block : textractModel.getBlocks()) {
if (block.getBlockType().equals("TABLE")) {
tableBlocks.add(block);
}
blockMap.put(block.getId(), block);
}
for (BlockModel blockModel : tableBlocks) {
Map<Long, Map<Long, String>> rowMap = new HashMap<>();
for (RelationshipModel relationship : blockModel.getRelationships()) {
if (relationship.getType().equals("CHILD")) {
for (String id : relationship.getIds()) {
BlockModel cell = blockMap.get(id);
if (cell.getBlockType().equals("CELL")) {
long rowIndex = cell.getRowIndex();
long columnIndex = cell.getColumnIndex();
if (!rowMap.containsKey(rowIndex)) {
rowMap.put(rowIndex, new HashMap<>());
}
Map<Long, String> columnMap = rowMap.get(rowIndex);
columnMap.put(columnIndex, getCellText(cell, blockMap));
}
}
}
}
tables.add(new TableModel(blockModel, rowMap));
}
System.out.println("row Map " + tables.toString());
}
} catch (Exception e) {
LOG.error("Could not get table from textract model", e);
}
return tables;
}
private static String getCellText(BlockModel cell, Map<String, BlockModel> blockMap) {
String text = "";
try {
if (cell != null
&& CollectionUtils.isNotEmpty(cell.getRelationships())) {
for (RelationshipModel relationship : cell.getRelationships()) {
if (relationship.getType().equals("CHILD")) {
for (String id : relationship.getIds()) {
BlockModel word = blockMap.get(id);
if (word.getBlockType().equals("WORD")) {
text += word.getText() + " ";
} else if (word.getBlockType().equals("SELECTION_ELEMENT")) {
if (word.getSelectionStatus().equals("SELECTED")) {
text += "X ";
}
}
}
}
}
}
} catch (Exception e) {
LOG.error("Could not get cell text of table", e);
}
return text;
}
TableModel to create the view from:
public class TableModel {
private BlockModel table;
private Map<Long, Map<Long, String>> rowMap;
public TableModel(BlockModel table, Map<Long, Map<Long, String>> rowMap) {
this.table = table;
this.rowMap = rowMap;
}
public BlockModel getTable() {
return table;
}
public void setTable(BlockModel table) {
this.table = table;
}
public Map<Long, Map<Long, String>> getRowMap() {
return rowMap;
}
public void setRowMap(Map<Long, Map<Long, String>> rowMap) {
this.rowMap = rowMap;
}
#Override
public String toString() {
return table.getId() + " - " + rowMap.toString();
}
I have something similar:
public class AnalyzeDocument {
public DocumentModel startProcess(byte[] content) {
Region region = Region.EU_WEST_2;
TextractClient textractClient = TextractClient.builder().region(region)
.credentialsProvider(EnvironmentVariableCredentialsProvider.create()).build();
return analyzeDoc(textractClient, content);
}
public DocumentModel analyzeDoc(TextractClient textractClient, byte[] content) {
try {
SdkBytes sourceBytes = SdkBytes.fromByteArray(content);
Util util = new Util();
Document myDoc = Document.builder().bytes(sourceBytes).build();
List<FeatureType> featureTypes = new ArrayList<FeatureType>();
featureTypes.add(FeatureType.FORMS);
featureTypes.add(FeatureType.TABLES);
AnalyzeDocumentRequest analyzeDocumentRequest = AnalyzeDocumentRequest.builder().featureTypes(featureTypes)
.document(myDoc).build();
AnalyzeDocumentResponse analyzeDocument = textractClient.analyzeDocument(analyzeDocumentRequest);
List<Block> docInfo = analyzeDocument.blocks();
// util.displayBlockInfo(docInfo);
PageModel pageModel = util.getTableResults(docInfo);
DocumentModel documentModel = new DocumentModel();
documentModel.getPages().add(pageModel);
Iterator<Block> blockIterator = docInfo.iterator();
while (blockIterator.hasNext()) {
Block block = blockIterator.next();
log.debug("The block type is " + block.blockType().toString());
}
return documentModel;
} catch (TextractException e) {
System.err.println(e.getMessage());
}
return null;
}
and this is the util file:
public PageModel getTableResults(List<Block> blocks) {
List<Block> tableBlocks = new ArrayList<>();
Map<String, Block> blockMap = new HashMap<>();
for (Block block : blocks) {
blockMap.put(block.id(), block);
if (block.blockType().equals(BlockType.TABLE)) {
tableBlocks.add(block);
log.debug("added table: " + block.text());
}
}
PageModel page = new PageModel();
if (tableBlocks.size() == 0) {
return null;
}
int i = 0;
for (Block table : tableBlocks) {
page.getTables().add(generateTable(table, blockMap, i++));
}
return page;
}
private TableModel generateTable(Block table, Map<String, Block> blockMap, int index) {
TableModel model = new TableModel();
Map<Integer, Map<Integer, String>> rows = getRowsColumnsMap(table, blockMap);
model.setTableId("Table_" + index);
for (Map.Entry<Integer, Map<Integer, String>> entry : rows.entrySet()) {
RowModel rowModel = new RowModel();
Map<Integer, String> value = entry.getValue();
for (int i = 0; i < value.size(); i++) {
rowModel.getCells().add(value.get(i));
}
model.getRows().add(rowModel);
}
return model;
}
private Map<Integer, Map<Integer, String>> getRowsColumnsMap(Block block, Map<String, Block> blockMap) {
Map<Integer, Map<Integer, String>> rows = new HashMap<>();
for (Relationship relationship : block.relationships()) {
if (relationship.type().equals(RelationshipType.CHILD)) {
for (String childId : relationship.ids()) {
Block cell = blockMap.get(childId);
if (cell != null) {
int rowIndex = cell.rowIndex();
int colIndex = cell.columnIndex();
if (rows.get(rowIndex) == null) {
Map<Integer, String> row = new HashMap<>();
rows.put(rowIndex, row);
}
rows.get(rowIndex).put(colIndex, getText(cell, blockMap));
}
}
}
}
return rows;
}
public String getText(Block block, Map<String, Block> blockMap) {
String text = "";
if (block.relationships() != null && block.relationships().size() > 0) {
for (Relationship relationship : block.relationships()) {
if (relationship.type().equals(RelationshipType.CHILD)) {
for (String childId : relationship.ids()) {
Block wordBlock = blockMap.get(childId);
if (wordBlock != null && wordBlock.blockType() != null) {
if (wordBlock.blockType().equals(BlockType.WORD))) {
text += wordBlock.text() + " ";
}
}
}
}
}
}
return text;
}
I have a class in Java which has an object
List<GroupNavigationItemSRO> children
Now Every GroupNaviagationItemSRO has the same List
List<GroupNavigationItemSRO> children
NOw i want to iterate through every GroupNavigationSRO and populate a List of String . Currently i am trying to do that like this
void getNavItems(List<GroupNavigationItemSRO> items,List<String> nitems){
System.out.println("PRINT");
for(GroupNavigationItemSRO item : items) {
nitems.add(item.getUrl());
System.out.println(item.getUrl());
// g.add(item.getUrl());
System.out.println("PRINT");
List<GroupNavigationItemSRO> nextItem = item.getChildren();
if (nextItem != null && nextItem.size()>0) {
getNavItems(nextItem,nitems);
}
}
}
When i only print the objects ,it doesn't give any errors But as soon as i try and add to the List the recurssion stops
nitems.add(item.getUrl())
Why is this happening . Here's the entire java file in case its helpful
#Service("labelSearchService")
public class LabelSearchServiceImpl extends AbstractSearchService {
private static final String version = SearchVersion.VERSION_2.getValue();
private static final Logger LOG = LoggerFactory.getLogger(LabelSearchServiceImpl.class);
private static final String EXPIRY_SET = "expiry";
private static final String DATA_SET = "data";
#Autowired
#Qualifier("searchServiceFactory")
private ISearchServiceFactory searchServiceFactory;
#Autowired
IAerospikeTopSellingBrandsCacheService topSellingBrandsCacheService;
#Autowired
private LabelSearchCacheServiceImplFactory labelSearchCacheServiceImplFactory;
#Autowired
AerospikeGuidedResponse aerospikeGuidedResponse ;
List<String> g = null;
#Override
public SearchSRO getSolrResponse(KeyGenerator keyGenerator, String queryFromBrowser, String searchTerm, Integer productCategoryId, int start, int number, String sortBy,
String userZone, String vertical, String clickSrc, boolean isSpellCheckEnabled, String categoryURL, boolean isNested) throws SearchException, ShardNotFoundException, IllegalAccessException {
String originalKeyword = searchTerm;
searchTerm = SearchUtils.modifySearchTerm(searchTerm);
SearchSRO sro = new SearchSRO();
boolean isPartialSearch = SearchUtils.isPartialSearchEnabled();
keyGenerator.setPartialSearch(SearchUtils.isPartialSearchEnabled());
LabelNodeSRO labelNode = SearchUtils.getLabelNodeByNodePath(categoryURL);
// for 'ALL' categories labelNode would be null.
LOG.info("categoryURL : " + categoryURL);
if (ALL.equals(categoryURL)) {
if (number == 0 && !CacheManager.getInstance().getCache(SearchConfigurationCache.class).getBooleanProperty(SearchProperty.ALLOW_ZERO_RESULT_REQUESTS)) {
return new SearchSRO();
}
sro = labelSearchCacheServiceImplFactory.getSearchCacheServiceImpl().getSearchBinResultsForAllLabels(keyGenerator, queryFromBrowser, searchTerm, labelNode, start, number, sortBy, userZone, vertical,
isPartialSearch, isSpellCheckEnabled, originalKeyword, false, isNested);
} else if (labelNode != null) {
sro = getSearchProducts(keyGenerator, queryFromBrowser, searchTerm, null, null, labelNode, start, number, sortBy, userZone, isPartialSearch, isSpellCheckEnabled,
originalKeyword, isNested, false,categoryURL);
} else {
throw new SearchException("Search was hit without selecting any category");
}
// this is the minimum number to results that should match for results to be shown on 'people who search this bought this widget'
SearchConfigurationCache cache = CacheManager.getInstance().getCache(SearchConfigurationCache.class);
if ((ClickSourceType.PWSTBT_WIDGET.getValue()).equalsIgnoreCase(clickSrc) && sro.getNoOfMatches() < cache.getIntegerProperty(SearchProperty.BEST_SELLER_MINIMUM_RESULTS)) {
LOG.info("The minimum number of results to match for PWSTBT widget are " + cache.getIntegerProperty(SearchProperty.BEST_SELLER_MINIMUM_RESULTS)
+ " but number of matched results are " + sro.getNoOfMatches());
sro = new SearchSRO();
}
return sro;
}
#Override
public SearchSRO getSolrResponseForMobile(KeyGenerator keyGenerator, String queryFromBrowser, String searchTerm, Integer productCategoryId, int start, int number,
String sortBy, String userZone, String vertical, String clickSrc, boolean isBinSearch, int noOfResultsPerBin, boolean isSpellCheckEnabled, boolean isPartialSearch,
String categoryURL) throws SearchException, ShardNotFoundException, IllegalAccessException {
String originalKeyword = searchTerm;
searchTerm = SearchUtils.modifySearchTerm(searchTerm);
SearchSRO sro = new SearchSRO();
isPartialSearch = isPartialSearch && SearchUtils.isPartialSearchEnabled();
// this is to disable partial search in case of PWSTBT
if (ClickSourceType.PWSTBT_WIDGET.getValue().equalsIgnoreCase(clickSrc)) {
isPartialSearch = false;
}
LabelNodeSRO labelNode = SearchUtils.getLabelNodeByNodePath(categoryURL);
// for 'ALL' categories labelNode would be null
if (ALL.equals(categoryURL)) {
if (number == 0 && !CacheManager.getInstance().getCache(SearchConfigurationCache.class).getBooleanProperty(SearchProperty.ALLOW_ZERO_RESULT_REQUESTS)) {
return new SearchSRO();
}
// Response for Search result page in mobile - same as web.
sro = labelSearchCacheServiceImplFactory.getSearchCacheServiceImpl().getSearchBinResultsForAllLabels(keyGenerator, queryFromBrowser, searchTerm, labelNode, start, number, sortBy, userZone, vertical,
isPartialSearch, isSpellCheckEnabled, originalKeyword, true, false);
} else if (labelNode != null) {
sro = getSearchProducts(keyGenerator, queryFromBrowser, searchTerm, null, null, labelNode, start, number, sortBy, userZone, isPartialSearch, isSpellCheckEnabled,
originalKeyword, false, true,categoryURL);
} else {
throw new SearchException("Search was hit without selecting any category");
}
// this is the minimum number to results that should match for results to be shown on 'people who search this bought this widget'
SearchConfigurationCache cache = CacheManager.getInstance().getCache(SearchConfigurationCache.class);
if ((ClickSourceType.PWSTBT_WIDGET.getValue()).equalsIgnoreCase(clickSrc) && sro.getNoOfMatches() < cache.getIntegerProperty(SearchProperty.BEST_SELLER_MINIMUM_RESULTS)) {
LOG.info("The minimum number of results to match for PWSTBT widget are " + cache.getIntegerProperty(SearchProperty.BEST_SELLER_MINIMUM_RESULTS)
+ " but number of matched results are " + sro.getNoOfMatches());
sro = new SearchSRO();
}
return sro;
}
#Autowired
private IUserPersonaSegmentService personaSegmentService;
#Autowired
private IContextHolder<SearchRequestContext> ctxProvider;
private boolean isClientPersonaEnabled(SearchRequestContext ctx) {
SearchConfigurationCache cache = CacheManager.getInstance().getCache(SearchConfigurationCache.class);
String clientsEnabled = cache.getProperty(SearchProperty.PERSONA_CLIENTS_ENABLED);
String client = ctx.req.getContextSRO().getAppIdent();
return !StringUtils.isEmpty(client) && !StringUtils.isEmpty(clientsEnabled) && Pattern.matches("(?i).*\\b" + client + "\\b.*", clientsEnabled);
}
protected UserSegmentDTO setupPersonalizationContext(LabelNodeSRO labelNode, String searchTerm, String sortBy) {
SearchRequestContext ctx = ctxProvider.getContext();
if (ctx == null || ctx.req == null) {
LOG.warn("No Request Context found");
return null;
}
SearchConfigurationCache cache = CacheManager.getInstance().getCache(SearchConfigurationCache.class);
// check if Personalization is enabled
if (labelNode == null || !cache.getBooleanProperty(SearchProperty.PERSONA_SEARCH_ENABLED) || StringUtils.isEmpty(searchTerm)
|| !SolrSortCategory.RELEVANCY.getValue().equalsIgnoreCase(sortBy) || !isClientPersonaEnabled(ctx)) {
LOG.debug("Personalization not enabled");
return null;
}
LOG.info("Trying to set up personalization context");
// setup the context for later use
ctx.personaSegments = personaSegmentService.getUserSegments(ctx.req.getUserTrackingId(), labelNode.getNodePath());
return ctx.personaSegments;
}
#Override
public SearchSRO getSearchProducts(KeyGenerator keyGenerator, String queryFromBrowser, String searchTerm, Integer campaignId, ProductCategorySRO pc, LabelNodeSRO labelNode,
int start, int number, String sortBy, String userZone, boolean isPartialSearch, boolean isSpellCheckEnabled, String originalKeyword, boolean isNested, boolean isMobile,String categoryURL)
throws SearchException, ShardNotFoundException, IllegalAccessException {
LOG.info("------------------Product category page---------------");
// build cache key considering campaign id
keyGenerator.setCampaignId(String.valueOf(campaignId));
// Search results will vary based on isNested flag even for exact same keywords hence, when we cache
// we cache both results with different key
keyGenerator.setNested(isNested);
SearchConfigurationCache cache = CacheManager.getInstance().getCache(SearchConfigurationCache.class);
LOG.info("sortBy : " + sortBy + ", personalization Enabled : " + cache.getBooleanProperty(SearchProperty.PERSONA_SEARCH_ENABLED) + ", labelNode : " + labelNode);
// try to set persona context
keyGenerator.setPersonaSegment(setupPersonalizationContext(labelNode, searchTerm, sortBy));
SearchSRO searchSRO = labelSearchCacheServiceImplFactory.getSearchCacheServiceImpl().getSearchProducts(keyGenerator, queryFromBrowser, searchTerm, campaignId, pc, labelNode, start, number, sortBy, userZone, isPartialSearch,
isSpellCheckEnabled, originalKeyword, isNested, isMobile,categoryURL);
/*SearchCoreContext coreContext = CoreContextHolderThreadLocal.getContext();
if (coreContext != null) {
if (coreContext.getCategoryUrlUsed().equalsIgnoreCase("ALL")) {
String cacheKey = keyGenerator.buildKey();
try {
final AerospikeClient aClient = AerospikeClientFactory.getInstance();
LOG.info("Clearing Cache as Category redirected was ambiguous so redirected to ALL and removing key " + cacheKey);
aClient.delete(null, new Key("search", EXPIRY_SET, cacheKey));
aClient.delete(null, new Key("search", DATA_SET, cacheKey));
} catch (AerospikeException e) {
e.printStackTrace();
}
}
}*/
return searchSRO;
}
#Override
public FilterListSRO getFiltersForProducts(Integer categoryId, Integer campaignId, String q, String keyword, boolean partialSearch, boolean isBrand, String categoryUrl,
String userZone, HyperlocalCriteria hyperlocalCriteria, Set<Integer> pinCodes, GetFiltersRequest request) throws SearchException, ShardNotFoundException {
String key = new KeyGenerator(String.valueOf(categoryId), String.valueOf(campaignId), q, keyword, partialSearch, null, categoryUrl, version, userZone, hyperlocalCriteria, pinCodes).buildFilterKey();
if (campaignId != null) {
LOG.info("Get Filters for Campaign Products wrt : " + key);
}
FilterListSRO filterListSRO = labelSearchCacheServiceImplFactory.getSearchCacheServiceImpl().getFiltersForProducts(key, categoryId, campaignId, q, keyword, partialSearch, isBrand, categoryUrl, userZone, hyperlocalCriteria, pinCodes,request);
return filterListSRO;
}
#Override
public FilterListSRO getFiltersForProducts(Integer categoryId, Integer campaignId, String q, String keyword,
boolean partialSearch, boolean isBrand, String categoryUrl, String userZone,
HyperlocalCriteria hyperlocalCriteria, Set<Integer> pinCodes)
throws SearchException, ShardNotFoundException {
return getFiltersForProducts(categoryId, campaignId, q, keyword, partialSearch, isBrand, categoryUrl, userZone, hyperlocalCriteria, pinCodes,null);
}
#Override
public FilterListSRO getFilterValuesForFilter(String categoryId, String campaignId, String q, String keyword, boolean partialSearch, String filterName, String fullQ,
String categoryUrl, String[] filtersToFetch, String userZone, HyperlocalCriteria hyperlocalCriteria, Set<Integer> pinCodes) throws SearchException, NumberFormatException, ShardNotFoundException {
String uniqueFilterKey = new KeyGenerator(categoryId, campaignId, q, keyword, partialSearch, filterName, categoryUrl, version, userZone, hyperlocalCriteria, pinCodes).buildFilterKey();
String[] filterNames = filterName.split(",");
FilterListSRO filterListSRO = labelSearchCacheServiceImplFactory.getSearchCacheServiceImpl().getFilterValuesForFilter(uniqueFilterKey, categoryId, campaignId, q, keyword, partialSearch, filterNames, categoryUrl, filtersToFetch,
userZone, hyperlocalCriteria, pinCodes);
/*SearchCoreContext coreContext = CoreContextHolderThreadLocal.getContext();
if (coreContext != null) {
if (coreContext.getCategoryUrlUsed().equalsIgnoreCase("ALL")) {
String cacheKey = uniqueFilterKey.concat(".FilterSRO");
try {
final AerospikeClient aClient = AerospikeClientFactory.getInstance();
LOG.info("Clearing Cache as Category redirected was ambiguous so redirected to ALL and removing key " + cacheKey);
aClient.delete(null, new Key("search", EXPIRY_SET, cacheKey));
aClient.delete(null, new Key("search", DATA_SET, cacheKey));
} catch (AerospikeException e) {
e.printStackTrace();
}
}
}*/
return filterListSRO;
}
#Override
public GroupNavigationSRO getGroupNavigation(KeyGenerator keyGenerator, String keyword, String q, String categoryUrl, Integer campaignId, boolean isSpellCheck, String userZone) throws IllegalAccessException {
GroupNavigationSRO sro = new GroupNavigationSRO();
try {
ProductCategoryCache categoryCache = CacheManager.getInstance().getCache(ProductCategoryCache.class);
LabelNodeSRO labelNode = ALL.equals(categoryUrl) ? null : categoryCache.getLabelForLabelPath(categoryUrl);
if (!ALL.equals(categoryUrl) && labelNode == null) {
LOG.error("Invalid label : " + categoryUrl);
return null;
}
// try to setup persona context - using sort to relevancy since group left nav doesn't change with sortBy
keyGenerator.setPersonaSegment(setupPersonalizationContext(labelNode, keyword, SolrSortCategory.RELEVANCY.getValue()));
sro = labelSearchCacheServiceImplFactory.getSearchCacheServiceImpl().getGroupNavigation(keyGenerator, keyword, q, categoryUrl, campaignId, isSpellCheck, userZone);
/*SearchCoreContext coreContext = CoreContextHolderThreadLocal.getContext();
if (coreContext != null) {
if (coreContext.getCategoryUrlUsed().equalsIgnoreCase("ALL")) {
String cacheKey = keyGenerator.buildKey().concat(".GroupNavigationSRO");
try {
final AerospikeClient aClient = AerospikeClientFactory.getInstance();
LOG.info("Clearing Cache as Category redirected was ambiguous so redirected to ALL and removing key " + cacheKey);
aClient.delete(null, new Key("search", EXPIRY_SET, cacheKey));
aClient.delete(null, new Key("search", DATA_SET, cacheKey));
} catch (AerospikeException e) {
e.printStackTrace();
}
}
}*/
} catch (SearchException e) {
LOG.error("Error in fetching GroupSRO: ", e);
}
return sro;
}
#Override
public QueryResponse setCategoryFilterQueryAndExecute(SearchCriteria sc, Integer id) throws SearchException {
labelSearchCacheServiceImplFactory.getSearchCacheServiceImpl().setCategoryFilterQuery(sc, id);
return labelSearchCacheServiceImplFactory.getSearchCacheServiceImpl().executeQuery(sc.buildQuery(), id);
}
#Override
public Long getProductCategoryCount(Integer categoryId, String categoryUrl) {
Long count = CacheManager.getInstance().getCache(ProductCategoryCache.class).getCategoryCountByUrl(categoryUrl);
if (count == null) {
count = new Long(0);
}
LOG.info("Product Category Counts for CategoryUrl: " + categoryUrl + " = " + count);
return count;
}
#Override
public List<TopSellingProductCategorySRO> getTopSellingProductsforCategories(List<Integer> categoryIds, List<String> categoryUrls) {
List<TopSellingProductCategorySRO> topSellingProductCategorySROs = new ArrayList<TopSellingProductCategorySRO>();
for (String categoryUrl : categoryUrls) {
try {
TopSellingProductCategorySRO topSellingProductCategorySRO = null;
Integer searchId = ShardResolverService.getSearchIdByLabel(categoryUrl);
QueryResponse rsp = searchServiceFactory.getSearchService(SearchVersion.VERSION_2.getValue()).getTopProductsInfoById(searchId,
CacheManager.getInstance().getCache(SearchConfigurationCache.class).getIntegerProperty(SearchProperty.MAX_TOP_SELLING_PRODUCTS_PER_CATEGORY));
List<Long> pogIds = SearchUtils.extractTopProductsByCategoryId(rsp);
if (pogIds != null && !pogIds.isEmpty()) {
topSellingProductCategorySRO = new TopSellingProductCategorySRO(categoryUrl, pogIds);
topSellingProductCategorySROs.add(topSellingProductCategorySRO);
}
} catch (Exception e) {
LOG.error("Unable to get Top Selling Products for categoryId: " + categoryUrl + ", Exception:" + e.getMessage());
}
}
return topSellingProductCategorySROs;
}
#Override
public List<TopSellingBrandSRO> getTopSellingBrandsforCategories(List<Integer> categoryIds, List<String> categoryUrls) {
List<TopSellingBrandSRO> topSellingBrandSROs = new ArrayList<TopSellingBrandSRO>();
for (String categoryUrl : categoryUrls) {
TopSellingBrandSRO topSellingBrandSRO = topSellingBrandsCacheService.getTopSellingBrandsByUrl(categoryUrl);
if (topSellingBrandSRO != null) {
topSellingBrandSROs.add(topSellingBrandSRO);
}
}
return topSellingBrandSROs;
}
#Override
public List<TopSellingBrandSRO> getAllTopSellingProducts(){
List<TopSellingBrandSRO> topSellingBrandSROs = topSellingBrandsCacheService.getAllTopSellingProducts();
return topSellingBrandSROs;
}
#Override
public FacetSRO getFacets(String cachekey, String keyword, String queryFieldName, String[] facetFields, Map<String, List<String>> filterMap, int number) throws SearchException {
// update values for mainCategoryXpath & categoryXpath fields
/*if(SolrFields.CATEGORY_XPATH.equals(queryFieldName) || SolrFields.MAIN_CATEGORY_XPATH.equals(queryFieldName)) {
String labelPath = SearchUtils.getLabelPathByUrl(keyword);
keyword = String.valueOf(ShardResolverService.getSearchIdByLabel(labelPath));
}*/
for (String filterField : filterMap.keySet()) {
if (SolrFields.CATEGORY_XPATH.equals(filterField) || SolrFields.MAIN_CATEGORY_XPATH.equals(filterField)) {
List<String> searchIds = new ArrayList<String>();
for (String val : filterMap.get(filterField)) {
String labelPath = SearchUtils.getLabelPathByUrl(val);
searchIds.add(String.valueOf(ShardResolverService.getSearchIdByLabel(labelPath)));
}
filterMap.put(filterField, searchIds);
}
}
return labelSearchCacheServiceImplFactory.getSearchCacheServiceImpl().getFacets(cachekey, keyword, queryFieldName, facetFields, filterMap, number);
}
#Override
public FilterListSRO getSRPFilters(KeyGenerator keyGenerator, String q, String keyword, boolean partialSearch, String categoryUrl, String userZone, HyperlocalCriteria hyperlocalCriteria, Set<Integer> pinCodes) throws SearchException {
if (StringUtils.isEmpty(keyword)) {
LOG.error("Invalid parameters.");
return null;
}
keyword = SearchUtils.modifySearchTerm(keyword);
if (StringUtils.isEmpty(keyword)) {
LOG.info(" Returning empty filters for empty keyword.");
return new FilterListSRO();
}
return labelSearchCacheServiceImplFactory.getSearchCacheServiceImpl().getSRPFilters(keyGenerator.buildKey(), q, keyword, partialSearch, categoryUrl, userZone, hyperlocalCriteria, pinCodes);
}
#Override
public SearchSRO getSearchProducts(KeyGenerator keyGenerator, String queryFromBrowser, String searchTerm,
Integer campaignId, ProductCategorySRO pc, LabelNodeSRO labelNode, int start, int number, String sortBy,
String userZone, boolean isPartialSearch, boolean isSpellCheckEnabled, String originalKeyword,
boolean isNested, boolean isMobile) throws SearchException, ShardNotFoundException, IllegalAccessException {
// TODO Auto-generated method stub
return getSearchProducts(keyGenerator, queryFromBrowser, searchTerm, campaignId, pc, labelNode, start, number, sortBy, userZone, isPartialSearch, isSpellCheckEnabled, originalKeyword, isNested, isMobile, null);
}
#Override
public String getmodelSearch(String query, String type) throws SearchException {
return labelSearchCacheServiceImplFactory.getSearchCacheServiceImpl().getmodelSearch(query, type);
}
#Override
public String classifierResponse(String query, String type) throws SearchException {
try {
return labelSearchCacheServiceImplFactory.getSearchCacheServiceImpl().getClassifierResponse(query, type);
} catch (JsonGenerationException e) {
return e.getMessage();
} catch (JsonMappingException e) {
return e.getMessage();
} catch (IOException e) {
return e.getMessage();
}
}
public GetGuidedSearchResponse getGuides(String query, String url) {
if(!StringUtils.isEmpty(query) && !StringUtils.isEmpty(url) && url.equalsIgnoreCase("ALL"))
{
KeyGenerator keyGenerator = new KeyGenerator();
keyGenerator.setQ("sNapDeAl.sEarcH.getGuides=" +"##"+ query+ "##"+ url);
return labelSearchCacheServiceImplFactory.getSearchCacheServiceImpl().getGuides(keyGenerator, query, url);
}
return null;
}
public GetGuidedSearchResponse getFilteredGuides(String query ,GetGroupLeftNavResponse leftNavBarResponse) {
g=null;
GroupNavigationSRO groups = leftNavBarResponse.getGroups();
List<GroupNavigationItemSRO> items = groups.getItems() ;
// List<String> nitems = getNavItems(items);
List<String> nitems = null;
getNavItems(items,nitems);
System.out.println("SIZE" + nitems.size());
List<String> navItems = new ArrayList<String>();
System.out.println("GETTING GUIDED FILE FROM AEROSPIKE");
List<String> guideItems = aerospikeGuidedResponse.getGuides(query);
//HashMap<String,String> nodeUrlMapping = new HashMap<String,String>();
if(guideItems.isEmpty())
{
System.out.println("\n\n\n\n" + "EMPTY GUIDED" + " \n\n\n\n\n");
}
guideItems.set(0, guideItems.get(0).trim());
System.out.println("GUIDED RESPONSE");
for(int i=0 ; i < guideItems.size() ;i ++)
{
System.out.println(guideItems.get(i));
}
/*for (int i =0 ;i < items.size() ;i++) {
List<GroupNavigationItemSRO> children_items = items.get(i).getChildren();
String s = items.get(i).getNodePath();
String m = items.get(i).getUrl();
System.out.println(s + " " + m);
navItems.add(m);
//nodeUrlMapping.put(s,m);
for (int j=0;j<children_items.size();j++) {
String r = children_items.get(j).getNodePath();
String n = children_items.get(j).getUrl();
System.out.println(r +" " + n);
// nodeUrlMapping.put(r,n);
navItems.add(n);
}
}*/
System.out.println("ITEM RESPONSE");
//navItems = g ;
for(int i=0 ; i < navItems.size() ;i ++)
{
System.out.println(navItems.get(i));
}
List<String> filteredGuides = new ArrayList<String>();
for(int i=0 ; i < guideItems.size() ;i++)
{
if(navItems.contains(guideItems.get(i)))
filteredGuides.add(guideItems.get(i));
else {
}
}
System.out.println("NAV ITEMS" + navItems.size() + navItems.toString());
System.out.println("GUIDE ITEMS" + filteredGuides.size() + filteredGuides.toString());
List<WidgetEntity> entities = new ArrayList<WidgetEntity>();
/* Iterator it = nodeUrlMapping.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pair = (Map.Entry)it.next();
System.out.println(pair.getKey() + " = " + pair.getValue());
}*/
for(int i=0;i<filteredGuides.size();i++)
{
String guide = filteredGuides.get(i);
guide = guide.trim();
System.out.println(guide);
LabelNodeSRO labelSRO = getLableSRO(guide);
System.out.println(labelSRO.toString() + guide);
WidgetEntity entity = new WidgetEntity();
entity.setId(labelSRO.getUrl());
entity.setName(labelSRO.getDisplayName());
entity.setType("category");
entities.add(entity);
}
System.out.println("ENTITIES DEtails" );
GetGuidedSearchResponse response = new GetGuidedSearchResponse();
for(int i =0 ;i<entities.size();i++)
{
System.out.println(entities.get(i).getId() + entities.get(i).getName() + entities.get(i).getType());
// System.out.println(nodeUrlMapping.get(entities.get(i).getId()));
}
response.setEntities(entities);
return response;
}
LabelNodeSRO getLableSRO(String guide)
{
System.out.println(guide + "GET");
LabelNodeSRO label =SearchUtils.getLabelNodeByNodePath(guide);
return label;
}
void getNavItems(List<GroupNavigationItemSRO> items,List<String> nitems){
System.out.println("PRINT");
for(GroupNavigationItemSRO item : items) {
nitems.add(item.getUrl());
System.out.println(item.getUrl());
// g.add(item.getUrl());
System.out.println("PRINT");
List<GroupNavigationItemSRO> nextItem = item.getChildren();
if (nextItem != null && nextItem.size()>0) {
getNavItems(nextItem,nitems);
}
}
}
}
You could try something like this, when you return a list with all Strings, and if there are no more elments stop the recursivity and returns an empty list
List<String> getNavItems(List<GroupNavigationItemSRO> items){
List<String> results = new ArrayList();
System.out.println("PRINT");
if(items != null && !items.isEmpty()){
for(GroupNavigationItemSRO item : items) {
results.add(item.getUrl());
System.out.println(item.getUrl());
// g.add(item.getUrl());
System.out.println("PRINT");
results.addAll(getNavItems(item.getChildren()));
}
}
}
return results;
}
In getFilteredGuides() method you're passing nitems as null and this would cause NullPointerException.
Just Pass it as the following:
List<String> nitems = new ArrayList<String>();
getNavItems(items,nitems);
Or you can add a check for null inside getNavItems() method and initialize it accordingly:
void getNavItems(List<GroupNavigationItemSRO> items,List<String> nitems){
if(nitems == null)
{
nitems = new ArrayList<String>();
}
System.out.println("PRINT");
for(GroupNavigationItemSRO item : items) {
nitems.add(item.getUrl());
System.out.println(item.getUrl());
// g.add(item.getUrl());
System.out.println("PRINT");
List<GroupNavigationItemSRO> nextItem = item.getChildren();
if (nextItem != null && nextItem.size()>0) {
getNavItems(nextItem,nitems);
}
}
}
Here's what myBATIS has on their own documentation for foreach.
<select id="selectPostIn" resultType="domain.blog.Post">
SELECT *
FROM POST P
WHERE ID in
<foreach item="item" index="index" collection="list"
open="(" separator="," close=")">
#{item}
</foreach>
</select>
However, if list contains over 1000 items and you're using Oracle DB, you get this exception:
java.sql.SQLSyntaxErrorException: ORA-01795: maximum number of expressions in a list is 1000
What can I do to fix this so it works with more than 1000 elements?
I'm not sure if this is the most elegant solution or not, but here's what I did:
<select id="selectPostIn" resultType="domain.blog.Post">
SELECT *
FROM POST P
WHERE ID in
<trim suffixOverrides=" OR ID IN ()">
<foreach item="item" index="index" collection="list"
open="(" close=")">
<if test="index != 0">
<choose>
<when test="index % 1000 == 999">) OR ID IN (</when>
<otherwise>,</otherwise>
</choose>
</if>
#{item}
</foreach>
</trim>
</select>
Explanation
Lets start with the foreach. We want to surround it in ( and ). Most elements we want commas between, except every thousand elements we want to stop the list and OR with another one. That's what the choose, when, otherwise construct handles. Except we don't want either of those before the first element, thus the if that the choose is inside of. Finally, the foreach ends with actually having the #{item} inserted.
The outer trim is just so that if we have exactly 1000 elements, for example, we don't end with OR ID IN () which would be invalid ((), specifically, is the invalid part. That's a syntax error in SQL, not an empty list like I hoped it would be.)
We have tried delete query in clause more then 1000 records with above reference:
<delete id="delete" parameterType="Map">
The following query working:
DELETE FROM Employee
where
emp_id = #{empId}
<foreach item="deptId" index= "index" collection="ids" open="AND DEPT_ID NOT IN (" close=")" >
<if test="index != 0">
<choose>
<when test="index % 1000 == 999">) AND DEPT_ID NOT IN (</when>
<otherwise>,</otherwise>
</choose>
</if>
#{deptId}
</foreach>
</delete>
Mybatis plugin query and then combine partitioned params :
#Intercepts({
#Signature(type = Executor.class, method = "query", args = {MappedStatement.class, Object.class, RowBounds.class, ResultHandler.class}),
#Signature(type = Executor.class, method = "query", args = {MappedStatement.class, Object.class, RowBounds.class, ResultHandler.class, CacheKey.class, BoundSql.class})}
)
public class BigSizeParamQueryPlugin implements Interceptor {
private final int singleBatchSize;
private static final HeavyParamContext NO_BIG_PARAM = new HeavyParamContext();
public BigSizeParamQueryPlugin() {
this.singleBatchSize = 1000;
}
public BigSizeParamQueryPlugin(Integer singleBatchSize) {
if (singleBatchSize < 500) {
throw new IllegalArgumentException("batch size less than 500 is not recommended");
}
this.singleBatchSize = singleBatchSize;
}
#Override
public Object intercept(Invocation invocation) throws Throwable {
Object[] args = invocation.getArgs();
Object parameter = args[1];
if (parameter instanceof MapperMethod.ParamMap && RowBounds.DEFAULT == args[2]) {
MapperMethod.ParamMap paramMap = (MapperMethod.ParamMap) parameter;
if (MapUtils.isNotEmpty(paramMap)) {
try {
HeavyParamContext context = findHeavyParam(paramMap);
if (context.hasHeavyParam()) {
QueryExecutor queryExecutor = new QueryExecutor(invocation, context);
return queryExecutor.query();
}
} catch (Throwable e) {
log.warn("BigSizeParamQueryPlugin process error", e);
return invocation.proceed();
}
}
}
return invocation.proceed();
}
private class QueryExecutor {
private final MappedStatement ms;
private final Map<String, Object> paramMap;
private final RowBounds rowBounds;
private final ResultHandler resultHandler;
private final Executor executor;
private final List<Object> finalResult;
private final Iterator<HeavyParam> heavyKeyIter;
public QueryExecutor(Invocation invocation, HeavyParamContext context) {
Object[] args = invocation.getArgs();
this.ms = (MappedStatement) args[0];
this.paramMap = context.getParameter();
this.rowBounds = (RowBounds) args[2];
this.resultHandler = (ResultHandler) args[3];
this.executor = (Executor) invocation.getTarget();
List<HeavyParam> heavyParams = context.getHeavyParams();
this.finalResult = new ArrayList<>(heavyParams.size() * singleBatchSize);
this.heavyKeyIter = heavyParams.iterator();
}
public Object query() throws SQLException {
while (heavyKeyIter.hasNext()) {
HeavyParam currKey = heavyKeyIter.next();
List<List<Object>> param = partitionParam(currKey.getParam());
doQuery(currKey, param);
}
return finalResult;
}
private void doQuery(HeavyParam currKey, List<List<Object>> param) throws SQLException {
if (!heavyKeyIter.hasNext()) {
for (List<Object> currentParam : param) {
updateParamMap(currKey, currentParam);
List<Object> oneBatchResult = executor.query(ms, paramMap, rowBounds, resultHandler);
finalResult.addAll(oneBatchResult);
}
return;
} else {
HeavyParam nextKey = heavyKeyIter.next();
log.warn("get mutil heavy key [{}], batchSize[{}]", nextKey.shadowHeavyKeys, nextKey.getParam().size());
List<List<Object>> nextParam = partitionParam(nextKey.getParam());
for (List<Object> currParam : param) {
updateParamMap(currKey, currParam);
doQuery(nextKey, nextParam);
}
}
}
private void updateParamMap(HeavyParam currKey, List<Object> param) {
for (String shadowKey : currKey.getShadowHeavyKeys()) {
paramMap.put(shadowKey, param);
}
}
}
private HeavyParamContext findHeavyParam(Map<String, Object> parameterMap) {
List<Map.Entry<String, Object>> heavyKeys = doFindHeavyParam(parameterMap);
if (heavyKeys == null) {
return BigSizeParamQueryPlugin.NO_BIG_PARAM;
} else {
HeavyParamContext result = new HeavyParamContext();
List<HeavyParam> heavyParams;
if (heavyKeys.size() == 1) {
heavyParams = buildSingleHeavyParam(heavyKeys);
} else {
heavyParams = buildMultiHeavyParam(heavyKeys);
}
result.setHeavyParams(heavyParams);
result.setParameter(new HashMap<>(parameterMap));
return result;
}
}
private List<HeavyParam> buildSingleHeavyParam(List<Map.Entry<String, Object>> heavyKeys) {
Map.Entry<String, Object> single = heavyKeys.get(0);
return Collections.singletonList(new HeavyParam((Collection) single.getValue(), Collections.singletonList(single.getKey())));
}
private List<List<Object>> partitionParam(Object o) {
Collection c = (Collection) o;
List res;
if (c instanceof List) {
res = (List) c.stream().distinct().collect(Collectors.toList());
} else {
res = new ArrayList(c);
}
return Lists.partition(res, singleBatchSize);
}
private List<HeavyParam> buildMultiHeavyParam(List<Map.Entry<String, Object>> heavyKeys) {
//when heavy keys used multi time in xml, its name will be different.
TreeMap<Collection, List<String>> params = new TreeMap<>(new Comparator<Collection>() {
#Override
public int compare(Collection o1, Collection o2) {
//fixme workable but have corner case.
return CollectionUtils.isEqualCollection(o1, o2) == true ? 0 : o1.hashCode() - o2.hashCode();
}
});
for (Map.Entry<String, Object> keyEntry : heavyKeys) {
String key = keyEntry.getKey();
List<String> keys = params.computeIfAbsent((Collection) keyEntry.getValue(), k -> new ArrayList<>(1));
keys.add(key);
}
List<HeavyParam> hps = new ArrayList<>(params.size());
for (Map.Entry<Collection, List<String>> heavyEntry : params.entrySet()) {
List<String> shadowKeys = heavyEntry.getValue();
hps.add(new HeavyParam(heavyEntry.getKey(), shadowKeys));
}
return hps;
}
private List<Map.Entry<String, Object>> doFindHeavyParam(Map<String, Object> parameterMap) {
List<Map.Entry<String, Object>> result = null;
for (Map.Entry<String, Object> p : parameterMap.entrySet()) {
if (p != null) {
Object value = p.getValue();
if (value != null && value instanceof Collection) {
int size = CollectionUtils.size(value);
if (size > singleBatchSize) {
if (result == null) {
result = new ArrayList<>(1);
}
result.add(p);
}
}
}
}
return result;
}
#Getter
#Setter
private static class HeavyParamContext {
private Boolean hasHeavyParam;
private List<HeavyParam> heavyParams;
private Map<String, Object> parameter;
public Boolean hasHeavyParam() {
return heavyParams != null;
}
}
#Data
#AllArgsConstructor
#NoArgsConstructor
private class HeavyParam {
private Collection param;
private List<String> shadowHeavyKeys;
}
#Override
public Object plugin(Object o) {
return Plugin.wrap(o, this);
}
#Override
public void setProperties(Properties properties) {
}
}