I am using QuickBooks reports API and I want to convert the JSON reports to pdf and also do some modifications. Most of the reports (like Balance Sheet) have dynamic nested structure, i.e. the depth of nesting is not static. How can I convert the JSON to Java Objects so that I can work on them. Collectively some rows make sections and a section can contain either rows or sub-sections (This nesting can go deeper). The way to differentiate is the presence of type data or section in the rows value.
I was thinking on lines of writing a recursive function, which goes through the rows until it finds a row with type=data. But I am looking for something better.
Link to QuickBooks Documentation.
This is the sample JSON.
{
"Header": {
"ReportName": "BalanceSheet",
"Option": [
{
"Name": "AccountingStandard",
"Value": "GAAP"
},
{
"Name": "NoReportData",
"Value": "false"
}
],
"DateMacro": "this calendar year-to-date",
"ReportBasis": "Accrual",
"StartPeriod": "2016-01-01",
"Currency": "USD",
"EndPeriod": "2016-10-31",
"Time": "2016-10-31T09:42:21-07:00",
"SummarizeColumnsBy": "Total"
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "ASSETS"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Current Assets"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Bank Accounts"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "35",
"value": "Checking"
},
{
"value": "1350.55"
}
],
"type": "Data"
},
{
"ColData": [
{
"id": "36",
"value": "Savings"
},
{
"value": "800.00"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "BankAccounts",
"Summary": {
"ColData": [
{
"value": "Total Bank Accounts"
},
{
"value": "2150.55"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Accounts Receivable"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "84",
"value": "Accounts Receivable (A/R)"
},
{
"value": "6383.12"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "AR",
"Summary": {
"ColData": [
{
"value": "Total Accounts Receivable"
},
{
"value": "6383.12"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Other current assets"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "81",
"value": "Inventory Asset"
},
{
"value": "596.25"
}
],
"type": "Data"
},
{
"ColData": [
{
"id": "4",
"value": "Undeposited Funds"
},
{
"value": "2117.52"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "OtherCurrentAssets",
"Summary": {
"ColData": [
{
"value": "Total Other current assets"
},
{
"value": "2713.77"
}
]
}
}
]
},
"type": "Section",
"group": "CurrentAssets",
"Summary": {
"ColData": [
{
"value": "Total Current Assets"
},
{
"value": "11247.44"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Fixed Assets"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"id": "37",
"value": "Truck"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "38",
"value": "Original Cost"
},
{
"value": "13495.00"
}
],
"type": "Data"
}
]
},
"type": "Section",
"Summary": {
"ColData": [
{
"value": "Total Truck"
},
{
"value": "13495.00"
}
]
}
}
]
},
"type": "Section",
"group": "FixedAssets",
"Summary": {
"ColData": [
{
"value": "Total Fixed Assets"
},
{
"value": "13495.00"
}
]
}
}
]
},
"type": "Section",
"group": "TotalAssets",
"Summary": {
"ColData": [
{
"value": "TOTAL ASSETS"
},
{
"value": "24742.44"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "LIABILITIES AND EQUITY"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Liabilities"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Current Liabilities"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Accounts Payable"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "33",
"value": "Accounts Payable (A/P)"
},
{
"value": "1984.17"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "AP",
"Summary": {
"ColData": [
{
"value": "Total Accounts Payable"
},
{
"value": "1984.17"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Credit Cards"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "41",
"value": "Mastercard"
},
{
"value": "157.72"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "CreditCards",
"Summary": {
"ColData": [
{
"value": "Total Credit Cards"
},
{
"value": "157.72"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Other Current Liabilities"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "89",
"value": "Arizona Dept. of Revenue Payable"
},
{
"value": "4.55"
}
],
"type": "Data"
},
{
"ColData": [
{
"id": "90",
"value": "Board of Equalization Payable"
},
{
"value": "401.98"
}
],
"type": "Data"
},
{
"ColData": [
{
"id": "43",
"value": "Loan Payable"
},
{
"value": "4000.00"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "OtherCurrentLiabilities",
"Summary": {
"ColData": [
{
"value": "Total Other Current Liabilities"
},
{
"value": "4406.53"
}
]
}
}
]
},
"type": "Section",
"group": "CurrentLiabilities",
"Summary": {
"ColData": [
{
"value": "Total Current Liabilities"
},
{
"value": "6548.42"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Long-Term Liabilities"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "44",
"value": "Notes Payable"
},
{
"value": "25000.00"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "LongTermLiabilities",
"Summary": {
"ColData": [
{
"value": "Total Long-Term Liabilities"
},
{
"value": "25000.00"
}
]
}
}
]
},
"type": "Section",
"group": "Liabilities",
"Summary": {
"ColData": [
{
"value": "Total Liabilities"
},
{
"value": "31548.42"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Equity"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "34",
"value": "Opening Balance Equity"
},
{
"value": "-9337.50"
}
],
"type": "Data"
},
{
"ColData": [
{
"id": "2",
"value": "Retained Earnings"
},
{
"value": "91.25"
}
],
"type": "Data"
},
{
"ColData": [
{
"value": "Net Income"
},
{
"value": "2440.27"
}
],
"type": "Data",
"group": "NetIncome"
}
]
},
"type": "Section",
"group": "Equity",
"Summary": {
"ColData": [
{
"value": "Total Equity"
},
{
"value": "-6805.98"
}
]
}
}
]
},
"type": "Section",
"group": "TotalLiabilitiesAndEquity",
"Summary": {
"ColData": [
{
"value": "TOTAL LIABILITIES AND EQUITY"
},
{
"value": "24742.44"
}
]
}
}
]
},
"Columns": {
"Column": [
{
"ColType": "Account",
"ColTitle": "",
"MetaData": [
{
"Name": "ColKey",
"Value": "account"
}
]
},
{
"ColType": "Money",
"ColTitle": "Total",
"MetaData": [
{
"Name": "ColKey",
"Value": "total"
}
]
}
]
}
}
JSON Object can ALWAYS be converted to Map. JSON List can ALWAYS be converted to List. Object in those structures can be among other things Map<String, Object> or List<Object>. So you get your multi-nested structure with no depth limitation. To convert JSON to Java class and Java class to JSON is called deserialization and serialization respectively. Today there are 2 major known libraries that do that (and much more) for you. One is Jackson-JSON (otherwise known as "Fast XML" - go figure...) and the other is GSON - a Google library based on Jackson-JSON but also supports passing binary objects. I personally prefer Jackson-JSON, but that's the matter of personal preference. For GSON library look here. For Jackson look here. For Maven artifacts for Jackson look here. If you choose to work with Jackson, then the main class that you need is ObjectMapper.
Start from looking into methods readValue() and writeValue(). Look for endless examples on the web on how to work with it. This should give you a good start
Use reflection apis in java to create your custom type:
Type myType = new TypeToken<ArrayList<HashMap<String, Object>>() {}.getType();
Then use Gson api to convert the JSON into the Java object:
List<Map<String, Object>> outputList = new Gson().fromJson(inputJSONObject, myType);
And then you can iterate over your list and create some consumers to get the sub-objects out of it:
UnaryOperator<Map<String, Object>> subMap = s -> {
Map<String, Object> map = new HashMap<>();
map.put("Header", s.remove("Header"));
...
...
return map;
};
This is a super nested JSON object. This makes it complicated to do in Java and hard to troubleshoot. Nevertheless it can be done.
You could use BSON to load your code and then parse out the values.
Start by importing your libraries.
//import java.util.ArrayList;
//import org.bson.Document;
Parse your JSON into a BSON Document
Document root = Document.parse("{ \"Header\" : { \"ReportName\" : \"BalanceSheet\", \"Option\" : [{ \"Name\" : \"AccountingStandard\", \"Value\" : \"GAAP\" }, { \"Name\" : \"NoReportData\", \"Value\" : \"false\" }], \"DateMacro\" : \"this calendar year-to-date\", \"ReportBasis\" : \"Accrual\", \"StartPeriod\" : \"2016-01-01\", \"Currency\" : \"USD\", \"EndPeriod\" : \"2016-10-31\", \"Time\" : \"2016-10-31T09:42:21-07:00\", \"SummarizeColumnsBy\" : \"Total\" }, \"Rows\" : { \"Row\" : [{ \"Header\" : { \"ColData\" : [{ \"value\" : \"ASSETS\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"Header\" : { \"ColData\" : [{ \"value\" : \"Current Assets\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"Header\" : { \"ColData\" : [{ \"value\" : \"Bank Accounts\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"ColData\" : [{ \"id\" : \"35\", \"value\" : \"Checking\" }, { \"value\" : \"1350.55\" }], \"type\" : \"Data\" }, { \"ColData\" : [{ \"id\" : \"36\", \"value\" : \"Savings\" }, { \"value\" : \"800.00\" }], \"type\" : \"Data\" }] }, \"type\" : \"Section\", \"group\" : \"BankAccounts\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"Total Bank Accounts\" }, { \"value\" : \"2150.55\" }] } }, { \"Header\" : { \"ColData\" : [{ \"value\" : \"Accounts Receivable\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"ColData\" : [{ \"id\" : \"84\", \"value\" : \"Accounts Receivable (A/R)\" }, { \"value\" : \"6383.12\" }], \"type\" : \"Data\" }] }, \"type\" : \"Section\", \"group\" : \"AR\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"Total Accounts Receivable\" }, { \"value\" : \"6383.12\" }] } }, { \"Header\" : { \"ColData\" : [{ \"value\" : \"Other current assets\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"ColData\" : [{ \"id\" : \"81\", \"value\" : \"Inventory Asset\" }, { \"value\" : \"596.25\" }], \"type\" : \"Data\" }, { \"ColData\" : [{ \"id\" : \"4\", \"value\" : \"Undeposited Funds\" }, { \"value\" : \"2117.52\" }], \"type\" : \"Data\" }] }, \"type\" : \"Section\", \"group\" : \"OtherCurrentAssets\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"Total Other current assets\" }, { \"value\" : \"2713.77\" }] } }] }, \"type\" : \"Section\", \"group\" : \"CurrentAssets\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"Total Current Assets\" }, { \"value\" : \"11247.44\" }] } }, { \"Header\" : { \"ColData\" : [{ \"value\" : \"Fixed Assets\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"Header\" : { \"ColData\" : [{ \"id\" : \"37\", \"value\" : \"Truck\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"ColData\" : [{ \"id\" : \"38\", \"value\" : \"Original Cost\" }, { \"value\" : \"13495.00\" }], \"type\" : \"Data\" }] }, \"type\" : \"Section\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"Total Truck\" }, { \"value\" : \"13495.00\" }] } }] }, \"type\" : \"Section\", \"group\" : \"FixedAssets\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"Total Fixed Assets\" }, { \"value\" : \"13495.00\" }] } }] }, \"type\" : \"Section\", \"group\" : \"TotalAssets\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"TOTAL ASSETS\" }, { \"value\" : \"24742.44\" }] } }, { \"Header\" : { \"ColData\" : [{ \"value\" : \"LIABILITIES AND EQUITY\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"Header\" : { \"ColData\" : [{ \"value\" : \"Liabilities\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"Header\" : { \"ColData\" : [{ \"value\" : \"Current Liabilities\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"Header\" : { \"ColData\" : [{ \"value\" : \"Accounts Payable\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"ColData\" : [{ \"id\" : \"33\", \"value\" : \"Accounts Payable (A/P)\" }, { \"value\" : \"1984.17\" }], \"type\" : \"Data\" }] }, \"type\" : \"Section\", \"group\" : \"AP\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"Total Accounts Payable\" }, { \"value\" : \"1984.17\" }] } }, { \"Header\" : { \"ColData\" : [{ \"value\" : \"Credit Cards\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"ColData\" : [{ \"id\" : \"41\", \"value\" : \"Mastercard\" }, { \"value\" : \"157.72\" }], \"type\" : \"Data\" }] }, \"type\" : \"Section\", \"group\" : \"CreditCards\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"Total Credit Cards\" }, { \"value\" : \"157.72\" }] } }, { \"Header\" : { \"ColData\" : [{ \"value\" : \"Other Current Liabilities\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"ColData\" : [{ \"id\" : \"89\", \"value\" : \"Arizona Dept. of Revenue Payable\" }, { \"value\" : \"4.55\" }], \"type\" : \"Data\" }, { \"ColData\" : [{ \"id\" : \"90\", \"value\" : \"Board of Equalization Payable\" }, { \"value\" : \"401.98\" }], \"type\" : \"Data\" }, { \"ColData\" : [{ \"id\" : \"43\", \"value\" : \"Loan Payable\" }, { \"value\" : \"4000.00\" }], \"type\" : \"Data\" }] }, \"type\" : \"Section\", \"group\" : \"OtherCurrentLiabilities\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"Total Other Current Liabilities\" }, { \"value\" : \"4406.53\" }] } }] }, \"type\" : \"Section\", \"group\" : \"CurrentLiabilities\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"Total Current Liabilities\" }, { \"value\" : \"6548.42\" }] } }, { \"Header\" : { \"ColData\" : [{ \"value\" : \"Long-Term Liabilities\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"ColData\" : [{ \"id\" : \"44\", \"value\" : \"Notes Payable\" }, { \"value\" : \"25000.00\" }], \"type\" : \"Data\" }] }, \"type\" : \"Section\", \"group\" : \"LongTermLiabilities\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"Total Long-Term Liabilities\" }, { \"value\" : \"25000.00\" }] } }] }, \"type\" : \"Section\", \"group\" : \"Liabilities\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"Total Liabilities\" }, { \"value\" : \"31548.42\" }] } }, { \"Header\" : { \"ColData\" : [{ \"value\" : \"Equity\" }, { \"value\" : \"\" }] }, \"Rows\" : { \"Row\" : [{ \"ColData\" : [{ \"id\" : \"34\", \"value\" : \"Opening Balance Equity\" }, { \"value\" : \"-9337.50\" }], \"type\" : \"Data\" }, { \"ColData\" : [{ \"id\" : \"2\", \"value\" : \"Retained Earnings\" }, { \"value\" : \"91.25\" }], \"type\" : \"Data\" }, { \"ColData\" : [{ \"value\" : \"Net Income\" }, { \"value\" : \"2440.27\" }], \"type\" : \"Data\", \"group\" : \"NetIncome\" }] }, \"type\" : \"Section\", \"group\" : \"Equity\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"Total Equity\" }, { \"value\" : \"-6805.98\" }] } }] }, \"type\" : \"Section\", \"group\" : \"TotalLiabilitiesAndEquity\", \"Summary\" : { \"ColData\" : [{ \"value\" : \"TOTAL LIABILITIES AND EQUITY\" }, { \"value\" : \"24742.44\" }] } }] }, \"Columns\" : { \"Column\" : [{ \"ColType\" : \"Account\", \"ColTitle\" : \"\", \"MetaData\" : [{ \"Name\" : \"ColKey\", \"Value\" : \"account\" }] }, { \"ColType\" : \"Money\", \"ColTitle\" : \"Total\", \"MetaData\" : [{ \"Name\" : \"ColKey\", \"Value\" : \"total\" }] }] } }");
You can then call out the values using the following syntax. For the arrays you can iterate through the number of values. In order for this to work you need to know what the values are being returned as (I.E. Int, String, or Document) . BSON returns the values as an "Object" so you will have to cast the value. For example: (String) bson.get("field"). This JSON is super large. Here are some examples of how to parse in a single line.
System.out.println(((String)((Document)root.get("Header")).get("ReportName")));
System.out.println(((String)((Document)((ArrayList)((Document)root.get("Header")).get("Option")).get(0)).get("Name")));
System.out.println(((String)((Document)((ArrayList)((Document)root.get("Header")).get("Option")).get(0)).get("Value")));
System.out.println(((String)((Document)root.get("Header")).get("DateMacro")));
System.out.println(((String)((Document)root.get("Header")).get("ReportBasis")));
System.out.println(((String)((Document)root.get("Header")).get("StartPeriod")));
System.out.println(((String)((Document)root.get("Header")).get("Currency")));
System.out.println(((String)((Document)root.get("Header")).get("EndPeriod")));
System.out.println(((String)((Document)root.get("Header")).get("Time")));
System.out.println(((String)((Document)root.get("Header")).get("SummarizeColumnsBy")));
System.out.println(((String)((Document)((ArrayList)((Document)((Document)((ArrayList)((Document)root.get("Rows")).get("Row")).get(0)).get("Header")).get("ColData")).get(0)).get("value")));
System.out.println(((String)((Document)((ArrayList)((Document)((Document)((ArrayList)((Document)((Document)((ArrayList)((Document)root.get("Rows")).get("Row")).get(0)).get("Rows")).get("Row")).get(0)).get("Header")).get("ColData")).get(0)).get("value")));
System.out.println(((String)((Document)((ArrayList)((Document)((Document)((ArrayList)((Document)((Document)((ArrayList)((Document)((Document)((ArrayList)((Document)root.get("Rows")).get("Row")).get(0)).get("Rows")).get("Row")).get(0)).get("Rows")).get("Row")).get(0)).get("Header")).get("ColData")).get(0)).get("value")));
System.out.println(((String)((Document)((ArrayList)((Document)((ArrayList)((Document)((Document)((ArrayList)((Document)((Document)((ArrayList)((Document)((Document)((ArrayList)((Document)root.get("Rows")).get("Row")).get(0)).get("Rows")).get("Row")).get(0)).get("Rows")).get("Row")).get(0)).get("Rows")).get("Row")).get(0)).get("ColData")).get(0)).get("id")));
System.out.println(((String)((Document)((ArrayList)((Document)((ArrayList)((Document)((Document)((ArrayList)((Document)((Document)((ArrayList)((Document)((Document)((ArrayList)((Document)root.get("Rows")).get("Row")).get(0)).get("Rows")).get("Row")).get(0)).get("Rows")).get("Row")).get(0)).get("Rows")).get("Row")).get(0)).get("ColData")).get(0)).get("value")));
System.out.println(((String)((Document)((ArrayList)((Document)((ArrayList)((Document)root.get("Columns")).get("Column")).get(1)).get("MetaData")).get(0)).get("Name")));
System.out.println(((String)((Document)((ArrayList)((Document)((ArrayList)((Document)root.get("Columns")).get("Column")).get(1)).get("MetaData")).get(0)).get("Value")));
Related
I've a document of type:
{...
"array1": [
{
"id": "id1",
"value": "v1"
},
{
"id": "id2",
"value": "v2"
}
],
"array2": [
{
"id": "id-1",
"value": "value1"
},
{
"id": "id-2",
"value": "value2"
}
]
}
to get only the value that matches my conditions from the array1 and array2 in js we can do this,
db.mydb.find({...},{ _id: 0, "array1" : {$elemMatch: {"id" :"id1"}},"array2" : {$elemMatch: {"id" :"id-2"}}})
but I wonder how to do the same thing in Spring boot
my expected answer :
{
"array1": [
{
"id": "id1",
"value": "v1"
}
],
"array2": [
{
"id": "id-2",
"value": "value2"
}
]
}
db={
"dashboard": [
{
"_id": "dashboard1",
"name": "test",
"user": 1
}
],
"templatefolders": [
{
"dashboardId": "dashboard1",
"folderId": "folder123",
"name": "folder",
"region": "XXX"
}
],
"folders": [
{
"_id": "folder123"
}
],
"user": [
{
"_id": 1,
"name": "alaa"
}
],
}
this is my function:
db.dashboard.aggregate([
{
"$lookup": {
"from": "templatefolders",
"localField": "_id",
"foreignField": "dashboardId",
"as": "joinDashboard"
}
},
{
"$lookup": {
"from": "folders",
"localField": "joinDashboard.folderId",
"foreignField": "_id",
"as": "joinDashboard.joinFolder"
}
},
])
Result :
[
{
"_id": "dashboard1",
"joinDashboard": {
"joinFolder": [
{
"_id": "folder123"
}
]
},
"name": "test",
"user": 1
}
]
[![enter image description here][1]][1]
Why the fields name and region in collection templatefolders are excluded ?
I want to know why this behavior ? I don't like to use unwind because i have multiple collections with multiple refrence relation.
Your second $lookup, is overriding the joinDashboard key completely. Since you want joinFolder to be within joinDashboard, you can try nested lookups like this:
db.dashboard.aggregate([
{
$lookup: {
from: "templatefolders",
let: {
"boardId": "$_id"
},
pipeline: [
{
$match: {
$expr: {
$eq: [
"$dashboardId",
"$$boardId"
]
}
}
},
{
$lookup: {
from: "folders",
let: {
"folderId": "$folderId"
},
pipeline: [
{
$match: {
$expr: {
$eq: [
"$_id",
"$$folderId"
]
}
}
},
],
as: "joinFolder"
},
},
],
as: "joinDashboard"
}
}
])
MongoPlayground link.
im using Kibana and ES, i have an index with objects and an array called "reviews" that inside has properties called positive_comment, negative_comment, reviewer_name and more.
There are some reviews that have the field positive_comment empty (not null, just empty).
I need to order by the length of the field positive_comment first, so reviews with empty positive_comment comes last. The results are the same when ordering in SQL with LEN() property.
This is my query.
I also tried:
.value.size() in the script, or type "string" but no results.
{
"_source":[
"reviews.positive_comment"
],
"query":{
"bool":{
"filter":[
{
"term":{
"id":214
}
}
]
}
},
"sort":{
"_script":{
"script":"doc['reviews.positive_comment'].value.length()",
"type":"number",
"order":"asc"
}
}
}
This is my result:
{
"_source":{
"reviews":[
{
"positive_comment":"Great"
},
{
"positive_comment":"Really good product"
},
{
"positive_comment":""
},
{
"positive_comment":""
},
{
"positive_comment":""
},
{
"positive_comment":""
},
{
"positive_comment":""
},
{
"positive_comment":""
},
{
"positive_comment":""
},
{
"positive_comment":""
},
{
"positive_comment":""
},
{
"positive_comment":""
},
{
"positive_comment":""
},
{
"positive_comment":""
},
{
"positive_comment":""
},
{
"positive_comment":"Awesome"
}
]
},
"sort":[
"0"
]
}
Elasticsearch doesn't support counting array elements, unless you use a script.
However, running a script for every matching document is going to degrade performance for every search query.
A better solution would be to count the values once, at index-time, and store the counts in dedicated fields (positive_comments_count, negative_comments_count, etc.) and use these fields for sorting.
Try following. It worked for me on ES 5.6.3. So, should work on higher versions too.
GET test/test/_search?filter_path=hits.hits
{
"query": {
"match_all": {}
},
"sort": {
"_script": {
"type": "number",
"script": {
"lang": "painless",
"source": "doc['reviews.positive_comment'].value.length()"
},
"order": "asc"
}
}
}
I have tested on ES 7.1 version too.
Mapping
PUT test
{
"settings": {
"number_of_shards": 1,
"number_of_replicas": 1
},
"mappings": {
"properties": {
"somefield": {
"type": "keyword"
},
"reviews": {
"properties": {
"positive_comment": {
"type": "keyword"
},
"item_id": {
"type": "double"
}
}
}
}
}
}
Query:
GET test/_search
{
"query": {
"match_all": {}
},
"_source": "reviews.positive_comment",
"sort": {
"_script": {
"type": "number",
"script": {
"lang": "painless",
"source": "doc['reviews.positive_comment'].value.length() % 100"
},
"order": "asc"
}
}
}
output
{
"hits" : {
"total" : {
"value" : 5,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "test",
"_type" : "_doc",
"_id" : "test5",
"_score" : null,
"_source" : {
"reviews" : [
{
"positive_comment" : ""
}
]
},
"sort" : [
0.0
]
},
{
"_index" : "test",
"_type" : "_doc",
"_id" : "test1",
"_score" : null,
"_source" : {
"reviews" : [
{
"positive_comment" : "Awesome"
}
]
},
"sort" : [
7.0
]
},
{
"_index" : "test",
"_type" : "_doc",
"_id" : "test3",
"_score" : null,
"_source" : {
"reviews" : [
{
"positive_comment" : "What a product"
}
]
},
"sort" : [
14.0
]
},
{
"_index" : "test",
"_type" : "_doc",
"_id" : "test2",
"_score" : null,
"_source" : {
"reviews" : [
{
"positive_comment" : "What a product.. amazing"
}
]
},
"sort" : [
24.0
]
},
{
"_index" : "test",
"_type" : "_doc",
"_id" : "test4",
"_score" : null,
"_source" : {
"reviews" : [
{
"positive_comment" : "Thats a great product.. "
}
]
},
"sort" : [
24.0
]
}
]
}
}
Am search my input keyword in multiple field using java api QueryBuilders.multiMatchQuery its not giving me exact results. Its giving some random results which is not relavant to my search query.
Please find my code below.
SearchRequest contentSearchRequest = new SearchRequest(ATTACHMENT);
SearchSourceBuilder contentSearchSourceBuilder = new SearchSourceBuilder();
contentSearchRequest.types(TYPE);
MultiMatchQueryBuilder attachmentQB = QueryBuilders.multiMatchQuery(keyword, "attachment.content","catalog_keywords","product_keywords");
contentSearchSourceBuilder.query(attachmentQB);
contentSearchSourceBuilder.size(5);
Please find my mapping details :
PUT document_attachment
{
"settings": {
"analysis": {
"analyzer": {
"custom_analyzer": {
"type": "custom",
"tokenizer": "whitespace",
"char_filter": [
"html_strip"
],
"filter": [
"lowercase",
"asciifolding"
]
},
"product_catalog_keywords_analyzer": {
"type": "custom",
"tokenizer": "whitespace",
"char_filter": [
"html_strip"
],
"filter": [
"lowercase",
"asciifolding"
]
}
}
}
},
"mappings" : {
"doc" : {
"properties" : {
"attachment" : {
"properties" : {
"content" : {
"type" : "text",
"analyzer": "custom_analyzer"
},
"content_length" : {
"type" : "long"
},
"content_type" : {
"type" : "text"
},
"language" : {
"type" : "text"
}
}
},
"fileContent" : {
"type" : "text"
},
"id": {
"type": "long"
},
"catalog_keywords" : {
"type" : "text",
"analyzer": "product_catalog_keywords_analyzer"
},
"product_keywords" : {
"type" : "text",
"analyzer": "product_catalog_keywords_analyzer"
},
}
}
}
}
Am trying for field mapping along with my custom_analyzer & tokenizer but am getting some error.
Please find the error below that am getting from kibana while mapping fields
Custom Analyzer [custom_analyzer] failed to find tokenizer under name [my_tokenizer]
Please find my mapping details.
PUT attach_local
{
"settings": {
"analysis": {
"analyzer": {
"custom_analyzer": {
"type": "custom",
"tokenizer": "my_tokenizer",
"char_filter": [
"html_strip"
],
"filter": [
"lowercase",
"asciifolding"
]
}
}
}
},
"tokenizer": {
"my_tokenizer": {
"type": "ngram",
"min_gram": 3,
"max_gram": 3,
"token_chars": [
"letter",
"digit"
]
}
},
"mappings" : {
"doc" : {
"properties" : {
"attachment" : {
"properties" : {
"content" : {
"type" : "text",
"analyzer": "custom_analyzer"
},
"content_length" : {
"type" : "long"
},
"content_type" : {
"type" : "text"
},
"language" : {
"type" : "text"
}
}
},
"resume" : {
"type" : "text"
}
}
}
}
}
It is very important to properly indent your JSON. You'd see that your tokenizer is not properly located inside the analysis section. Here is the right definition:
{
"settings": {
"analysis": {
"analyzer": {
"custom_analyzer": {
"type": "custom",
"tokenizer": "my_tokenizer",
"char_filter": [
"html_strip"
],
"filter": [
"lowercase",
"asciifolding"
]
}
},
"tokenizer": {
"my_tokenizer": {
"type": "ngram",
"min_gram": 3,
"max_gram": 3,
"token_chars": [
"letter",
"digit"
]
}
}
}
},
"mappings": {
"doc": {
"properties": {
"attachment": {
"properties": {
"content": {
"type": "text",
"analyzer": "custom_analyzer"
},
"content_length": {
"type": "long"
},
"content_type": {
"type": "text"
},
"language": {
"type": "text"
}
}
},
"resume": {
"type": "text"
}
}
}
}
}