We Keep Coding

JOLT shift transformation: filter by inner value of a property (not the name) - only some properties

I am trying to transform a JSON using Jolt transformation looking for some input here.
I am trying to filter by the inner value of the attribute.
My goal is to get an array that contains only the items with the typeName 'xx'. But not all the item object, only some of the fields
{
"id": "11_1",
"action": "add",
"payment": {
"paied": true,
"coin": "dollar"
},
"type": {
"id": "11_1_xx",
"typeName": "xx"
},
"details": {
"place": {
"id": "123",
"name": "xx"
},
"status": {
"id": "123",
"name": "xx"
}
}
}
Here is my input and expected output:
Input:
{
"id": 11,
"item": [
{
"id": "11_1",
"action": "add",
"payment": {
"paied": true,
"coin": "dollar"
},
"type": {
"id": "11_1_xx",
"typeName": "xx",
"typeGroup": "xx",
"typeValue": "xx"
},
"details": {
"place": {
"id": "123",
"name": "xx"
},
"status": {
"id": "123",
"name": "xx"
},
"reason": {
"id": "123",
"name": "xx"
}
},
"item": [
{
"id": "11_1_1",
"action": "add",
"payment": {
"paied": true,
"coin": "dollar"
},
"type": {
"id": "11_1_1_zz",
"typeName": "zz",
"typeGroup": "zz",
"typeValue": "zz"
},
"details": {
"place": {
"id": "123",
"name": "xx"
},
"status": {
"id": "123",
"name": "xx"
},
"reason": {
"id": "123",
"name": "xx"
}
},
"item": [
{
"id": "11_1_1_1",
"action": "add",
"payment": {
"paied": true,
"coin": "NIS"
},
"type": {
"id": "11_1_1_1_xx",
"typeName": "xx",
"typeGroup": "xx",
"typeValue": "xx"
},
"details": {
"place": {
"id": "123",
"name": "xx"
},
"status": {
"id": "123",
"name": "xx"
},
"reason": {
"id": "123",
"name": "xx"
}
}
}
]
},
{
"id": "11_1_2",
"action": "add",
"payment": {
"paied": false,
"coin": "dollar"
},
"type": {
"id": "11_1_2_xx",
"typeName": "xx",
"typeGroup": "xx",
"typeValue": "xx"
},
"details": {
"place": {
"id": "123",
"name": "xx"
},
"status": {
"id": "123",
"name": "xx"
},
"reason": {
"id": "123",
"name": "xx"
}
},
"item": [
{
"id": "11_1_2_1",
"action": "add",
"payment": {
"paied": false,
"coin": "NIS"
},
"type": {
"id": "11_1_2_1_zz",
"typeName": "zz",
"typeGroup": "zz",
"typeValue": "zz"
},
"details": {
"place": {
"id": "123",
"name": "xx"
},
"status": {
"id": "123",
"name": "xx"
},
"reason": {
"id": "123",
"name": "xx"
}
}
}
]
}
]
}
]
}
Expected output:
[
{
"id": "11_1",
"action": "add",
"payment": {
"paied": true,
"coin": "dollar"
},
"type": {
"id": "11_1_xx",
"typeName": "xx"
},
"details": {
"place": {
"id": "123",
"name": "xx"
},
"status": {
"id": "123",
"name": "xx"
}
}
},
{
"id": "11_1_1_1",
"action": "add",
"payment": {
"paied": true,
"coin": "NIS"
},
"type": {
"id": "11_1_1_1_xx",
"typeName": "xx"
},
"details": {
"place": {
"id": "123",
"name": "xx"
},
"status": {
"id": "123",
"name": "xx"
}
}
},
{
"id": "11_1_2",
"action": "add",
"payment": {
"paied": false,
"coin": "dollar"
},
"type": {
"id": "11_1_2_xx",
"typeName": "xx"
},
"details": {
"place": {
"id": "123",
"name": "xx"
},
"status": {
"id": "123",
"name": "xx"
}
}
}
]
Can you please help me to write a simple spec that will do this?

You can use the following explained spec
[
{
// Determine all key-value pairs partitioned under the main value and type.typeValue combinations
"operation": "shift",
"spec": {
"item": {
"*": {
"item": {
"*": {
"item": {
"*": {
"type": {
"#(1,id)": "#(2,id).#(1,typeName).id", //traverse } character twice in order to reach the main id of the object
"#(1,action)": "#(2,id).#(1,typeName).action",
"*": "#(2,id).#(1,typeName).&1.&" // &1 replicates "type" key, & does the leaf value
},
"*": "#(1,id).#(1,type.typeName).&"
}
},
"type": {
"#(1,id)": "#(2,id).#(1,typeName).id",
"#(1,action)": "#(2,id).#(1,typeName).action",
"*": "#(2,id).#(1,typeName).&1.&"
},
"*": "#(1,id).#(1,type.typeName).&"
}
},
"type": {
"#(1,id)": "#(2,id).#(1,typeName).id",
"#(1,action)": "#(2,id).#(1,typeName).action",
"*": "#(2,id).#(1,typeName).&1.&"
},
"*": "#(1,id).#(1,type.typeName).&"
}
}
}
},
{
// Filter out by the desired value(in this case it's "xx")
"operation": "shift",
"spec": {
"*": {
"xx": ""
}
}
},
{
// Get rid of same repeating components of id and action arrays
"operation": "cardinality",
"spec": {
"*": {
"id": "ONE",
"action": "ONE"
}
}
},
{
// Get rid of undesired attributes
"operation": "remove",
"spec": {
"*": {
"type": {
"typeGroup": "",
"typeValue": ""
},
"det*": {
"reason": ""
}
}
}
}
]

You may try library Josson. Function map() build a new ObjectNode. Function field() modify the current ObjectNode and can be used to remove fields. The transformation statement is short and easy to understand.
https://github.com/octomix/josson
Josson josson = Josson.fromJsonString(inputJSON);
JsonNode node = josson.getNode(
"cumulateCollect(" +
" [type.typeName='xx']" + // filter
" .field(type.map(id, typeName)" + // need "type.id" and "type.typeName"
" ,details.field(reason:)" + // remove "details.reason"
" ,item:)" + // remove "item"
" ,item)"); // next round
System.out.println(node.toPrettyString());

will spring-data-rest expose methods in QueryByExampleExecutor interface?

i just learned how to use spring-data-rest to expose methods in repository. i wonder will the spring-data-rest expose methods in QueryByExampleExecutor<T>interface.if not, is there anyway to expose those methods. thx :).
i have a repository class that extend JpaRepository and i did not find any Restful api like methods in QueryByExampleExecutor<T> with resource discovery api.
this is what i got from resource discovery api
{
"alps": {
"version": "1.0",
"descriptor": [
{
"id": "studentEntity-representation",
"href": "http://127.0.0.1:8080/data/profile/studentEntities",
"descriptor": [
{
"name": "name",
"type": "SEMANTIC"
},
{
"name": "age",
"type": "SEMANTIC"
},
{
"name": "height",
"type": "SEMANTIC"
},
{
"name": "weight",
"type": "SEMANTIC"
},
{
"name": "hasPassed",
"type": "SEMANTIC"
}
]
},
{
"id": "create-studentEntities",
"name": "studentEntities",
"type": "UNSAFE",
"descriptor": [],
"rt": "#studentEntity-representation"
},
{
"id": "get-studentEntities",
"name": "studentEntities",
"type": "SAFE",
"descriptor": [
{
"name": "page",
"type": "SEMANTIC",
"doc": {
"format": "TEXT",
"value": "The page to return."
}
},
{
"name": "size",
"type": "SEMANTIC",
"doc": {
"format": "TEXT",
"value": "The size of the page to return."
}
},
{
"name": "sort",
"type": "SEMANTIC",
"doc": {
"format": "TEXT",
"value": "The sorting criteria to use to calculate the content of the page."
}
}
],
"rt": "#studentEntity-representation"
},
{
"id": "delete-studentEntity",
"name": "studentEntity",
"type": "IDEMPOTENT",
"descriptor": [],
"rt": "#studentEntity-representation"
},
{
"id": "update-studentEntity",
"name": "studentEntity",
"type": "IDEMPOTENT",
"descriptor": [],
"rt": "#studentEntity-representation"
},
{
"id": "get-studentEntity",
"name": "studentEntity",
"type": "SAFE",
"descriptor": [],
"rt": "#studentEntity-representation"
},
{
"id": "patch-studentEntity",
"name": "studentEntity",
"type": "UNSAFE",
"descriptor": [],
"rt": "#studentEntity-representation"
}
]
}
}

Elasticsearch High Level Rest Client Java sorting not working properly

I am pretty new with Elasticsearch High Level Rest Client (Java).
I have a very simple query which lists all records, but it seems sorting is not working properly.
Some of the fields are text types, so I needed to set fielddata to true.
Update:
Thanks Andrei for the solution.
I need to add another field which is an array of objects user.groups
I added .keyword to the mapping fields (text) which are sortable.
But I am getting unexpected results.
Example:
...
"groups": [
{"name", "ECPay", ... },
{"name", "Abangers", ... }
]
...
Based from the output (below), if there are two group items ECPay and Abangers... using user.groups.name.keyword sorting will consider Abangers.
I would like to consider the first element of user.groups array as a basis of the sort.
Based from the example (above), tt will base the sort with ECPay since it is the first element of the array.
To understand the problem (updated), let us check the search result (sort) below.
Search Result (Output):
{
"took": 18,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 74,
"max_score": null,
"hits": [
{
"_index": "acme_users",
"_type": "user",
"_id": "5",
"_score": null,
"_source": {
"name": "Ericsson Joseph Sultan Atutuli",
"country": "",
"uuid": "5",
"userId": 5,
"email": "ejsultanatutuli#gmail.com",
"deletedInd": false,
"groups": [
{
"name": "ECPay",
"id": 2
},
{
"name": "Abangers",
"id": 4
}
],
"company": ""
},
"sort": [
"Abangers"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "54",
"_score": null,
"_source": {
"name": "Florentina Atutuli",
"country": null,
"uuid": "54",
"userId": 54,
"email": "florentina.atutuli#gmail.com",
"deletedInd": false,
"groups": [
{
"name": "Abangers",
"id": 4
},
{
"name": "Test Group",
"id": 5
}
],
"company": null
},
"sort": [
"Abangers"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "37",
"_score": null,
"_source": {
"name": "dsfsdfsdf",
"country": null,
"uuid": "37",
"userId": 37,
"email": "asdf#sdf.com",
"deletedInd": false,
"groups": [
{
"name": "Abangers",
"id": 4
},
{
"name": "Test Group",
"id": 5
}
],
"company": null
},
"sort": [
"Abangers"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "3",
"_score": null,
"_source": {
"name": "Erick Atutuli",
"country": "Philippines",
"email": "erickatutuli#pakyas.com",
"userId": 3,
"uuid": "d8f4ab43-d33e-4a82-a08b-eb73342a0546",
"groups": [
{
"name": "ECPay",
"id": 2
},
{
"name": "Abangers",
"id": 4
}
],
"deletedInd": false,
"company": "Hotlegs Incorporated"
},
"sort": [
"Abangers"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "34",
"_score": null,
"_source": {
"name": "Chun-Li",
"country": null,
"email": "chunli#pakyas.com",
"uuid": "34",
"userId": 34,
"deletedInd": false,
"groups": [
{
"name": "Customers AU",
"id": 1
}
],
"company": null
},
"sort": [
"Customers AU"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "57",
"_score": null,
"_source": {
"name": "Eddy Bear",
"country": "US",
"email": "eddybear#pakyas.com",
"uuid": "57",
"userId": 57,
"deletedInd": false,
"groups": [
{
"name": "Customers AU",
"id": 1
}
],
"company": "Jollibee"
},
"sort": [
"Customers AU"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "42",
"_score": null,
"_source": {
"name": "Alfredo Pitik Mingaw",
"country": "",
"email": "akomykel#gmail.com",
"userId": 42,
"uuid": "42",
"deletedInd": false,
"groups": [
{
"name": "ECPay",
"id": 2
}
],
"company": ""
},
"sort": [
"ECPay"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "20",
"_score": null,
"_source": {
"name": "test",
"country": "Åland Islands",
"userId": 20,
"email": "test102#email.com",
"uuid": "20",
"groups": [
{
"name": "ECPay",
"id": 2
}
],
"deletedInd": false,
"company": "test"
},
"sort": [
"ECPay"
]
},
...
]
}
}
Now I added keyword field in the user mapping:
...
"groups": {
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
...
Make text fields sortable, removed fielddata and added keywords
http://localhost:9200/acme_users/_mapping/user
{
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"company": {
"type": "text",
"fielddata": true
},
"country": {
"type": "text",
"fielddata": true
},
"groups": {
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"email": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
The query below is generated from my Java application (Spring).
This is a very simple case wherein I just sort the results into name ascending order.
Query
Added .keyword to the sorting field. In the sample below... it will sort using groups.name.keyword which was originally groups.name
http://localhost:9200/acme_users/user/_search
{
"from" : 0,
"size" : 15,
"query" : {
"match_all" : {
"boost" : 1.0
}
},
"sort" : [
{
"groups.name.keyword" : {
"order" : "asc"
}
}
]
}
Original Problem:
But my problem now is, it seems ES sort does not work properly. The results changes when I change the order from asc to desc (vice versa).
Though the results changes, it seems that the names (result) is not sorting properly (a-z) or (z-a).
Seems that it is getting the last part of name and using it as the basis for the sort. I would like to base it on the first character value of name.
Original Search Result:
{
"took": 4,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 73,
"max_score": null,
"hits": [
{
"_index": "acme_users",
"_type": "user",
"_id": "22",
"_score": null,
"_source": {
"name": "Popeye Partner 01",
"country": null,
"userId": 22,
"email": "popeye_partner_01#kugmo.com",
"uuid": "22",
"deletedInd": false,
"company": null
},
"sort": [
"01"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "24",
"_score": null,
"_source": {
"name": "Dummy User 01",
"country": null,
"userId": 24,
"email": "dummy#dummy.com",
"uuid": "24",
"deletedInd": false,
"company": null
},
"sort": [
"01"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "23",
"_score": null,
"_source": {
"name": "Popeye Partner 02",
"country": null,
"userId": 23,
"email": "popeye_partner_02#kugmo.com",
"uuid": "23",
"deletedInd": false,
"company": null
},
"sort": [
"02"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "25",
"_score": null,
"_source": {
"name": "RT Administrator",
"country": null,
"userId": 25,
"email": "rt_administrator#kugmo.com",
"uuid": "25",
"deletedInd": false,
"company": null
},
"sort": [
"administrator"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "48",
"_score": null,
"_source": {
"name": "John Patrick Abnoy",
"country": null,
"userId": 48,
"email": "patrickabnoy#gmail.com",
"uuid": "48",
"deletedInd": false,
"company": null
},
"sort": [
"abnoy"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "5",
"_score": null,
"_source": {
"name": "Ericsson John Santol Atutuli",
"country": "",
"uuid": "5",
"userId": 5,
"email": "ejsantolatutuli#gmail.com",
"deletedInd": false,
"company": ""
},
"sort": [
"atutuli"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "54",
"_score": null,
"_source": {
"name": "Florentina Atutuli",
"country": null,
"uuid": "54",
"userId": 54,
"email": "florentina.atutuli#gmail.com",
"deletedInd": false,
"company": null
},
"sort": [
"atutuli"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "49",
"_score": null,
"_source": {
"name": "Laarnie Santol Atutuli",
"country": "",
"email": "lmsantolatutuli#gmail.com",
"userId": 49,
"uuid": "49",
"deletedInd": false,
"company": ""
},
"sort": [
"atutuli"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "3",
"_score": null,
"_source": {
"name": "Eric Atutuli",
"country": "Philippines",
"uuid": "d8f4ab43-d33e-4a82-a08b-eb73342a0546",
"userId": 3,
"email": "ericatutuli#kugmo.com",
"deletedInd": false,
"company": "Hotlegs Incorporated"
},
"sort": [
"atutuli"
]
},
{
"_index": "acme_users",
"_type": "user",
"_id": "29",
"_score": null,
"_source": {
"name": "Auberto Matulis",
"country": null,
"userId": 29,
"email": "bert.matulis#gmail.com",
"uuid": "29",
"deletedInd": false,
"company": null
},
"sort": [
"auberto"
]
}
]
}
}
Thanks!

name is a text field which means it is analyzed. Analyzed means it's split into tokens. If Popeye Partner 01 is split into popeye, partner, 01 which field you want to use for sorting? Probably none of them, since you want the sort to happen on the original text. For this to happen, add a sub-field to your name field
{
"name": {
"type" "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
and make it keyword then in your query use that for sorting:
"sort" : [
{
"name.keyword" : {
"order" : "asc"
}
}
]

We ran into the same issue and then we looked at the mapping which we created at the time of indexCreation and we observed a couple of things.
All the fields which always have one word as the value should be changed to keyword so we dont need to append .keyword in all the fields names(dont want to mantain schema on client side to append .keyword) or add "fielddata": true( avoid this as much as possible, because it takes up your heap mem)
"type":"keyword"
All the values which are defined as text but actually they are number, so we changed to either long or scaled_float. This helped us in sorting and aggregation.
"type":"long"
OR
"type": "scaled_float",
"scaling_factor": 100

Highly nested json data

I have three JSON arrays named item, topping and batter.
Now I want to store this data into single json object. It is highly nested data I don't know how to store data into single object like this.
Resultant object should be like this:
{
"item": [{
"id": "0001",
"type": "donut",
"name": "Cake",
"ppu": 0.55,
"batter": [{
"id": "1001",
"type": "Regular"
},
{
"id": "1002",
"type": "Chocolate"
},
{
"id": "1003",
"type": "Blueberry"
},
{
"id": "1004",
"type": "Devil's Food"
}
],
"topping": [{
"id": "5001",
"type": "None"
},
{
"id": "5002",
"type": "Glazed"
},
{
"id": "5005",
"type": "Sugar"
},
{
"id": "5007",
"type": "Powdered Sugar"
},
{
"id": "5006",
"type": "Chocolate with Sprinkles"
},
{
"id": "5003",
"type": "Chocolate"
},
{
"id": "5004",
"type": "Maple"
}
]
},
{
"id": "0002",
"type": "donut",
"name": "Raised",
"ppu": 0.55,
"batter": [{
"id": "1001",
"type": "Regular"
}],
"topping": [{
"id": "5001",
"type": "None"
},
{
"id": "5002",
"type": "Glazed"
},
{
"id": "5005",
"type": "Sugar"
},
{
"id": "5003",
"type": "Chocolate"
},
{
"id": "5004",
"type": "Maple"
}
]
},
{
"id": "0003",
"type": "donut",
"name": "Old Fashioned",
"ppu": 0.55,
"batter": [{
"id": "1001",
"type": "Regular"
},
{
"id": "1002",
"type": "Chocolate"
}
],
"topping": [{
"id": "5001",
"type": "None"
},
{
"id": "5002",
"type": "Glazed"
},
{
"id": "5003",
"type": "Chocolate"
},
{
"id": "5004",
"type": "Maple"
}
]
}
]
}

How can I extract the array of hashmap object from a hashmap object?

{
"errorCode": 200,
"opMessage": {
"personalDetails": {
"dob": {
"mm": 2,
"dd": 2,
"yyyy": 1975
},
"name": {
"fname": "sai gampa",
"lName": "krishna"
},
"maritalStatus": "Single",
"ssn": "123-45-7777"
},
"id": "0101",
"contactDetails": {
"address": [
{
"isPrimary": true,
"id": "HOME;2017-06-30",
"type": "Home",
"addr": {
"country": "USA",
"addr2": "W 24th St",
"addr1": "518",
"city": "New York",
"postalCode": "11122",
"state": "NY"
}
},
{
"isPrimary": false,
"id": "BUSN;2017-07-04",
"type": "Work",
"addr": {
"country": "USA",
"addr2": "Agsuaiajs",
"addr1": "Qwertyy",
"city": "Shshshsjsj",
"postalCode": "71899",
"state": "CO"
}
}
],
"bestWayToContact": "Phone",
"ph": [
{
"isPrimary": false,
"num": "666-666-6666",
"id": "Work",
"type": "Work"
},
{
"isPrimary": false,
"num": "444-444-4444",
"id": "Cell",
"type": "Cell"
},
{
"isPrimary": true,
"num": "111-111-1111",
"id": "Home",
"type": "Home"
}
],
"eMail": [
{
"isPrimary": true,
"id": "HOME",
"addr": "Shyam#gmail.con"
}
]
}
}
}

HashMap has a method called entrySet(). With entrySet() you can get all the entries in the hashmap.