edit: I'm using elastic search 7.3.0
I'm trying to do a query with an aggregation and sub aggregation, but the sub aggregation is absent from the SearchResponse.
As part of debugging, I ran my query in a unit test, copied the query, and ran it manually with postman. There, the response is exactly what I expect, but for some reason, in my java code, parts are missing.
SearchRequest request = new SearchRequest("index");
SearchSourceBuidler search = new SearchSourceBuilder();
SortBuilder sortByDate = SortBuilders
.fieldSort("date")
.order(SortOrder.DESC);
// Getting the latest result for each bucket
TopHitsAggregationBuilder latestResults = AggregationBuilders
.topHits("latest")
.sort(sortByDate)
.fetchSource("*","")
.size(1);
// Aggregate per service
TermsAggregationBuilder perService = AggregationBuilders
.terms("services")
.field("service.service_id")
.subAggregation(latestResults);
search.aggregation(perService);
search.size(1);
request.source(search);
SearchResponse response = client.search(request, RequestOptions.DEFAULT);
Here is the request generated:
{
"size": 0,
"aggregations": {
"services": {
"terms": {
"field": "service.service_id",
"size": 10,
"min_doc_count": 1,
"shard_min_doc_count": 0,
"show_term_doc_count_error": false,
"order": [
{
"_count": "desc"
},
{
"_key": "asc"
}
]
},
"aggregations": {
"latest": {
"top_hits": {
"from": 0,
"size": 1,
"version": false,
"seq_no_primary_term": false,
"explain": false,
"_source": {
"includes": [
"*"
],
"excludes": [
""
]
},
"sort": [
{
"date": {
"order": "desc"
}
}
]
}
}
}
}
}
}
In my code, response is
{
...
"aggregations": {
"sterms#services": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": []
}
}
}
If I run the same query manually I get
{
...
"aggregations": {
"services": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "09045f59-3709-4769-8c92-d611f773a401",
"doc_count": 2,
"latest": {
"hits": {
"total": {
"value": 2,
"relation": "eq"
},
"max_score": null,
"hits": [ ... ]
I have a JSON
{
"Id": "xxx",
"Type": "Transaction.Create",
"Payload": {
"result": 2,
"description": "Pending",
"body": {
"redirect": {
"url": "xxx",
"fields": {
"MD": "8a829449620619e80162252adeb66a39"
}
},
"card": {
"expiryMonth": "1",
"expiryYear": "2033"
},
"order": {
"amount": 1
}
}
}
}
And I want to remove the card info of it like this:
{
"Id": "xxx",
"Type": "Transaction.Create",
"Payload": {
"result": 2,
"description": "Pending",
"body": {
"redirect": {
"url": "xxx",
"fields": {
"MD": "8a829449620619e80162252adeb66a39"
}
},
"order": {
"amount": 1
}
}
}
}
How can I do this with Apache velocity?
What works is:
#set($content = $util.urlEncode($input.json('$')))
#set($new = $content.replaceAll("2033","2055"))
Action=SendMessage&MessageBody={"body": "$new","Event-Signature": "$util.urlEncode($input.params('Event-Signature'))"}
This gives me
{
"Id": "xxx",
"Type": "Transaction.Create",
"Payload": {
"result": 2,
"description": "Pending",
"body": {
"redirect": {
"url": "xxx",
"fields": {
"MD": "8a829449620619e80162252adeb66a39"
}
},
"card": {
"expiryMonth": "1",
"expiryYear": "2050"
},
"order": {
"amount": 1
}
}
}
}
But now I want to remove the card part but it does not work:
#set($content = $util.urlEncode($input.json('$')))
#set($new = $content.delete("$.Payload.body.card"))
Action=SendMessage&MessageBody={"body": "$new","Event-Signature": "$util.urlEncode($input.params('Event-Signature'))"}
what am I doing wrong?
Main goal is transform a mapping template in API Gateway for a webhook. The webhook contains to many information and we want to remove some part of the JSON POST call.
Try using the below
#set($dummy=$content.Payload.remove("card"))
I have a huge amount of elasticsearch data, I neeed to make aggregations and return buckets. I need to limit data size returned from elasticsearch to only get a sample for the data not all data.
I have tried adding "size" attribute. But it's not acceptable in bucketing aggregations.
{
"size": 0,
"query": {
"bool": {
"adjust_pure_negative": true,
"boost": 1
}
},
"aggregations": {
"my_agg_1": {
"histogram": {
"field": "coAt",
"interval": 86400,
"offset": 1558216800,
"order": {
"_key": "asc"
},
"keyed": false,
"min_doc_count": 1
},
"aggregations": {
"my_agg_2": {
"terms": {
"field": "atr1",
"missing": "NaN",
"value_type": "string",
"size": 2147483647,
"min_doc_count": 1,
"shard_min_doc_count": 0,
"show_term_doc_count_error": false,
"order": [
{
"_count": "desc"
},
{
"_key": "asc"
}
]
},
"aggregations": {
"atr2": {
"top_hits": {
"from": 0,
"size": 1,
"version": false,
"explain": false,
"sort": [
{
"coAt": {
"order": "desc"
}
}
]
}
},
"clientIP_count": {
"value_count": {
"field": "clientIP"
}
}
}
}
}
}
}
}
I'm trying to create index following this guide: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/master/java-rest-high-create-index.html#_providing_the_whole_source
The problem is, that index is not created properly. Looks like whole settings section as well as completion type is ignored.
My json file:
{
"settings": {
"number_of_shards": 3,
"number_of_replicas": 1,
"analysis": {
"filter": {},
"analyzer": {
"keyword_analyzer": {
"filter": [
"lowercase",
"asciifolding",
"trim"
],
"char_filter": [],
"type": "custom",
"tokenizer": "keyword"
}
}
}
},
"mappings": {
"my_type": {
"properties": {
"first": {
"type": "text",
"fields": {
"keywordstring": {
"type": "text",
"analyzer": "keyword_analyzer"
},
"completion": {
"type": "completion"
}
},
"analyzer": "standard"
},
"second": {
"type": "text",
"fields": {
"keywordstring": {
"type": "text",
"analyzer": "keyword_analyzer"
},
"completion": {
"type": "completion"
}
},
"analyzer": "standard"
},
"third": {
"type": "text",
"fields": {
"keywordstring": {
"type": "text",
"analyzer": "keyword_analyzer"
},
"completion": {
"type": "completion"
}
},
"analyzer": "standard"
},
"fourth": {
"type": "text",
"fields": {
"keywordstring": {
"type": "text",
"analyzer": "keyword_analyzer"
},
"completion": {
"type": "completion"
}
},
"analyzer": "standard"
}
}
}
}
}
Java code:
CreateIndexRequest indexRequest = new CreateIndexRequest(ESClientConfiguration.INDEX_NAME);
URL url = Resources.getResource(TERYT_INDEX_CONFIGURATION_FILE_NAME);
return Try.of(() -> Resources.toString(url, Charsets.UTF_8))
.map(jsonIndexConfiguration -> indexRequest.source(jsonIndexConfiguration, XContentType.JSON))
.get();
createIndexRequest.setTimeout(TimeValue.timeValueMinutes(2));
Try.of(() -> client.indices().create(createIndexRequest, RequestOptions.DEFAULT))...
Index is created but when I look into Index Metadata, it looks completly wrong:
{
"state": "open",
"settings": {
"index": {
"creation_date": "1556379012380",
"number_of_shards": "1",
"number_of_replicas": "1",
"uuid": "L5fmkrjeQ6eKmuDyZ3MP3g",
"version": {
"created": "7000099"
},
"provided_name": "my_index"
}
},
"mappings": {
"my_type": {
"properties": {
"first": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"second": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"third": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"fourth": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
}
}
}
},
"aliases": [],
"primary_terms": {
"0": 1
},
"in_sync_allocations": {
"0": [
"Cx6tBeohR8mzbTO74dwsCw",
"FcTUhpb_SL2LiaEyy_uwkg"
]
}
}
There is only 1 shard without replicas, I also don't see any informations about completion type. Does someone could tell me what I'm doing wrong here?
I think that this line:
Try.of(() -> client.indices().create(createIndexRequest, RequestOptions.DEFAULT))...
is hiding an important exception.
Here you are using elasticsearch 7.0.0 which does not allow anymore giving a "type" name in your mapping.
Instead of
"mappings": {
"my_type": {
"properties": {
You should write:
"mappings": {
"properties": {
Because of the exception and the fact that probably after index creation you are indexing some documents, default index settings and mapping are applied.
Which explains what you are seeing.
You need to fix your index settings first.
I'd recommend doing that in Kibana dev console.
Here I have a response from the facebook server that returns the list of albums of a selected user. I want to create an array for all the album names labeled "name" as well as the "link" and "cover_photo" to be ready to processed into a listview
{
"data": [
{
"id": "664462156031",
"from": {
"name": "Nate Ellender",
"id": "68004222"
},
"name": "Mobile Uploads",
"link": "https://www.facebook.com/album.php?fbid=664462156031&id=68004222&aid=2081375",
"cover_photo": "754991180141",
"count": 88,
"type": "mobile",
"created_time": "2012-05-12T00:38:14+0000",
"updated_time": "2013-11-04T20:45:08+0000",
"can_upload": false
},
{
"id": "813288706431",
"from": {
"name": "Nate Ellender",
"id": "68004222"
},
"name": "Bitstrips Photos",
"link": "https://www.facebook.com/album.php?fbid=813288706431&id=68004222&aid=1073741826",
"cover_photo": "813288781281",
"count": 1,
"type": "app",
"created_time": "2013-10-31T21:53:11+0000",
"updated_time": "2013-10-31T21:53:15+0000",
"can_upload": false
},
{
"id": "757384214481",
"from": {
"name": "Nate Ellender",
"id": "68004222"
},
"name": "Galveston Vacation 2013",
"description": "Our trip before joining Air Force",
"link": "https://www.facebook.com/album.php?fbid=757384214481&id=68004222&aid=1073741825",
"cover_photo": "757221350861",
"count": 8,
"type": "normal",
"created_time": "2013-05-15T18:37:19+0000",
"updated_time": "2013-05-15T22:12:52+0000",
"can_upload": false,
"likes": {
"data": [
{
"id": "100002572634186",
"name": "Misty O'Quain"
},
{
"id": "100000582072776",
"name": "Clifford Joyce"
},
{
"id": "1045514613",
"name": "Caity Ellender"
}
],
"paging": {
"cursors": {
"after": "MTA0NTUxNDYxMw==",
"before": "MTAwMDAyNTcyNjM0MTg2"
}
}
}
},
{
"id": "542202136091",
"from": {
"name": "Nate Ellender",
"id": "68004222"
},
"name": "Profile Pictures",
"link": "https://www.facebook.com/album.php?fbid=542202136091&id=68004222&aid=2054735",
"cover_photo": "749743202131",
"count": 20,
"type": "profile",
"created_time": "2010-09-02T04:43:32+0000",
"updated_time": "2013-04-13T13:46:14+0000",
"can_upload": false
},
{
"id": "646032913381",
"from": {
"name": "Nate Ellender",
"id": "68004222"
},
"name": "Cover Photos",
"link": "https://www.facebook.com/album.php?fbid=646032913381&id=68004222&aid=2079483",
"cover_photo": "681058471881",
"count": 2,
"type": "cover",
"created_time": "2012-03-22T00:25:50+0000",
"updated_time": "2012-07-17T15:26:24+0000",
"can_upload": false
},
{
"id": "599115930391",
"from": {
"name": "Nate Ellender",
"id": "68004222"
},
"name": "Timeline Photos",
"link": "https://www.facebook.com/album.php?fbid=599115930391&id=68004222&aid=2073818",
"cover_photo": "599115935381",
"count": 1,
"type": "wall",
"created_time": "2011-10-20T14:25:23+0000",
"updated_time": "2011-10-20T14:25:23+0000",
"can_upload": false
},
{
"id": "551798524851",
"from": {
"name": "Nate Ellender",
"id": "68004222"
},
"name": "My Year - 2010",
"link": "https://www.facebook.com/album.php?fbid=551798524851&id=68004222&aid=2060899",
"cover_photo": "551798544811",
"count": 1,
"type": "normal",
"created_time": "2010-12-19T05:11:51+0000",
"updated_time": "2010-12-19T05:12:14+0000",
"can_upload": false
},
{
"id": "522841060841",
"from": {
"name": "Nate Ellender",
"id": "68004222"
},
"name": "Night Lightnin",
"description": "Always wanted to do this, because lightning is way cooler at night. Perhaps next time I'll be able to refine my methods a little. Read as \"bring tripod so you don't have to prop cam up on roof of car\"",
"location": "Just north of Sulphur",
"link": "https://www.facebook.com/album.php?fbid=522841060841&id=68004222&aid=2036447",
"cover_photo": "522841195571",
"count": 15,
"type": "normal",
"created_time": "2009-09-16T04:38:43+0000",
"updated_time": "2009-09-16T04:42:32+0000",
"can_upload": false
},
{
"id": "513270939441",
"from": {
"name": "Nate Ellender",
"id": "68004222"
},
"name": "The Stuff That Doesn't Make Another Album",
"description": "See title.",
"location": "Good question",
"link": "https://www.facebook.com/album.php?fbid=513270939441&id=68004222&aid=2029083",
"cover_photo": "513271029261",
"count": 9,
"type": "normal",
"created_time": "2009-03-03T06:43:09+0000",
"updated_time": "2009-04-01T02:20:22+0000",
"can_upload": false,
"comments": {
"data": [
{
"id": "513270939441_3444",
"from": {
"name": "Misty Bylsma Royal",
"id": "592607110"
},
"message": "I love how the beautiful lady pile is in the same album as the bull pics. Nice.",
"can_remove": false,
"created_time": "2009-03-03T22:37:18+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "513270939441_3451",
"from": {
"name": "Nate Ellender",
"id": "68004222"
},
"message": "Only because they are both things that i only had a few pictures of.",
"can_remove": false,
"created_time": "2009-03-04T02:46:11+0000",
"like_count": 0,
"user_likes": false
}
],
"paging": {
"cursors": {
"after": "Mg==",
"before": "MQ=="
}
}
}
},
{
"id": "511605661671",
"from": {
"name": "Nate Ellender",
"id": "68004222"
},
"name": "What the...?",
"description": "What's wrong with the rain? It's flaky and piles up on stuff... Did the plants leak something?\n",
"location": "Outside",
"link": "https://www.facebook.com/album.php?fbid=511605661671&id=68004222&aid=2026641",
"cover_photo": "511605681631",
"count": 54,
"type": "normal",
"created_time": "2008-12-11T11:59:27+0000",
"updated_time": "2008-12-13T18:06:00+0000",
"can_upload": false
},
{
"id": "511565886381",
"from": {
"name": "Nate Ellender",
"id": "68004222"
},
"name": "The Christmas Tree Farm",
"description": "An Ellender Tradition",
"location": "Grant",
"link": "https://www.facebook.com/album.php?fbid=511565886381&id=68004222&aid=2026590",
"cover_photo": "511566170811",
"count": 19,
"type": "normal",
"created_time": "2008-12-09T02:38:54+0000",
"updated_time": "2008-12-09T02:55:21+0000",
"can_upload": false
}
],
"paging": {
"cursors": {
"after": "NTExNTY1ODg2Mzgx",
"before": "NjY0NDYyMTU2MDMx"
}
}
}
Also, here is where the json data is returned (response). I will need to start parsing the data under response.
Session.NewPermissionsRequest np = new Session.NewPermissionsRequest(this, "friends_photos");
Session.getActiveSession().requestNewReadPermissions(np);
Request rq = new Request(Session.getActiveSession(), userID + "/albums", null, HttpMethod.GET, new Request.Callback() {
#Override
public void onCompleted(Response response) {
}
});
rq.executeAsync();
Any example would be helpful. I would like to study the code to get a better understanding on parsing json data.
You can get a good example of how to do here: http://www.androidhive.info/2012/01/android-json-parsing-tutorial/
Basically you have to use JSONObject and JSONArray, that you can get from getJSONObject() and getJSONArray().
Once you're on the good spot, use getJSONString() to get the desired value.
In your case it will be something like this:
JSONObject json = new JSONObject(response)
JSONArray jarray = json.getJSONArray("data");
for(int i = 0; i < jarray.length(); i++){
JSONObject oneAlbum = jarray.getJSONObject(i);
//get your values
oneAlbum.getJSONString("name"); // this will return you the album's name.
}
Hope this will help you.
Try this
Session.NewPermissionsRequest np = new Session.NewPermissionsRequest(this, "friends_photos");
Session.getActiveSession().requestNewReadPermissions(np);
Request rq = new Request(Session.getActiveSession(), userID + "/albums", null, HttpMethod.GET, new Request.Callback() {
#Override
public void onCompleted(Response response) {
JSONArray albumArr = response.getGraphObject().getInnerJSONObject().getJSONArray("data");
for (int i = 0; i < albumArr.length(); i++) {
JSONObject item = albumArr.getJSONObject(i);
System.out.println("id : " + item.getString("id"));
System.out.println("fromName : " + item.getJSONObject("from").getString("name"));
System.out.println("fromid : " + item.getJSONObject("from").getString("id"));
System.out.println("link : " + item.getString("link"));
System.out.println("cover_photo : " + item.getString("cover_photo"));
System.out.println("count : " + item.getString("count"));
System.out.println("created_time : " + item.getString("created_time"));
System.out.println("updated_time : " + item.getString("updated_time"));
System.out.println("can_upload : " + item.getString("can_upload"));
}
}
});
rq.executeAsync();