I'm new to elastic search and I have a scenario where I need to group by field in nested path and sort each group based on the field in outer level.
example :- data format looks like this.
`
[
{
"percent": "1.0481265",
"subject": {
"id": "1234"
},
"updatedDate": "2016-12-31T00:00:00.000Z",
"id": "test"
},
{
"percent": "1.0581265",
"subject": {
"id": "1234"
},
"updatedDate": "2017-12-31T00:00:00.000Z",
"id": "test"
}
]
`
I need to group by subject.id and sort by updatedDate in each group and get the latest updated data as result with all level fields.
`
{
"query": {
"bool": {
"filter": [
{
"nested": {
"path": "subject",
"ignore_unmapped": true,
"query": {
"bool": {
"filter": [
{
"terms": {
"subject.id": [
"1234"
]
}
}
]
}
}
}
}
]
}
},
"sort": [
{
"updatedDate": {
"order": "desc",
"unmapped_type": "long"
}
}],
"aggs": {
"agg_ids": {
"nested": {
"path": "subject"
},
"aggs": {
"by_id": {
"terms": {
"field": "subject.id"
}
}
}
}
}
}
`
I'm able to group data by id, but not able to sort each with all the fields using updated date.
Related
I want to add one more field inside match inside function block in my query, but when i am adding, i am getting an error ------ "reason" : "[match] query doesn't support multiple fields, found [gender] and [id]",
How do i do it?
GET exp/_search
{
"_source": ["score","answer","gender","id"]
, "query": {
"function_score": {
"query": {
"match": {
"score": 10
}
},
"functions": [
{
"filter": {
"match":{
"gender":"male",
"id":1
}
},
"weight": 2
}
]
}
}
}
You can create bool query inside filter and it will be resolved your issue. match query does not support providing 2 diffrent field and values. You can use bool query for same purpose.
{
"_source": [
"score",
"answer",
"gender",
"id"
],
"query": {
"function_score": {
"query": {
"match": {
"score": 10
}
},
"functions": [
{
"filter": {
"bool": {
"must": [
{
"match": {
"gender": "male"
}
},
{
"match": {
"id": 1
}
}
]
}
},
"weight": 2
}
]
}
}
}
Also, If you want to apply two different boosting value for gender and id then you can give two filter clause as shown below:
{
"_source": [
"score",
"answer",
"gender",
"id"
],
"query": {
"function_score": {
"query": {
"match": {
"score": 10
}
},
"functions": [
{
"filter": {
"match": {
"gender": "male"
}
},
"weight": 2
},
{
"filter": {
"match": {
"id": 1
}
},
"weight": 1
}
]
}
}
}
I am have a set of data in the following structure:
[
{
"productId": "ProductId1",
"customerNumbers": [
"customer": {
"name": "Prod1Cust1",
"totalOrders": 23
},
"customer": {
"name": "Prod2Cust1",
"totalOrders": 5
},
"customer": {
"name": "Prod3Cust1",
"totalOrders": 5
}
]
},
{
"productId": "ProductId2",
"customerNumbers": [
"customer": {
"name": "Prod2Cust1",
"totalOrders": 23
},
"customer": {
"name": "Prod2Cust1",
"totalOrders": 5
}
]
}
]
and I need to fetch all the records which have a prefix of "Prod1 as in name field(in the example avoid, only first record should be returned i.e. ProductId1). Also, when the data is returned, I need to just fetch just the customer number whose prefix is Prod1 i.e:
Correct Output:
{
"productId": "ProductId1",
"customerNumbers": [
"customer": {
"name": "Prod1Cust1",
"totalOrders": 23
}
]
}
Instead of:
{
"productId": "ProductId1",
"customerNumbers": [
"customer": {
"name": "Prod1Cust1",
"totalOrders": 23
},
"customer": {
"name": "Prod2Cust1",
"totalOrders": 5
},
"customer": {
"name": "Prod3Cust1",
"totalOrders": 5
}
]
}
I'm able to fetch the records whose Name prefix is "Prod1" using nested query coupled with MatchPhrasePrefixQuery (this returns me result with all the customer numbers). How can I further filter the data to get customer numbers whose Name prefix is "Prod1".
Following is my current query:
{
"from": 0,
"size": 10,
"sort": [
{
"name.keyword": {
"missing": "_first",
"order": "asc"
}
}
],
"query": {
"bool": {
"must": [
{
"bool": {
"must": [
{
"nested": {
"query": {
"bool": {
"must": [
{
"match": {
"customerNumbers.name": {
"query": "Prod1",
"type": "phrase_prefix"
}
}
}
]
}
},
"path": "customerNumbers"
}
}
]
}
}
]
}
}
}
P.S: I'm using ElasticSearch 5.x with Nest.
Try using inner_hits:
PUT products
{"mappings":{"_doc":{"properties":{"customerNumbers":{"type":"nested"}}}}}
POST products/_doc
{"productId":"ProductId1","customerNumbers":[{"name":"Prod1Cust1","totalOrders":23},{"name":"Prod2Cust1","totalOrders":5},{"name":"Prod3Cust1","totalOrders":5}]}
POST products/_doc
{"productId":"ProductId2","customerNumbers":[{"name":"Prod2Cust1","totalOrders":23},{"name":"Prod2Cust1","totalOrders":5}]}
GET products/_search
{
"_source": "inner_hits",
"from": 0,
"size": 10,
"query": {
"bool": {
"must": [
{
"bool": {
"must": [
{
"nested": {
"path": "customerNumbers",
"query": {
"bool": {
"must": [
{
"match_phrase_prefix": {
"customerNumbers.name": {
"query": "Prod1"
}
}
}
]
}
},
"inner_hits": {}
}
}
]
}
}
]
}
}
}
yielding the following hits
[
{
"_index":"products",
"_type":"_doc",
"_id":"tyQGo3EBdiyDG0RsTa0N",
"_score":0.9808292,
"_source":{
},
"inner_hits":{
"customerNumbers":{
"hits":{
"total":1,
"max_score":0.9808292,
"hits":[
{
"_index":"products",
"_type":"_doc",
"_id":"tyQGo3EBdiyDG0RsTa0N",
"_nested":{
"field":"customerNumbers",
"offset":0
},
"_score":0.9808292,
"_source":{
"name":"Prod1Cust1", <-----
"totalOrders":23
}
}
]
}
}
}
}
]
I am learning to use Elasticsearch as a basic recommender engine.
My elasticsearch document contains records with nested entities as follows
PUT recs/user/1
{
"name" : "Brad Pitt",
"movies_liked": [
{
"name": "Forrest Gump",
"score": 1
},
{
"name": "Terminator",
"score": 4
},
{
"name": "Rambo",
"score": 4
},
{
"name": "Rocky",
"score": 4
},
{
"name": "Good Will Hunting",
"score": 2
}
]
}
PUT recs/user/2
{
"name" : "Tom Cruise",
"movies_liked": [
{
"name": "Forrest Gump",
"score": 2
},
{
"name": "Terminator",
"score": 1
},
{
"name": "Rocky IV",
"score": 1
},
{
"name": "Rocky",
"score": 1
},
{
"name": "Rocky II",
"score": 1
},
{
"name": "Predator",
"score": 4
}
]
}
I would like to search for users who specifically like "Forrest Gump","Terminator" and "Rambo".
I have used a nested query which currently looks like this
POST recs/user/_search
{
"query": {
"nested": {
"path": "movies_liked",
"query": {
"terms": {
"movies_liked.name": ["Forrest Gump","Terminator","Rambo"]
}
}
}
}
}
However when I execute this search, I expected to see only the first record which has all the required terms, but in the results I am getting both the records. In the second record the user clearly does not have "Rambo" in his liked list. I understand that this query is doing an "OR" operation with the given terms, How do I tweak this query to do an "AND" operation so that only the records having all the terms get matched?
How do I tweak this query to do an "AND" operation so that only the records having all the terms get matched?
By using a bool query:
POST recs/user/_search
{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "movies_liked",
"query": {
"bool": {
"must": [
{
"terms": {
"movies_liked.name": [
"Forrest Gump"
]
}
}
]
}
}
}
},
{
"nested": {
"path": "movies_liked",
"query": {
"bool": {
"must": [
{
"terms": {
"movies_liked.name": [
"Terminator"
]
}
}
]
}
}
}
},
{
"nested": {
"path": "movies_liked",
"query": {
"bool": {
"must": [
{
"terms": {
"movies_liked.name": [
"Rambo"
]
}
}
]
}
}
}
}
]
}
}
}
Note that bool wraps around several nested queries, not the other way around. It is important because the scope of a nested query is the nested document, because it basically a hidden separate object.
Hope that helps!
I'm trying to search for keyword and then add nested queries for amenities which is a nested field of an array of objects.
With the query below I am able to search when I'm only matching one amenity id but when I have more than one it doesn't return anything.
Anyone have an idea what is wrong with my query ?
{
"sort": [
{
"_score": {
"order": "desc"
}
},
{
"_geo_distance": {
"geolocation": [
100,
10
],
"order": "asc",
"unit": "m",
"mode": "min",
"distance_type": "sloppy_arc"
}
}
],
"query": {
"bool": {
"must": [
{
"multi_match": {
"fields": [
"name^2",
"city",
"state",
"zip"
],
"fuzziness": 5,
"query": "complete"
}
},
{
"nested": {
"path": "amenities",
"query": {
"bool": {
"must": [
{
"term": {
"amenities.id": "1"
}
},
{
"term": {
"amenities.id": "2"
}
}
]
}
}
}
}
]
}
}
}
When you do:
"must": [
{
"term": {
"amenities.id": "1"
}
},
{
"term": {
"amenities.id": "2"
}
}]
What you're actually saying is find me any document where "amenities.id"="1" and "amenities.id"="2" which unless "amenities.id" is a list of values it won't work.
What you probably want to say is find me any document where "amenities.id"="1" or "amenities.id"="2"
To do that you should use should instead of must:
"should": [
{
"term": {
"amenities.id": "1"
}
},
{
"term": {
"amenities.id": "2"
}
}]
let say I have following Document:
{
id: 1,
name: "xyz",
users: [
{
name: 'abc',
surname: 'def'
},
{
name: 'xyz',
surname: 'wef'
},
{
name: 'defg',
surname: 'pqr'
}
]
}
I want to Get only matching nested objects with All Top level fields in search response.
I mean If I search/filter for users with name 'abc', I want below response
{
id: 1,
name: "xyz",
users: [
{
name: 'abc',
surname: 'def'
}
]
}
How can I do that?
Reference : select matching objects from array in elasticsearch
If you're ok with having all root fields except the nested one and then only the matching inner hits in the nested field, then we can re-use the previous answer like this by specifying a slightly more involved source filtering parameter:
{
"_source": {
"includes": [ "*" ],
"excludes": [ "users" ]
},
"query": {
"nested": {
"path": "users",
"inner_hits": { <---- this is where the magic happens
"_source": [
"name", "surname"
]
},
"query": {
"bool": {
"must": [
{
"term": {
"users.name": "abc"
}
}
]
}
}
}
}
}
Maybe late, I use nested sorting to limit element on my nested relation, here a example :
"sort": {
"ouverture.periodesOuvertures.dateDebut": {
"order": "asc",
"mode": "min",
"nested_filter": {
"range": {
"ouverture.periodesOuvertures.dateFin": {
"gte": "2017-08-29",
"format": "yyyy-MM-dd"
}
}
},
"nested_path": "ouverture.periodesOuvertures"
}
},
Since 5.5 ES (I think) you can use filter on nested query.
Here a example of nested query filter I use:
{
"nested": {
"path": "ouverture.periodesOuvertures",
"query": {
"bool": {
"must": [
{
"range": {
"ouverture.periodesOuvertures.dateFin": {
"gte": "2017-08-29",
"format": "yyyy-MM-dd"
}
}
},
{
"range": {
"ouverture.periodesOuvertures.dateFin": {
"lte": "2017-09-30",
"format": "yyyy-MM-dd"
}
}
}
],
"filter": [
{
"range": {
"ouverture.periodesOuvertures.dateFin": {
"gte": "2017-08-29",
"format": "yyyy-MM-dd"
}
}
},
{
"range": {
"ouverture.periodesOuvertures.dateFin": {
"lte": "2017-09-30",
"format": "yyyy-MM-dd"
}
}
}
]
}
}
}
}
Hope this can help ;)
Plus if you ES is not in the last version (5.5) inner_hits could slow your query Including inner hits drastically slows down query results
https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-inner-hits.html#nested-inner-hits-source
"inner_hits": {
"_source" : false,
"stored_fields" : ["name", "surname"]
}
but you may need to change mapping to set those fields as "stored_fields" , otherwise you can use
"inner_hits": {}
to get a result that not that perfect.
You can make such a request, but the response will have internal fields starting with _
{
"_source": {
"includes": [ "*" ],
"excludes": [ "users" ]
},
"query": {
"nested": {
"path": "users",
"inner_hits": {},
"query": {
"bool": {
"must": [
{ "match": { "users.name": "abc" }}
]
}
}
}
}
}
In one of my projects, My expectation was to retrieve unique conversation messages text(inner fields like messages.text) having specific tags. So instead of using inner_hits, I used aggregation like below,
final NestedAggregationBuilder aggregation = AggregationBuilders.nested("parentPath", "messages").subAggregation(AggregationBuilders.terms("innerPath").field("messages.tag"));
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.addAggregation(aggregation).build();
final Aggregations aggregations = elasticsearchOperations.search(searchQuery, Conversation.class).getAggregations();
final ParsedNested parentAgg = (ParsedNested) aggregations.asMap().get("parentPath");
final Aggregations childAgg = parentAgg.getAggregations();
final ParsedStringTerms childParsedNested = (ParsedStringTerms) childAgg.asMap().get("innerPath");
// Here you will get unique expected inner fields in key part.
Map<String, Long> agg = childParsedNested.getBuckets().stream().collect(Collectors.toMap(Bucket::getKeyAsString, Bucket::getDocCount));
I use the following body to get that result (I have set the full path to the values):
{
"_source": {
"includes": [ "*" ],
"excludes": [ "users" ]
},
"query": {
"nested": {
"path": "users",
"inner_hits": {
"_source": [
"users.name", "users.surname"
]
},
"query": {
"bool": {
"must": [
{
"term": {
"users.name": "abc"
}
}
]
}
}
}
}
}
Also another way exists:
{
"_source": {
"includes": [ "*" ],
"excludes": [ "users" ]
},
"query": {
"nested": {
"path": "users",
"inner_hits": {
"_source": false,
"docvalue_fields": [
"users.name", "users.surname"
]
},
"query": {
"bool": {
"must": [
{
"term": {
"users.name": "abc"
}
}
]
}
}
}
}
}
See results in inner_hits of the result hits.
https://www.elastic.co/guide/en/elasticsearch/reference/7.15/inner-hits.html#nested-inner-hits-source