Empty inner_hits in compound Elasticsearch filter - elasticsearch

I'm seeing what appears to be aberrant behavior in inner_hits results within nested boolean queries.
Test data (abbreviated for brevity):
# MAPPING
PUT unit_testing
{
"mappings": {
"document": {
"properties": {
"display_name": {"type": "text"},
"metadata": {
"properties": {
"NAME": {"type": "text"}
}
}
}
},
"paragraph": {
"_parent": {"type": "document"},
"_routing": {"required": true},
"properties": {
"checksum": {"type": "text"},
"sentences": {
"type": "nested",
"properties": {
"text": {"type": "text"}
}
}
}
}
}
}
# DOCUMENT X 2 (d0, d1)
PUT unit_testing/document/doc_id_d0
{
"display_name": "Test Document d0",
"paragraphs": [
"para_id_d0p0",
"para_id_d0p1"
],
"metadata": {"NAME": "Test Document d0 Metadata"}
}
# PARAGRAPH X 2 (d0p0, d1p0)
PUT unit_testing/paragraph/para_id_d0p0?parent=doc_id_d0
{
"checksum": "para_checksum_d0p0",
"sentences": [
{"text": "Test sentence d0p0s0"},
{"text": "Test sentence d0p0s1 ODD"},
{"text": "Test sentence d0p0s2 EVEN"},
{"text": "Test sentence d0p0s3 ODD"},
{"text": "Test sentence d0p0s4 EVEN"}
]
}
This initial query behaves as I would expect (I'm aware that the metadata filter isn't actually necessary in this example case):
GET unit_testing/paragraph/_search
{
"_source": "false",
"query": {
"bool": {
"must": [
{
"has_parent": {
"query": {
"match_phrase": {
"metadata.NAME": "Test Document d0 Metadata"
}
},
"type": "document"
}
},
{
"nested": {
"inner_hits": {},
"path": "sentences",
"query": {
"match": {
"sentences.text": "d0p0s0"
}
}
}
}
]
}
}
}
It yields an inner_hits object containing the one sentence that matched the predicate (some fields removed for clarity):
{
"hits": {
"hits": [
{
"_source": {},
"inner_hits": {
"sentences": {
"hits": {
"hits": [
{
"_source": {
"text": "Test sentence d0p0s0"
}
}
]
}
}
}
}
]
}
}
The following query is an attempt to embed the query above within a parent "should" clause, to create a logical OR between the initial query, and an additional query that matches a single sentence:
GET unit_testing/paragraph/_search
{
"_source": "false",
"query": {
"bool": {
"should": [
{
"bool": {
"must": [
{
"has_parent": {
"query": {
"match_phrase": {
"metadata.NAME": "Test Document d0 Metadata"
}
},
"type": "document"
}
},
{
"nested": {
"inner_hits": {},
"path": "sentences",
"query": {
"match": {
"sentences.text": "d0p0s0"
}
}
}
}
]
}
},
{
"nested": {
"inner_hits": {},
"path": "sentences",
"query": {
"match": {
"sentences.text": "d1p0s0"
}
}
}
}
]
}
}
}
While the "d1" query outputs the result one would expect, with an inner_hits object containing the matching sentence, the original "d0" query now yields an empty inner_hits object:
{
"hits": {
"hits": [
{
"_source": {},
"inner_hits": {
"sentences": {
"hits": {
"total": 0,
"hits": []
}
}
}
},
{
"_source": {},
"inner_hits": {
"sentences": {
"hits": {
"hits": [
{
"_source": {
"text": "Test sentence d1p0s0"
}
}
]
}
}
}
}
]
}
}
Although I'm using the elasticsearch_dsl Python library to build and combine these queries, and I'm something of a novice with respect to the Query DSL, the query format looks solid to me.
What am I missing?

I think what is missing is the name parameter for inner_hits - you have two inner_hits clauses at two different queries that would end up with the same name. Try giving the inner_hits a name parameter (0).
0 - https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-inner-hits.html#_options

Related

Query for nested fields returns results as if there was no nested mapping

I am having difficulties understanding, why a query across nested fields is returning unexpected results.
I have the following template for my index
PUT /_template/nested_test
{
"index_patterns": [ "nested-*" ],
"settings": { "index.mapping.coerce": false },
"mappings": {
"dynamic": "strict",
"properties" {
"vNested": {
"type": "nested",
"properties": {
"v1": { "type": "keyword" },
"v2": {
"properties": {
"v21": {
"type": long"
}
}
}
}
}
}
}
}
I will post two documents to an index that matches the template.
POST /nested-example/_doc
{
"vNested": [
{
"v1": "User1",
"v2": {
"v21": 1
}
},
{
"v1": "User3",
"v2": {
"v21": 3
}
}
]
}
POST /nested-example/_doc
{
"vNested": [
{
"v1": "User1",
"v2": {
"v21": 3
}
},
{
"v1": "User2",
"v2": {
"v21": 2
}
}
]
}
Now I will create a query with the goal of only getting the results of those documents, where there exists User1 with a corresponding v21 value of 3. As far as I understand, my nested mapping should ensure that I will only get the second document as query result.
The following query:
GET /nested-example/_search
{
"query" : {
"bool": {
"filter": {
"bool": {
"must": [
{
"nested: {
"path": "vNested",
"query": {
"match": {
"vNested.v1": "User1"
}
}
}
},
{
"nested: {
"path": "vNested",
"query": {
"match": {
"vNested.v2.v21": "3"
}
}
}
}
]
}
}
}
}
}
returns both documents, not only the single document that I expected
I understand that the query string is not the most elegant - this is due to some business logic + front-end framework logic in place for creating the query strings based on user input and any suggestions on how to remove redundancies there are welcome as well.
However I struggle to understand why does this query return both documents including the one where the vNested object with v1=User1, and v21=1. Shouldn't the nested mapping of the vNested field prevent just that issue?
You need to use bool/must query inside the nested query since you are querying on a single object and not on multiple objects. Modify your query as
{
"query": {
"bool": {
"filter": {
"bool": {
"must": [
{
"nested": {
"path": "vNested",
"query": {
"bool": {
"must": [
{
"match": {
"vNested.v1": "User1"
}
},
{
"match": {
"vNested.v2.v21": "3"
}
}
]
}
},
"inner_hits":{}
}
}
]
}
}
}
}
}
Search Result is
"hits": [
{
"_index": "nested-example",
"_type": "_doc",
"_id": "AAu0IXkBKyWl6Va6kmTU",
"_score": 0.0,
"_source": {
"vNested": [
{
"v1": "User1",
"v2": {
"v21": 3
}
},
{
"v1": "User2",
"v2": {
"v21": 2
}
}
]
},
"inner_hits": {
"vNested": {
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 1.6931472,
"hits": [
{
"_index": "nested-example",
"_type": "_doc",
"_id": "AAu0IXkBKyWl6Va6kmTU",
"_nested": {
"field": "vNested",
"offset": 0
},
"_score": 1.6931472,
"_source": {
"v1": "User1",
"v2": {
"v21": 3
}
}
}
]
}
}
}
}
]

How to search on multiple fields of array in elasticsearch

I have a index in elastic search called professor
If for cross field i need "AND" condition
for same field array i need to OR condition
I need to search subject which is Physics or Accounting this is array of fields(OR) statement
AND
I need to search type is Permanent or GUEST condition this is array of fields(OR) statement
AND
I need to search Location is NY(&) condition
test = [{'id':1,'name': 'A','subject': ['Maths','Accounting'],'type':'Contract', 'Location':'NY'},
{ 'id':2,'name': 'AB','subject': ['Physics','Engineering'],'type':'Permanent','Location':'NY'},
{'id':3,'name': 'ABC','subject': ['Maths','Engineering'],'type':'Permanent','Location':'NY'},
{'id':4,'name':'ABCD','subject': ['Physics','Engineering'],'type':['Contract','Guest'],'Location':'NY'}]
Query is below,3rd one got it, How to add 1 and 2
content_search = es.search(index="professor", body={
"query": {
"bool": {
"must": {
"match_all": {}
},
"filter": [
{
"term": {
"Location.keyword": "NY"
}
}
]
}
}
})
content_search ['hits']['hits']
Expected out is id [{ 'id':2,'name': 'AB','subject': ['Physics','Engineering'],'type':'Permanent','Location':'NY'},{'id':4,'name':'ABCD','subject': ['Physics','Engineering'],'type':['Contract','Guest'],'Location':'NY'}]
The filter clause (query) must appear in matching documents. However
unlike must the score of the query will be ignored. Filter clauses are
executed in filter context, meaning that scoring is ignored and
clauses are considered for caching.
Please go through this Elasticsearch documentation on bool queries, to get a detailed understanding about it.
Adding a working example with index data(same as that in question), search query, and search result
Search Query:
{
"query": {
"bool": {
"must": {
"match": {
"Location.keyword": "NY"
}
},
"filter": [
{
"bool": {
"should": [
{
"match": {
"subject.keyword": "Accounting"
}
},
{
"match": {
"subject.keyword": "Physics"
}
}
]
}
},
{
"bool": {
"should": [
{
"match": {
"type.keyword": "Permanent"
}
},
{
"match": {
"type.keyword": "Guest"
}
}
]
}
}
]
}
}
}
Search Result:
"hits": [
{
"_index": "stof_64370980",
"_type": "_doc",
"_id": "2",
"_score": 0.10536051,
"_source": {
"id": 2,
"name": "AB",
"subject": [
"Physics",
"Engineering"
],
"type": "Permanent",
"Location": "NY"
}
},
{
"_index": "stof_64370980",
"_type": "_doc",
"_id": "4",
"_score": 0.10536051,
"_source": {
"id": 4,
"name": "ABCD",
"subject": [
"Physics",
"Engineering"
],
"type": [
"Contract",
"Guest"
],
"Location": "NY"
}
}
]
Another Search Query:
You can even use terms query that returns documents that contain
one or more exact terms in a provided field.The terms query is the
same as the term query, except you can search for multiple values.
{
"query": {
"bool": {
"must": [
{
"terms": {
"subject.keyword": [
"Physics",
"Accounting"
]
}
},
{
"terms": {
"type.keyword": [
"Guest",
"Permanent"
]
}
},
{
"match": {
"Location.keyword": "NY"
}
}
]
}
}
}
Update 1:
{
"query": {
"bool": {
"must": [
{
"terms": {
"subject.keyword": [
"Physics",
"Accounting"
]
}
},
{
"terms": {
"type.keyword": [
"Guest",
"Permanent"
]
}
},
{
"match": {
"Location.keyword": "NY"
}
},
{
"query_string": {
"query": "ABCD"
}
}
]
}
}
}

How to use filter in match_all in elastic search

Query is below
{
"from" : 0,
"size" : 100,
"query": {
"match_all": {}
}
}
I need to filter from the match_all if name is test
i tried with
{
"from" : 0,
"size" : 100,
"query": {
"match_all": {}
},
"filter": [ "term": { "name": "test" }}]
}
I got error 'Unknown key for a START_ARRAY in [filter].')
You will need to wrap your query in a bool query , try out this search query:
{
"from":0,
"size":10,
"query": {
"bool": {
"must": {
"match_all": {}
},
"filter": [
{
"term": {
"grocery_name": "elastic"
}
}
]
}
}
}
Update 1:
According to the comment mentioned by #Nons
Search Query:
Terms query return documents that contain an exact term in a provided
field.
{
"from":0,
"size":10,
"query": {
"bool": {
"must": {
"match_all": {}
},
"filter": [
{
"term": {
"parentName.keyword": "Developer" <-- note this
}
}
]
}
}
}
Search Result:
"hits": [
{
"_index": "stof_64275684",
"_type": "_doc",
"_id": "1",
"_score": 1.0,
"_source": {
"id": "1",
"name": "A",
"parentName": "Developer",
"Data": [
{
"id": "455",
"name": "Google",
"lastUpdatedDate": "2020-09-10",
"parent_id": "1"
}
],
"Function": [
{
"id": "1",
"name": "Major"
}
]
}
}
]
You can even use a match query where the provided text is analyzed
before matching.
{
"from": 0,
"size": 10,
"query": {
"bool": {
"must": {
"match": {
"parentName": "developer"
}
}
}
}
}
I would recommend to use the Chrome ElasticSearch Head plugin. It allows to test and run searches against Elastic very easily (functionality is similar to MySql Workbech).
Please find example of usage of plugin below (combination of condition and aggregation).

Using a Kibana view query from application

I used the following filter and then searched for query string using Lucene to get the view that I was looking for.
{
"query": {
"match": {
"eventSource": {
"query": "ec2.amazonaws.com",
"type": "phrase"
}
}
}
}
I do not want to return event names those start with the word describe or get. Rest of the event names from ec2 event source should be returned.
!(eventName.keyword: Describe* OR eventName.keyword:
Get* )
The question is how to combine these 2 search requests into one?
I need to use that query from my application.
Update:
The Inspect menu of Kibana Discover tab generates this query. I am just trying to rewrite query_string part with usual match or match_phrase using boolean OR clause.
"query": {
"bool": {
"must": [
{
"query_string": {
"query": "!(eventName.keyword: Describe* OR eventName.keyword: Get* )",
"analyze_wildcard": true
}
},
{
"match_phrase": {
"eventSource": {
"query": "ec2.amazonaws.com"
}
}
},
{
"range": {
"#timestamp": {
"format": "strict_date_optional_time",
"gte": "2020-07-09T08:39:15.947Z",
"lte": "2020-07-24T08:39:15.947Z"
}
}
}
],
"filter": [],
"should": [],
"must_not": []
}
}
You can easily use the boolean query's must_not clause to exclude the documents which you don't want in your search result and you can add as many as must_not as you want, it's fairly easy to do and can be done in a single query.
Please refer the example in the same link to get more info. Created sample in my local to show your the correct query, Please note instead of wildcard I am using the prefix query which is better and server your use-case.
Create index mapping
{
"mappings": {
"properties": {
"eventName": {
"type": "keyword"
}
}
}
}
Index sample doc
{
"eventName" : "Describe the events"
}
{
"eventName" : "the Describe events"
}
{
"eventName" : "Get the event"
}
{
"eventName" : "event Get"
}
Now search query to get only 2 and 3rd doc according to your req
{
"query": {
"bool": {
"must_not": [
{
"prefix": {
"eventName": "Desc"
}
},
{
"prefix": {
"eventName": "Get"
}
}
]
}
}
}
Search result
"hits": [
{
"_index": "ngramkey",
"_type": "_doc",
"_id": "2",
"_score": 0.0,
"_source": {
"eventName": "the Describe events"
}
},
{
"_index": "ngramkey",
"_type": "_doc",
"_id": "4",
"_score": 0.0,
"_source": {
"eventName": "event Get"
}
}
]
As suggested by the user "Opster Elasticsearch Ninja", I have merged must not boolean query like this...
{
"query": {
"bool": {
"must": [
{
"bool": {
"must_not": [
{
"prefix": {
"eventName.keyword": "Desc"
}
},
{
"prefix": {
"eventName.keyword": "Get"
}
}
]
}
},
{
"match_phrase": {
"eventSource": {
"query": "ec2.amazonaws.com"
}
}
},
{
"range": {
"#timestamp": {
"format": "strict_date_optional_time",
"gte": "2020-07-09T08:39:15.947Z",
"lte": "2020-07-24T08:39:15.947Z"
}
}
}
],
"filter": [],
"should": [],
"must_not": []
}
}
}

Nested filtering in elasticsearch with more than one term of the same nested type

I'm new to elasticsearch, so maybe my approach is plain wrong, but I want to make an index of recipes and allow the user to filter it down with the aggregated ingredients that are still found in the subset.
Maybe I'm using the wrong language to explain so maybe this example will clarify. I would like to search for recipes with the term salt; which results in three recipes:
with ingredients: salt, flour, water
with ingredients: salt, pepper, egg
with ingredients: water, flour, egg, salt
The aggregate on the results ingredients returns salt, flour, water, pepper, egg. When I filter with flour I only want recipe 1 and 3 to appear in the search results (and the aggregate on ingredients should only return salt, flour, water, egg and salt). When I add another filter egg I want only recipe 3 to appear (and the aggregate should only return water, flour, egg, salt).
I can't make the latter to work: one filter next to the default query does narrow down the results as desired but when adding the other term (egg) to the terms filter the results again start to include b as well, as if it were an OR filter. Adding AND however to the filter execution results in NO results ... what am I doing wrong?
My mapping:
{
"recipe": {
"properties": {
"title": {
"analyzer": "dutch",
"type": "string"
},
"ingredients": {
"type": "nested",
"properties": {
"name": {
"type": "string",
"analyzer": "dutch",
"include_in_parent": true,
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
}
}
}
My query:
{
"query": {
"filtered": {
"query": {
"bool": {
"should": [
{
"match": {
"_all": "salt"
}
}
]
}
},
"filter": {
"nested": {
"path": "ingredients",
"filter": {
"terms": {
"ingredients.name": [
"flour",
"egg"
],
"execution": "and"
}
}
}
}
}
},
"size": 50,
"aggregations": {
"ingredients": {
"nested": {
"path": "ingredients"
},
"aggregations": {
"count": {
"terms": {
"field": "ingredients.name.raw"
}
}
}
}
}
}
Why are you using a nested mapping here? Its main purpose is to keep relations between the sub-object attributes, but your ingredients field has just one attribute and can be modeled simply as a string field.
So, if you update your mapping like this :
POST recipes
{
"mappings": {
"recipe": {
"properties": {
"title": {
"type": "string"
},
"ingredients": {
"name": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
}
}
}
You can still index your recipes as :
{
"title":"recipe b",
"ingredients":["salt","pepper","egg"]
}
And this query gives you the result you are waiting for :
POST recipes/recipe/_search
{
"query": {
"filtered": {
"query": {
"match": {
"_all": "salt"
}
},
"filter": {
"terms": {
"ingredients": [
"flour",
"egg"
],
"execution": "and"
}
}
}
},
"size": 50,
"aggregations": {
"ingredients": {
"terms": {
"field": "ingredients"
}
}
}
}
which is :
{
...
"hits": {
"total": 1,
"max_score": 0.22295055,
"hits": [
{
"_index": "recipes",
"_type": "recipe",
"_id": "PP195TTsSOy-5OweArNsvA",
"_score": 0.22295055,
"_source": {
"title": "recipe c",
"ingredients": [
"salt",
"flour",
"egg",
"water"
]
}
}
]
},
"aggregations": {
"ingredients": {
"buckets": [
{
"key": "egg",
"doc_count": 1
},
{
"key": "flour",
"doc_count": 1
},
{
"key": "salt",
"doc_count": 1
},
{
"key": "water",
"doc_count": 1
}
]
}
}
}
Hope this helps.

Resources