ES multiple must query returns wrong results - elasticsearch

{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "categories",
"query": {
"terms": {
"categories.id": [
9
]
}
}
}
},
{
"terms": {
"deleted": [
false
]
}
},
{
"terms": {
"published": [
true
]
}
},
{
"nested": {
"path": "vendors",
"query": {
"terms": {
"vendors.isDeletedVendor": [
false
]
}
}
}
},
{
"nested": {
"path": "vendors",
"query": {
"terms": {
"vendors.isPublishedVendor": [
true
]
}
}
}
}
],
"should": [
{
"bool": {
"must": [
{
"bool": {
"should": [
{
"bool": {
"must_not": [
{
"nested": {
"path": "manufacturers",
"query": {
"exists": {
"field": "manufacturers"
}
}
}
}
]
}
}
]
}
},
{
"bool": {
"should": [
{
"nested": {
"path": "vendors",
"query": {
"terms": {
"vendors.id": [
87
]
}
}
}
}
]
}
}
]
}
},
{
"bool": {
"must": [
{
"bool": {
"should": [
{
"bool": {
"must_not": [
{
"nested": {
"path": "manufacturers",
"query": {
"exists": {
"field": "manufacturers"
}
}
}
}
]
}
}
]
}
},
{
"bool": {
"should": [
{
"nested": {
"path": "vendors",
"query": {
"terms": {
"vendors.id": [
56
]
}
}
}
}
]
}
}
]
}
}
]
}
},
"size": 1000,
"_source": {
"includes": [
"modelName",
"vendors.id",
"manufacturers.id",
"id"
]
}
}
So what I am doing here is something like this:
Select * from data where
(manufacturerIds is null and vendorIds in 87) or
(manufacturerIds is null and vendorIds in 56)
But it is returning results like this:
"hits" : [
{
"_index" : "onoff-live",
"_type" : "_doc",
"_id" : "130011",
"_score" : 5.0,
"_source" : {
"modelName" : "Galaxy A20",
"manufacturers" : [
{
"id" : 216
}
],
"id" : 130011,
"vendors" : [
{
"id" : 23
}
]
}
}]
I don't know what I am missing, please give me suggestions, if you want mapping I will post it in reply...

Related

ElasticSearch - combining search queries not working

I would like to have an intersection of 2 queries
I got 3 documents in the index:
"_id": "68c220aa-ea51-4f84-b880-29af3302cae9",
"_id": "b6c1c3c5-e959-480f-a145-f5598fafea66",
"_id": "2d30de72-0a2b-465c-8770-970ad9760d47",
Query1:
{
"from": 0,
"query": {
"nested": {
"path": "attributes",
"query": {
"bool": {
"must": [
{
"bool": {
"must": [
{
"match_phrase": {
"attributes.asReference": {
"query": "8670ff39-6a0d-4ae8-e217-08d88efd4771"
}
}
},
{
"match_phrase": {
"attributes.attributeId": {
"query": "f51ca670-4223-4ea2-8007-d111dd38a14f"
}
}
}
]
}
}
]
}
}
}
},
"size": 10,
"sort": [
{
"modified": {
"order": "asc"
}
},
{
"created": {
"order": "asc"
}
}
]
}
returns all 3 documents as it should
"_id": "68c220aa-ea51-4f84-b880-29af3302cae9",
"_id": "b6c1c3c5-e959-480f-a145-f5598fafea66",
"_id": "2d30de72-0a2b-465c-8770-970ad9760d47",
Then I do query2:
{
"from": 0,
"query": {
"nested": {
"path": "attributes",
"query": {
"bool": {
"must": [
{
"bool": {
"must": [
{
"match_phrase": {
"attributes.asShortString": {
"query": "RA-005"
}
}
},
{
"match_phrase": {
"attributes.attributeId": {
"query": "7ff3dbc1-3586-4475-9162-5430bb06c6d0"
}
}
}
]
}
}
]
}
}
}
},
"size": 10,
"sort": [
{
"modified": {
"order": "asc"
}
},
{
"created": {
"order": "asc"
}
}
]
}
returns 1 document:
"_id": "b6c1c3c5-e959-480f-a145-f5598fafea66"
But when I combine the queries to:
{
"from": 0,
"query": {
"nested": {
"path": "attributes",
"query": {
"bool": {
"must": [
{
"bool": {
"must": [
{
"match_phrase": {
"attributes.asReference": {
"query": "8670ff39-6a0d-4ae8-e217-08d88efd4771"
}
}
},
{
"match_phrase": {
"attributes.attributeId": {
"query": "f51ca670-4223-4ea2-8007-d111dd38a14f"
}
}
}
]
}
},
{
"bool": {
"must": [
{
"match_phrase": {
"attributes.asShortString": {
"query": "RA-005"
}
}
},
{
"match_phrase": {
"attributes.attributeId": {
"query": "7ff3dbc1-3586-4475-9162-5430bb06c6d0"
}
}
}
]
}
}
]
}
}
}
},
"size": 10,
"sort": [
{
"modified": {
"order": "asc"
}
},
{
"created": {
"order": "asc"
}
}
]
}
Here I do not get any documents
So the subqueries are working but combined it does not work (it produces 0 results)
What am I missing here?
Due to the way nested documents and queries work, you need to have two separate nested queries in your bool/must query, because each will/might match a different nested document of the same parent document:
{
"from": 0,
"query": {
"bool": {
"must": [
{
"nested": {
"path": "attributes",
"query": {
"bool": {
"must": [
{
"bool": {
"must": [
{
"match_phrase": {
"attributes.asReference": {
"query": "8670ff39-6a0d-4ae8-e217-08d88efd4771"
}
}
},
{
"match_phrase": {
"attributes.attributeId": {
"query": "f51ca670-4223-4ea2-8007-d111dd38a14f"
}
}
}
]
}
}
]
}
}
}
},
{
"nested": {
"path": "attributes",
"query": {
"bool": {
"must": [
{
"bool": {
"must": [
{
"match_phrase": {
"attributes.asShortString": {
"query": "RA-005"
}
}
},
{
"match_phrase": {
"attributes.attributeId": {
"query": "7ff3dbc1-3586-4475-9162-5430bb06c6d0"
}
}
}
]
}
}
]
}
}
}
}
]
}
},
"size": 10,
"sort": [
{
"modified": {
"order": "asc"
}
},
{
"created": {
"order": "asc"
}
}
]
}

How to query elasticsearch for a specific kibana visualization data?

In our cluster's kibana dashboard, I see a visualization which gives me the total count of incoming traffic to an application. What I want is to get the same incoming traffic count using a curl call so that I can automate some reporting. To do so, first I inspect the visualization and click on request, below is what I got
{
"aggs": {},
"size": 0,
"_source": {
"excludes": []
},
"stored_fields": [
"*"
],
"script_fields": {},
"docvalue_fields": [
{
"field": "#timestamp",
"format": "date_time"
},
{
"field": "time",
"format": "date_time"
}
],
"query": {
"bool": {
"must": [],
"filter": [
{
"bool": {
"filter": [
{
"bool": {
"must_not": {
"bool": {
"should": [
{
"query_string": {
"fields": [
"remote_addr"
],
"query": "\\1\\0\\.\\0\\.*"
}
}
],
"minimum_should_match": 1
}
}
}
},
{
"bool": {
"filter": [
{
"bool": {
"must_not": {
"bool": {
"should": [
{
"query_string": {
"fields": [
"remote_addr"
],
"query": "\\1\\0\\0\\.\\0\\.*"
}
}
],
"minimum_should_match": 1
}
}
}
},
{
"bool": {
"filter": [
{
"bool": {
"must_not": {
"bool": {
"should": [
{
"match_phrase": {
"upstream_addr.keyword": “IP_ADDR:PORT”
}
}
],
"minimum_should_match": 1
}
}
}
},
{
"bool": {
"filter": [
{
"bool": {
"must_not": {
"bool": {
"should": [
{
"match_phrase": {
"upstream_addr.keyword": “IP_ADDR:PORT”
}
}
],
"minimum_should_match": 1
}
}
}
},
{
"bool": {
"filter": [
{
"bool": {
"must_not": {
"bool": {
"should": [
{
"match_phrase": {
"upstream_addr.keyword": “IP_ADDR:PORT”
}
}
],
"minimum_should_match": 1
}
}
}
},
{
"bool": {
"must_not": {
"bool": {
"should": [
{
"match_phrase": {
"upstream_addr.keyword": “IP_ADDR:PORT”
}
}
],
"minimum_should_match": 1
}
}
}
}
]
}
}
]
}
}
]
}
}
]
}
}
]
}
},
{
"match_all": {}
},
{
"match_phrase": {
"kubernetes.labels.app.keyword": {
"query": "kong"
}
}
},
{
"exists": {
"field": "status"
}
},
{
"range": {
"#timestamp": {
"format": "strict_date_optional_time",
"gte": "2021-01-05T09:32:46.946Z",
"lte": "2021-01-05T09:47:46.946Z"
}
}
}
],
"should": [],
"must_not": [
{
"bool": {
"should": [
{
"match_phrase": {
"http_user_agent": "CloudWatchSynthetics"
}
},
{
"match_phrase": {
"http_user_agent": "Amazon-Route53-Health-Check-Service"
}
}
],
"minimum_should_match": 1
}
}
]
}
}
}
Now, I took this request body, and made a curl call to elasticsearch like below
curl -u elastic:password -x GET "localhost:9200/_mget?pretty" -H 'Content-Type: application/json' -d'
<request_body_that_I_have_pasted_above>
'
But, this throws below error
{
"error" : {
"root_cause" : [
{
"type" : "parsing_exception",
"reason" : "unexpected token [START_OBJECT], expected [FIELD_NAME] or [START_ARRAY]",
"line" : 3,
"col" : 11
}
],
"type" : "parsing_exception",
"reason" : "unexpected token [START_OBJECT], expected [FIELD_NAME] or [START_ARRAY]",
"line" : 3,
"col" : 11
},
"status" : 400
}
Is my approach right? what am I doing wrong here?

Adding a condition to query

I have a query that works grate. How ever I need to add one condition.
I want to get a documents that also have the field "marked" :"true"
This is my query.
{
"from": 0,
"size": 100,
"min_score": 0.6,
"query": {
"bool": {
"should": [
{ "multi_match" : {
"fields" : ["_all"],
"query" : " Test " ,
"fuzziness" : "1.5" ,
"prefix_length" : "2"
}
}
],
"must": { "bool": { "must": [
{ "terms": { "language.id":["1"] }},
{ "term": { "forbidden":"false" }}
]
}}}
}, "sort": [{ "_score": { "order": "desc"}} ]
}
I have been trying ti add a should clause after both must clauses.
Where can I put this condition?
try this
{
"query": {
"bool": {
"should": [
{
"multi_match": {
"fields": [
"_all"
],
"query": " Test ",
"fuzziness": "1.5",
"prefix_length": "2"
}
}
],
"must": [
{
"terms": {
"language.id": [
"1"
]
}
},
{
"term": {
"forbidden": "false"
}
},
{
"term": {
"marked": "true"
}
}
]
}
}
}

Elasticsearch 2.x - new bool query

After upgrading to Elasticsearch 2.x I got an issue with the following query:
{ "query": {
"filtered": {
"filter": {
"bool": {
"should": [
{
"bool": {
"must": [
{
"terms": {
"_type": [
"xxx",
"yyy"
]
}
},
{
"exists": {
"field": "aaa"
}
},
{
"exists": {
"field": "bbb"
}
},
{
"exists": {
"field": "ccc"
}
}
]
}
},
{
"bool": {
"must": [
{
"term": {
"_type": "eee"
}
},
{
"term": {
"f": 0
}
}
]
}
}
]
}
}
} } }
Basically, I do not know how to replace the 'must' inside the 'should' filter with the new query DSL rules in Elasticsearch 2.x.
Thanks in advance.
You can simply remove the filtered/filter part and modify your query like this:
{
"query": {
"bool": {
"should": [
{
"bool": {
"must": [
{
"terms": {
"_type": [
"xxx",
"yyy"
]
}
},
{
"exists": {
"field": "aaa"
}
},
{
"exists": {
"field": "bbb"
}
},
{
"exists": {
"field": "ccc"
}
}
]
}
},
{
"bool": {
"must": [
{
"term": {
"_type": "eee"
}
},
{
"term": {
"f": 0
}
}
]
}
}
]
}
}
}

Boosting query is not working properly

{
"sort": [
{
"is_active": "asc"
}
],
"fields": [
"is_job_seeking", "is_active"
],
"query": {
"bool": {
"must": [
{
"bool": {
"must": {
"term": {
"is_job_seeking": 1
}
}
}
}
]
}
}
}
this query return me all document which has is_job_seeking=1, and is_active=0 and is_active=1 and that's fine, now when I want to boost score for document which has is_active=1 I have add boosting like
{
"sort": [
{
"is_active": "asc"
}
],
"fields": [
"is_job_seeking", "is_active"
],
"query": {
"bool": {
"must": [
{
"bool": {
"must": {
"term": {
"is_job_seeking": 1
}
}
}
},
{
"boosting": {
"positive": {
"term": {
"is_active": 1
}
},
"negative": {
"term": {
"is_active": 0
}
},
"negative_boost": 0.3
}
}
]
}
}
}
but this give me results only with is_active=1
Try this:
{
"sort": [
{
"is_active": "asc"
}
],
"fields": [
"is_job_seeking",
"is_active"
],
"query": {
"bool": {
"must": [
{
"bool": {
"must": {
"term": {
"is_job_seeking": 1
}
}
}
}
],
"should": [
{
"boosting": {
"positive": {
"term": {
"is_active": 1
}
},
"negative": {
"term": {
"is_active": 0
}
},
"negative_boost": 0.3
}
}
]
}
}
}

Resources