I need to find all documents, that contain given id from a list and have no field "device_data".
My search query:
{
"query": {
"bool" : {
"must" : [
{
"terms" : {
"id" : [
"1cbe0c01-6e0c-11e8-b79f-097b2a39b616"
]
}
}
],
"must_not" : [
{
"exists" : {
"field" : "device_data"
}
}
]
}
}
}
Still returns this document, where i expect it not to be found as "device_data" is present. What am I doing wrong?
{
"took": 6,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 4.9881625,
"hits": [
{
"_index": "iot_data",
"_type": "sensors_by_id",
"_id": "[\"1cbe0c01-6e0c-11e8-b79f-097b2a39b616\",\"1cbe0c00-6e0c-11e8-b79f-097b2a39b616\"]",
"_score": 4.9881625,
"_source": {
"field_id": "123",
"device_data": {
"comm_nr": "xxxx1",
"id": "542b9010-67b6-11e8-ab71-997fe8a668b8",
"tag": "",
"type": ""
},
"groups": "group-test",
"id": "1cbe0c01-6e0c-11e8-b79f-097b2a39b616",
"time": "1cbe0c00-6e0c-11e8-b79f-097b2a39b616",
"username": "group-test"
}
}
]
}
}
You need to use a terminal field, such as device_data.id for instance:
"must_not" : [
{
"nested": {
"path": "device_data",
"query": {
"exists" : {
"field" : "device_data.id"
}
}
}
}
]
Related
I am new to ElasticSearch and can't quite figure out what I want is possible or not.
I can query like this:
GET entity/_search
{
"query": {
"bool": {
"must": [
{ "match": { "searchField": "searchValue" }}
]
}
},
"aggs" : {
"uniq_Id" : {
"terms" : { "field" : "Id", "size":500 }
}
}
}
and it will return top search results and the term aggregation buckets. But ideally what I would like for the search results to return, is only one (perhaps the top one, does not matter) for each of unique Id's defined in the aggregation terms.
You can make use of Terms Aggregation along with the Top Hits Aggregation to give you the result you are looking for.
Now once you do that, specify the size as 1 in the Top Hits Aggregation
Based on your query I've created sample mapping,documents, aggregation query and the response for your reference.
Mapping:
PUT mysampleindex
{
"mappings": {
"mydocs": {
"properties": {
"searchField":{
"type": "text"
},
"Id": {
"type": "keyword"
}
}
}
}
}
Sample Documents:
POST mysampleindex/mydocs/1
{
"searchField": "elasticsearch",
"Id": "1000"
}
POST mysampleindex/mydocs/2
{
"searchField": "elasticsearch is awesome",
"Id": "1000"
}
POST mysampleindex/mydocs/3
{
"searchField": "elasticsearch is awesome",
"Id": "1001"
}
POST mysampleindex/mydocs/4
{
"searchField": "elasticsearch is pretty cool",
"Id": "1001"
}
POST mysampleindex/mydocs/5
{
"searchField": "elasticsearch is pretty cool",
"Id": "1002"
}
Query:
POST mysampleindex/_search
{
"size": 0,
"query": {
"bool": {
"must": [
{
"match": {
"searchField": "elasticsearch"
}
}
]
}
},
"aggs": {
"myUniqueIds": {
"terms": {
"field": "Id",
"size": 10
},
"aggs": {
"myDocs": {
"top_hits": { <---- Top Hits Aggregation
"size": 1 <---- Note this
}
}
}
}
}
}
Sample Response:
{
"took": 7,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 5,
"max_score": 0,
"hits": []
},
"aggregations": {
"myUniqueIds": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "1000",
"doc_count": 2,
"myDocs": {
"hits": {
"total": 2,
"max_score": 0.2876821,
"hits": [
{
"_index": "mysampleindex",
"_type": "mydocs",
"_id": "1",
"_score": 0.2876821,
"_source": {
"searchField": "elasticsearch",
"Id": "1000"
}
}
]
}
}
},
{
"key": "1001",
"doc_count": 2,
"myDocs": {
"hits": {
"total": 2,
"max_score": 0.25316024,
"hits": [
{
"_index": "mysampleindex",
"_type": "mydocs",
"_id": "3",
"_score": 0.25316024,
"_source": {
"searchField": "elasticsearch is awesome",
"Id": "1001"
}
}
]
}
}
},
{
"key": "1002",
"doc_count": 1,
"myDocs": {
"hits": {
"total": 1,
"max_score": 0.2876821,
"hits": [
{
"_index": "mysampleindex",
"_type": "mydocs",
"_id": "5",
"_score": 0.2876821,
"_source": {
"searchField": "elasticsearch is pretty cool",
"Id": "1002"
}
}
]
}
}
}
]
}
}
}
Notice that I am not returning any bool results in the above, the search result you are looking for comes in the form of Top Hits Aggregation.
Hope this helps!
Here I need to fetch the key, value pair of particular index based on the some conditions.
I need only few fields in that index doc not all fields
I need only fields which i given in input.
GET /_search?&pretty=true&size=3
{
"query": {
"query_string": {
"query": "countryCode:SA AND serviceName:SMS",
"fields": ["level","cause","to"]
}
}
}
Output:
{
"took": 2854,
"timed_out": false,
"num_reduce_phases": 4,
"_shards": {
"total": 1891,
"successful": 1891,
"failed": 0
},
"hits": {
"total": 14032,
"max_score": 8.429943,
"hits": [
{
"_index": "postman-2019.01.21",
"_type": "syslog",
"_id": "AWhvN1KDl97BCeGFfgpe",
"_score": 8.429943,
"_source": {
"eId": "346589962",
"level": "info",
"prevStatus": "SUCCESS",
"cause": "SUCCESS",
"serviceName": "SMS",
"loggingAction": "SMS_CALLBACK_REPORTS",
"application": "POSTMAN",
"countryCode": "SA",
"client": "CRS",
"to": "+966572444531",
"externalServiceName": "gupshupInternationalChannelA",
"time": "Mon Jan 21 07:02:02 UTC 2019",
"category": "OTP",
"dId": "3762647059352507724-309778596902014991",
"uIdentifier": "2e262115-d09c-4bef-a04b-d0860d064930",
"epochTime": 1548054122491,
"status": "DELIVERED",
"#version": "1",
"#timestamp": "2019-01-21T07:02:03.124Z",
"path": "/mnt/logs/logstash",
"host": "ip-10-20-10-164",
"type": "syslog"
}
}
]
}
}
this is output i got, but i need only fields which i given in input
Here is what you can try :
POST cars/_search
{
"_source": {
"includes": [ "price_eur", "stk_year" ],
"excludes": [ "mileage" ]
},
"query" : {
"match" : { "maker" : "audi" }
}
}
POST cars/_search?&pretty=true&size=3
{
"_source": {
"includes": [ "price_eur", "stk_year" ],
"excludes": [ "mileage" ]
},
"query" : {
"match" : { "maker" : "audi" }
}
}
I am trying to do a search within elasticsearch using the regexp filters. Following is my query:
{
"from": 0,
"size": 10,
"_source":["CODE"],
"query": {
"bool": {
"must": [
{
"bool": {
"should": [
{
"regexp" : {
"CODE" : {
"value" : "[0]?[0]?[0]?[0]?3410086456[0-9]?",
"flags_value" : 0,
"boost" : 20.0
}
}
},
{
"regexp" : {
"CODE" : {
"value" : "[0]?[0]?[0]?[0]?83560900204[0-9]?",
"flags_value" : 0,
"boost" : 20.0
}
}
}
]
}
},
{
"terms": {
"CODETYPE": [
"TYPE1", "TYPE2", "TYPE3"
]
}
}
]
}
}
}
Below is the result of the query:
{
"took": 5,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 2,
"max_score": 20.091797,
"hits": [
{
"_index": "index1",
"_type": "type1",
"_id": "142242",
"_score": 20.091797,
"_source": {
"CODE": "003410086456"
}
},
{
"_index": "index1",
"_type": "type1",
"_id": "375897",
"_score": 20.091797,
"_source": {
"CODE": "083560900204"
}
}
]
}
}
What I need to get additionally in my output is the input term against which each result has matched. Something like this:
{
"took": 5,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 2,
"max_score": 20.091797,
"hits": [
{
"_index": "index1",
"_type": "type1",
"_id": "142242",
"_score": 20.091797,
"_source": {
"CODE": "003410086456",
"INPUT": "3410086456"
}
},
{
"_index": "index1",
"_type": "type1",
"_id": "375897",
"_score": 20.091797,
"_source": {
"CODE": "083560900204",
"INPUT": "83560900204"
}
}
]
}
}
Notice the additional INPUT field above. That way I can map what pattern has mapped to which result. Is there any possibility in elasticsearch I can do this? I am currently unable to find any way of achieving this.
Appreciate your help on this. Let me know if I need to furnish any more information.
you could use highlighting, though it won't in _source, it would create a separate field highlight which gives the field value.
{
"from": 0,
"size": 10,
"_source": [
"CODE"
],
"query": {
"bool": {
"must": [
{
"bool": {
"should": [
{
"regexp": {
"CODE": {
"value": "[0]?[0]?[0]?[0]?3410086456[0-9]?",
"flags_value": 0,
"boost": 20
}
}
},
{
"regexp": {
"CODE": {
"value": "[0]?[0]?[0]?[0]?83560900204[0-9]?",
"flags_value": 0,
"boost": 20
}
}
}
]
}
},
{
"terms": {
"CODETYPE": [
"TYPE1",
"TYPE2",
"TYPE3"
]
}
}
]
}
},
"highlight": {
"fields": {
"CODE": {}
}
}
}
Refer: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-highlighting.html#search-request-highlighting
I have created a elasticsearch query with function score and top_hit. This query will remove the duplicate and return top 1 record for each bucket.
GET employeeid/info/_search
{"size": 0,
"query" : {
"function_score" : {
"query" : {
"match" : {
"employeeID" : "23141A"
}
},
"functions" : [{
"linear" : {
"AcquiredDate" : {
"scale" : "90d",
"decay" : 0.5
}
}
}, {
"filter" : {
"match" : {
"name" : "sorna"
}
},
"boost_factor" : 10
}, {
"filter" : {
"match" : {
"name" : "lingam"
}
},
"boost_factor" : 7
}
],
"boost_mode" : "replace"
}
},
"aggs": {
"duplicateCount": {
"terms": {
"field": "employeehash",
"min_doc_count": 1
},
"aggs": {
"duplicateDocuments": {
"top_hits": {
"size":1
}
}
}
}
}
}
I am getting the expected result, But the problem is i want to sort the result using _score.
Following is my simple o/p
{
"key": "567",
"doc_count": 2,
"duplicateDocuments": {
"hits": {
"total": 2,
"max_score": 0.40220365,
"hits": [
{
"_index": "employeeid",
"_type": "info",
"_id": "5",
"_score": 0.40220365,
"_source": {
"name": "John",
"organisation": "google",
"employeeID": "23141A",
"employeehash": "567",
"AcquiredDate": "2016-02-01T07:57:28Z"
}
}
]
}
}
},
{
"key": "102",
"doc_count": 1,
"duplicateDocuments": {
"hits": {
"total": 1,
"max_score": 2.8154256,
"hits": [
{
"_index": "employeeid",
"_type": "info",
"_id": "8",
"_score": 2.8154256,
"_source": {
"name": "lingam",
"organisation": "google",
"employeeID": "23141A",
"employeehash": "102",
"AcquiredDate": "2016-02-01T07:57:28Z"
}
}
]
}
}
}
Question: How to sort _score : desc ?
i have not enabled groovy so i can not use script
need your help to understand the behaviour of elasticsearch scripting based sorting.
First of all let me paste the mappings of my elasticsearch types :
{
"nestedDateType" : {
"properties" : {
"message" : {
"properties" : {
"date" : {
"type" : "date",
"format" : "dateOptionalTime"
}
}
}
}
},
"nonNestedDateType" : {
"properties" : {
"date" : {
"type" : "date",
"format" : "dateOptionalTime"
}
}
}
}
now what I want to do is to query these 2 types and sort based on the date.
The problem is in nestedDateType, the date path is "message.date" where in nonNestedDateType, the date path is "date".
I understand that I have to use scripting based sort to do this. However, the script that I made did not work as expected. This is the query that I tried:
POST http://locahost:9200/index/nonNestedDateType,nestedDateType/_search?size=5000
{
"query": {
"filtered": {
"filter": {
"bool": {
"must": [
{
"or": [
{
"range": {
"date": {
"gte": "2015-04-01"
}
}
},
{
"range": {
"message.date": {
"gte": "2015-04-01"
}
}
}
]
}
]
}
}
}
},
"sort": {
"_script": {
"script": "doc.containsKey('message') ? doc.message.date.value : doc.date.value",
"type": "number",
"order": "desc"
}
}
}
and these were the result that I got :
{
"took": 60,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 15,
"max_score": null,
"hits": [
{
"_index": "***",
"_type": "nonNestedDateType",
"_id": "***",
"_score": null,
"_source": {
"docId": "***",
"date": 1461634484557
},
"sort": [
1461634484557
]
},
{
"_index": "***",
"_type": "nonNestedDateType",
"_id": "***",
"_score": null,
"_source": {
"docId": "***",
"date": 1461634483528
},
"sort": [
1461634483528
]
},
{
"_index": "***",
"_type": "nestedDateType",
"_id": "***",
"_score": null,
"_source": {
"docId": "***",
"message": {
"date": 1461548078310
}
},
"sort": [
0
]
}
]
}
}
as you can see from the last result of the type nestedDateType, I was expecting the sort = 1461548078310 instead of 0. Could anyone explains to me what I was doing wrong?
noted that some fields have been removed for confidentiality.
i can finally make it works by changing
script": "doc.containsKey('message') ? doc.message.date.value : doc.date.value"
into
script": "doc.date.value == 0 ? doc['message.date'].value : doc.date.value"
still curious though why doc.containsKey('message') never return true