ElasticSearch 5.2 Unknown key for a START_OBJECT in [script]. request - elasticsearch

Hello I am trying to remove record by searching in nested data of array via script.
Is possible to remove data with using script for _delete_by_query ?
Version of elasticsearch 5.2
My request looks like
POST /test_index/_delete_by_query
{
"query": {
"bool": {
"must": [
{
"exists": {
"field":"userPermission"
}
}
]
}
},
"script":{
"inline":"""
for (int i = 0; i < ctx._source.userPermission.size(); i++) {
if(ctx._source.userPermission[i].id == '760100000-100000')
{
return true
}
}
return false
"""
}
}
I get an error:
{
"error": {
"type": "parsing_exception",
"reason": "Unknown key for a START_OBJECT in [script].",
"line": 1,
"col": 77
},
"status": 400
}
It is example of data:
{
"_index": "test_index",
"_type": "doc",
"_id": "AXXDZFKKgDFBfUY9kVS6",
"_score": 1,
"_source": {
"test_field": "Test",
"userPermission": [
{
"fullName": "Test 55",
"id": '760100000-100000'
},
{
"fullName": "Test33",
"id": 555
},
{
"fullName": "Test 1",
"id": 444
}
]
}
}

The delete by query endpoint doesn't support any script content. What you need to do is to use the update by query endpoint and the delete operation if the condition is satisfied:
POST /test_index/_update_by_query
{
"query": {
"bool": {
"must": [
{
"exists": {
"field":"userPermission"
}
}
]
}
},
"script":{
"inline":"""
for (int i = 0; i < ctx._source.userPermission.size(); i++) {
if(ctx._source.userPermission[i].id == '760100000-100000')
{
ctx.op = 'delete';
}
}
"""
}
}

Related

Elasticsearch :: Unknown key for a START_OBJECT in [runtime_mappings]

I want to get the sum of run_time variant, by using runtime_mapping I got mistakes.
I know that it'a new feature in ES 7.12, but I can't upgrade it because it's not depend on me.
So how could I get the answer without runtime_mapping?
GET log/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"username": "xxx"
}
},{
"range": {
"time_end": {
"gte": "2021-12-01 11:40:06",
"lte": "2021-12-12 11:40:06"
}
}
}
]
}
}
, "_source": ["username", "time_start","time_submit", "time_end", "gpus_alloc"]
,
"runtime_mappings": {
"exec_time.weighted": {
"type": "long",
"script": """
long exec_time;
long timediff = doc['time_end'].date.getMillis() - doc['time_start'].date.getMillis();
String gpus = doc['gpus_alloc.keyword'].value;
int idx = gpus.lastIndexOf(':');
if(idx != -1)
exectime = timediff * Integer.parseInt(gpus.substring(idx+1));
else
exectime = 0;
emit(exectime);
"""
}
},
"aggs": {
"U1": {
"sum": {
"field": "exec_time.weighted"
}
}
}
}
the Error information is as follow:
{
"error": {
"root_cause": [
{
"type": "parsing_exception",
"reason": "Unknown key for a START_OBJECT in [runtime_mappings].",
"line": 22,
"col": 23
}
],
"type": "parsing_exception",
"reason": "Unknown key for a START_OBJECT in [runtime_mappings].",
"line": 22,
"col": 23
},
"status": 400
}

Query for nested fields returns results as if there was no nested mapping

I am having difficulties understanding, why a query across nested fields is returning unexpected results.
I have the following template for my index
PUT /_template/nested_test
{
"index_patterns": [ "nested-*" ],
"settings": { "index.mapping.coerce": false },
"mappings": {
"dynamic": "strict",
"properties" {
"vNested": {
"type": "nested",
"properties": {
"v1": { "type": "keyword" },
"v2": {
"properties": {
"v21": {
"type": long"
}
}
}
}
}
}
}
}
I will post two documents to an index that matches the template.
POST /nested-example/_doc
{
"vNested": [
{
"v1": "User1",
"v2": {
"v21": 1
}
},
{
"v1": "User3",
"v2": {
"v21": 3
}
}
]
}
POST /nested-example/_doc
{
"vNested": [
{
"v1": "User1",
"v2": {
"v21": 3
}
},
{
"v1": "User2",
"v2": {
"v21": 2
}
}
]
}
Now I will create a query with the goal of only getting the results of those documents, where there exists User1 with a corresponding v21 value of 3. As far as I understand, my nested mapping should ensure that I will only get the second document as query result.
The following query:
GET /nested-example/_search
{
"query" : {
"bool": {
"filter": {
"bool": {
"must": [
{
"nested: {
"path": "vNested",
"query": {
"match": {
"vNested.v1": "User1"
}
}
}
},
{
"nested: {
"path": "vNested",
"query": {
"match": {
"vNested.v2.v21": "3"
}
}
}
}
]
}
}
}
}
}
returns both documents, not only the single document that I expected
I understand that the query string is not the most elegant - this is due to some business logic + front-end framework logic in place for creating the query strings based on user input and any suggestions on how to remove redundancies there are welcome as well.
However I struggle to understand why does this query return both documents including the one where the vNested object with v1=User1, and v21=1. Shouldn't the nested mapping of the vNested field prevent just that issue?
You need to use bool/must query inside the nested query since you are querying on a single object and not on multiple objects. Modify your query as
{
"query": {
"bool": {
"filter": {
"bool": {
"must": [
{
"nested": {
"path": "vNested",
"query": {
"bool": {
"must": [
{
"match": {
"vNested.v1": "User1"
}
},
{
"match": {
"vNested.v2.v21": "3"
}
}
]
}
},
"inner_hits":{}
}
}
]
}
}
}
}
}
Search Result is
"hits": [
{
"_index": "nested-example",
"_type": "_doc",
"_id": "AAu0IXkBKyWl6Va6kmTU",
"_score": 0.0,
"_source": {
"vNested": [
{
"v1": "User1",
"v2": {
"v21": 3
}
},
{
"v1": "User2",
"v2": {
"v21": 2
}
}
]
},
"inner_hits": {
"vNested": {
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 1.6931472,
"hits": [
{
"_index": "nested-example",
"_type": "_doc",
"_id": "AAu0IXkBKyWl6Va6kmTU",
"_nested": {
"field": "vNested",
"offset": 0
},
"_score": 1.6931472,
"_source": {
"v1": "User1",
"v2": {
"v21": 3
}
}
}
]
}
}
}
}
]

How to search for a nested key in kibana

I have kibana documents that look like this
{
"_index": "echo.caspian-test.2020-06-11.idx.2",
"_type": "status",
"_id": "01754abe95fd084495da20646194fdf7",
"_score": 1,
"_source": {
"applicationVersion": "9f80e49dea1c647fa1baf2e70665aba3a74158eb",
"echoClientVersion": "1.5.1",
"echoMetadata": {
"transportType": "echo"
},
"dataCenter": "hdc-digital-non-prod",
"echoLoggerVersion": "EchoLogbackAppender-1.5.1",
"host": "e22ab1e4-9256-438b-5855-ad04",
"type": "INFO",
"message": "AddUpdate process method ends",
"messageDetail": {
"logger": "com.kroger.cxp.app.transformer.processor.AddUpdateTransformerImpl",
"thread": "DispatchThread: [com.ibm.mq.jmqi.remote.impl.RemoteSession[:/1f6e1b6c][connectionId=414D5143514D2E4150504C2E54455354967C7F5F0407B82E]]"
},
"routingKey": "caspian-test",
"timestamp": "1603276805250"
},
"fields": {
"timestamp": [
"2020-10-21T10:40:05.250Z"
]
}
}
I need to search all the docs having a particular connectionId which is present in
"messageDetail": {
"logger": "com.kroger.cxp.app.transformer.processor.AddUpdateTransformerImpl",
"thread": "DispatchThread: [com.ibm.mq.jmqi.remote.impl.RemoteSession[:/1f6e1b6c][connectionId=414D5143514D2E4150504C2E54455354967C7F5F0407B82E]]"
}
How can i do that . I have tried searching for messageDetail.thread=%$CONNECTION_ID% but it didn't work
You need to add a nested path in your search query to make it work and your messageDetail must be of nested datatype, something like below
{
"query": {
"nested": {
"path": "messageDetail", --> note this
"query": {
"bool": {
"must": [
{
"match": {
"messageDetail. thread": "CONNECTION_ID"
}
}
]
}
}
}
}
}
Adding a working sample with mapping, search query, and result
Index mapping
{
"mappings": {
"properties": {
"messageDetail": {
"type" : "nested"
}
}
}
}
Index sample doc
{
"applicationVersion": "9f80e49dea1c647fa1baf2e70665aba3a74158eb",
"echoClientVersion": "1.5.1",
"echoMetadata": {
"transportType": "echo"
},
"dataCenter": "hdc-digital-non-prod",
"echoLoggerVersion": "EchoLogbackAppender-1.5.1",
"host": "e22ab1e4-9256-438b-5855-ad04",
"type": "INFO",
"message": "AddUpdate process method ends",
"messageDetail": {
"logger": "com.kroger.cxp.app.transformer.processor.AddUpdateTransformerImpl",
"thread": "DispatchThread: [com.ibm.mq.jmqi.remote.impl.RemoteSession[:/1f6e1b6c][connectionId=414D5143514D2E4150504C2E54455354967C7F5F0407B82E]]"
},
"routingKey": "caspian-test",
"timestamp": "1603276805250"
}
And search query
{
"query": {
"nested": {
"path": "messageDetail",
"query": {
"bool": {
"must": [
{
"match": {
"messageDetail.thread": "DispatchThread"
}
}
]
}
}
}
}
}
And search res
"hits": [
{
"_index": "nestedmsg",
"_type": "_doc",
"_id": "1",
"_score": 0.2876821,
"_source": {
"applicationVersion": "9f80e49dea1c647fa1baf2e70665aba3a74158eb",
"echoClientVersion": "1.5.1",
"echoMetadata": {
"transportType": "echo"
},
"dataCenter": "hdc-digital-non-prod",
"echoLoggerVersion": "EchoLogbackAppender-1.5.1",
"host": "e22ab1e4-9256-438b-5855-ad04",
"type": "INFO",
"message": "AddUpdate process method ends",
"messageDetail": {
"logger": "com.kroger.cxp.app.transformer.processor.AddUpdateTransformerImpl",
"thread": "DispatchThread: [com.ibm.mq.jmqi.remote.impl.RemoteSession[:/1f6e1b6c][connectionId=414D5143514D2E4150504C2E54455354967C7F5F0407B82E]]"
},
"routingKey": "caspian-test",
"timestamp": "1603276805250"
}
}
]

Elasticsearch filter by nested fields

I have a problem with creating a query to Elasticsearch with many conditions. My model looks like:
data class Product(
#Id
val id: String? = null,
val category: String,
val imagesUrls: List<String>,
#Field(type = FieldType.Double)
val price: Double?,
#Field(type = FieldType.Nested)
val parameters: List<Parameter>?
)
data class Parameter(
val key: String,
val values: List<String>
)
I would like to query products by:
category (for example cars)
price (between 20k $ and 50k $)
and parameters -> For example products with many parameters, like key capacity values 4L, 5L and second parameter gear transmission values manual
My current query looks like this:
GET data/_search
{
"size": 10,
"query": {
"bool": {
"must": [
{
"term": {
"category.keyword": {
"value": "cars"
}
}
},
{
"nested": {
"path": "parameters",
"query": {
"bool": {
"must": [
{"term": {
"parameters.key.keyword": {
"value": "Capacity"
}
}},
{
"term": {
"parameters.key": {
"value": "4L, 5L"
}
}
}
]
}
}
}
}
]
}
}
Could you tell me how to filter the product when parameter key is equal to Capacity and check that the values list contains one of the values?
How to combine many this kind operations in one query?
Example data:
{
"category":"cars",
"name":"Ferrari",
"price":50000,
"parameters":[
{
"key":"capacity",
"values":"4L"
},
{
"key":"gear transmission",
"values":"automcatic"
}
]
}
The search query shown below queries the data based on:
category (for example cars)
And parameters -> For example products with many parameters, like key capacity values 4L, 5L and second parameter gear transmission
values manual
Adding a working example with index data, mapping, search query, and search result
Index Mapping:
{
"mappings": {
"properties": {
"parameters": {
"type": "nested"
}
}
}
}
Index Data:
{
"category":"cars",
"name":"Ferrari",
"price":50000,
"parameters":[
{
"key":"gear transmission",
"values":["4L","5L"]
},
{
"key":"capacity",
"values":"automcatic"
}
]
}
{
"category":"cars",
"name":"Ferrari",
"price":50000,
"parameters":[
{
"key":"capacity",
"values":["4L","5L"]
},
{
"key":"gear transmission",
"values":"automcatic"
}
]
}
{
"category":"cars",
"name":"Ferrari",
"price":50000,
"parameters":[
{
"key":"capacity",
"values":"4L"
},
{
"key":"gear transmission",
"values":"automcatic"
}
]
}
Search Query:
{
"query": {
"bool": {
"must": [
{
"term": {
"category.keyword": {
"value": "cars"
}
}
},
{
"nested": {
"path": "parameters",
"query": {
"bool": {
"must": [
{
"match": {
"parameters.key": "capacity"
}
},
{
"terms": {
"parameters.values": [
"4l",
"5l"
]
}
}
]
}
}
}
},
{
"nested": {
"path": "parameters",
"query": {
"bool": {
"must": [
{
"match": {
"parameters.key": "gear transmission"
}
},
{
"match": {
"parameters.values": "automcatic"
}
}
]
}
}
}
}
]
}
}
}
Search Result:
"hits": [
{
"_index": "bstof",
"_type": "_doc",
"_id": "1",
"_score": 3.9281754,
"_source": {
"category": "cars",
"name": "Ferrari",
"price": 50000,
"parameters": [
{
"key": "capacity",
"values": "4L"
},
{
"key": "gear transmission",
"values": "automcatic"
}
]
}
},
{
"_index": "bstof",
"_type": "_doc",
"_id": "2",
"_score": 3.9281754,
"_source": {
"category": "cars",
"name": "Ferrari",
"price": 50000,
"parameters": [
{
"key": "capacity",
"values": [
"4L",
"5L"
]
},
{
"key": "gear transmission",
"values": "automcatic"
}
]
}
}
]
When you need to match any one from a list then you can use terms query instead of term. Update the part in query from:
{
"term": {
"parameters.key": {
"value": "4L, 5L"
}
}
}
to below:
{
"terms": {
"parameters.values": {
"value": [
"4L",
"5L"
]
}
}
}
Note that if parameters.key is analysed field and there exist a keyword sub-field for the same, then use it instead. e.g parameters.values.keyword
You can read more on terms query here.

ElasticSearch simple query

I have structure like this in my ElasticSearch
{
_index: 'index',
_type: 'product',
_id: '896',
_score: 0,
_source: {
entity_id: '896',
category: [
{
category_id: 2,
is_virtual: 'false'
},
{
category_id: 82,
is_virtual: 'false'
}
]
}
}
I want return all "producs" that have "82" category_id.
{
"query": {
"bool": {
"filter": {
"terms": {
"category.category_id": [
82
]
}
}
}
}
}
This query gives me 0 hits.
What is right way to do this?
Adding working example, you need to define the category as nested field and modify your search query by including the nested path
Index Mapping
{
"mappings": {
"properties": {
"entity_id": {
"type": "text"
},
"category": {
"type": "nested"
}
}
}
}
Index your document
{
"entity_id": "896",
"category": [
{
"category_id": 2,
"is_virtual": false
},
{
"category_id": 82,
"is_virtual": false
}
]
}
Proper search query, note we are using nested query which doesn't support normal filter(so your query gives error)
{
"query": {
"nested": {
"path": "category",
"query": {
"bool": {
"must": [
{
"match": {
"category.category_id": 82
}
}
]
}
}
}
}
}
Search result retuns indexed doc
"hits": [
{
"_index": "complexnested",
"_type": "_doc",
"_id": "1",
"_score": 1.0,
"_source": {
"entity_id": "896",
"category": [
{
"category_id": 2,
"is_virtual": false
},
{
"category_id": 82,
"is_virtual": false
}
]
}
}
]
If your query gives you no results, I suspect that category is of type nested in your index mapping. If that's the case, that's good and you can modify your query like this to use the nested query:
{
"query": {
"bool": {
"filter": {
"nested": {
"path": "category",
"query": {
"terms": {
"category.category_id": [
82
]
}
}
}
}
}
}
}

Resources