How to update and strip the first character of an array field in elasticsearch - elasticsearch

I have the following within an index in elastic search listed as a json doc
Current
{
"bar": {
"bar": [{
"bar": [{
"bar": [{
"foo": "Y111111111111"
}
]
}
]
}
}
}
Update needed
{
"bar": {
"bar": [{
"bar": [{
"bar": [{
"foo": "111111111111"
}
]
}
]
}
}
}
How would I go about updating the index to strip the first character of the string where equal to bar?
I tried the following syntax which works for a single field but I receive an exception when running on an array field
{
"error": {
"root_cause": [
{
"type": "script_exception",
"reason": "runtime error",
"script_stack": [
"ctx._source.foo = ctx._source.foo.substring(1);",
" ^---- HERE"
],
"script": "ctx._source.foo = ctx._source.foo.substring(1);",
"lang": "painless"
}
],
"type": "script_exception",
"reason": "runtime error",
"script_stack": [
"ctx._source.foo = ctx._source.foo.substring(1);",
" ^---- HERE"
],
"script": "ctx._source.foo = ctx._source.foo.substring(1);",
"lang": "painless",
"caused_by": {
"type": "illegal_argument_exception",
"reason": "Illegal list shortcut value [foo]."
}
},
"status": 400
}
POST test/_update_by_query
{
"query": {
"prefix": {
"foo": "Y"
}
},
"script": {
"source": "ctx._source.foo = ctx._source.foo.substring(1);"
}
Mapping
{
"TEST": {
"mappings": {
"properties": {
"bar": {
"properties": {
"bar": {
"properties": {
"bar: {
"properties": {
"bar": {
"properties": {
"foo": {
"type": "keyword"
}
}
}
}
}
}
}
}
}
}
}
}
}

I would do it as follows. Find all documents whose foo field starts with Y and then update all the foo fields by stripping the first character:
POST test/_update_by_query
{
"query": {
"prefix": {
"bar.bar.bar.bar.foo": "Y"
}
},
"script": {
"source": "ctx._source.bar.bar[0].bar[0].bar[0].foo = ctx._source.bar.bar[0].bar[0].bar[0].foo.substring(1);"
}
}
PS: the query will depend on whether your bar fields are nested or not, but in case they are not, the above query should work.

Related

Scripted Metric Aggregation fails on unexpected = character

I'm trying to have some object containing all time deltas between two logs with the same recordId.
In order to do so, I'm executing the following query (Scripted Metric Aggregation) on elastic search, where the logs are, using fiddler:
{
"query": {
"exists": {
"field": "recordId"
}
},
"aggs": {
"deltas": {
"scripted_metric": {
"init_script": {
"source": "state.deltas = {};",
"lang": "expression"
},
"map_script": {
"source": " if (!(doc['topic'].value in state.deltas)) { state.deltas[doc['topic'].value] = {} } state.deltas[doc['topic'].value][doc['recordId'].value] = !(doc['recordId'].value in state.deltas[doc['topic'].value]) ? doc['#timestamp'].date.millisOfDay : Math.abs(state.deltas[doc['topic'].value][doc['recordId'].value] - doc['#timestamp'].date.millisOfDay) ",
"lang": "expression"
},
"reduce_script": {
"source": " res = {}; for (s in states) { for(topic in Object.keys(s.deltas)) { if (!(topic in res)) { res[topic] = {} } for(recordId in Object.keys(s.deltas[topic])) { res[topic][recordId] = !(recordId in res[topic]) ? s.deltas[topic][recordId] : Math.abs(res[topic][recordId] - s.deltas[topic][recordId]) } } } return res;",
"lang": "expression"
}
}
}
}
}
But it fails because of unexpected '=' character...
Tried other ways but it always throws the same error.
What am I missing?
{
"error": {
"root_cause": [
{
"type": "lexer_no_viable_alt_exception",
"reason": "lexer_no_viable_alt_exception: null"
}
],
"type": "search_phase_execution_exception",
"reason": "all shards failed",
"phase": "query",
"grouped": true,
"failed_shards": [
{
"shard": 0,
"index": "il0:il1-sys-logs-2020.10.07",
"node": "5bmIYI_iSVCtx41qEEbASg",
"reason": {
"type": "script_exception",
"reason": "compile error",
"script_stack": [
"state.deltas = {};",
" ^---- HERE"
],
"script": "state.deltas = {};",
"lang": "expression",
"caused_by": {
"type": "parse_exception",
"reason": "parse_exception: unexpected character '= ' on line (1) position (13)",
"caused_by": {
"type": "lexer_no_viable_alt_exception",
"reason": "lexer_no_viable_alt_exception: null"
}
}
}
}
],
"caused_by": {
"type": "lexer_no_viable_alt_exception",
"reason": "lexer_no_viable_alt_exception: null"
}
},
"status": 500
}

How to use nested field in Elasticsearch filter script

I have following mappings:
"properties": {
"created": {
"type": "date"
},
"id": {
"type": "keyword"
},
"identifier": {
"type": "keyword"
},
"values": {
"properties": {
"description_created-date": {
"properties": {
"<all_channels>": {
"properties": {
"<all_locales>": {
"type": "date"
}
}
}
}
},
"footwear_size-option": {
"properties": {
"<all_channels>": {
"properties": {
"<all_locales>": {
"type": "keyword"
}
}
}
}
}
}
}
}
Now I would like to create a query based on description_created-date field and use this value in painless script by comparing to some date.
GET index/pim_catalog_product/_search
{
"query": {
"constant_score": {
"filter": {
"bool": {
"filter": [
{
"script": {
"script": {
"source": "doc['values']['description_created-date']['<all_channels>']['<all_locales>'].value == '2019-12-19'",
"lang": "painless"
}
}
}
]
}
}
}
}
}
But I get following error:
{
"shard": 0,
"index": "index",
"node": "cmh1RMS1SHO92SA3jPAkJA",
"reason": {
"type": "script_exception",
"reason": "runtime error",
"script_stack": [
"org.elasticsearch.search.lookup.LeafDocLookup.get(LeafDocLookup.java:81)",
"org.elasticsearch.search.lookup.LeafDocLookup.get(LeafDocLookup.java:39)",
"doc['values']['description_created-date']['<all_channels>']['<all_locales>'].value == '2019-12-19'",
" ^---- HERE"
],
"script": "doc['values']['description_created-date']['<all_channels>']['<all_locales>'].value == '2019-12-19'",
"lang": "painless",
"caused_by": {
"type": "illegal_argument_exception",
"reason": "No field found for [values] in mapping with types [pim_catalog_product]"
}
}
}
(I know I can't compare dates like this, but this is another problem).
Searching by values.description_created-date field works:
GET index/pim_catalog_product/_search
{
"query": {
"match": {
"values.description_created-date.<all_channels>.<all_locales>": "2019-12-19"
}
}
}
And when I get specific document, value of this field is presented like this:
"values": {
"description_created-date": {
"<all_channels>": {
"<all_locales>": "2019-12-19"
}
}
}
How can I use this field in script filter? I need this to perform something like this:
(pseudocode)
"source": "doc['values']['stocks_created-date'].value > doc['created'].value + 2 days"
I'm using elasicsearch v6.5.0, here is a docker-compose file with elasticsearch and kibana:
version: '3'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:6.5.0
environment:
- discovery.type=single-node
ports:
- 9200:9200
kibana:
image: docker.elastic.co/kibana/kibana:6.5.0
ports:
- 5601:5601
and gist with full mappings and sample data here
Thanks.
Thanks for the expanded mappings! When calling a field within a nested object, try referring to the inner field using the dot notation. Example:
"source": "doc['values.description_created.<all_channels>.<all_locales>'].value == 2019"
Also, you could reduce your compound queries to just your main constant_score compound query. Example:
GET index/_search
{
"query": {
"constant_score": {
"filter": {
"script": {
"script": {
"source": "doc['values.description_created.<all_channels>.<all_locales>'].value == 2019"
}
}
},
"boost": 1
}
}
}
NOTE: The "boost" value is optional, but it's the default if you don't provide a boost value.

Elasticsearch search length of array

Here is my object profile:
{
"mappings": {
"_doc": {
"properties": {
"name": {
"type": "text"
},
"posts": {
"properties": {
"id": {
"type": "text"
},
"create_date": {
"type": "long"
}
}
}
}
}
}
}
I want to make a search: return all profiles which
1. have name "bob"
2. and have more than 5 posts
Here is an example that I found, but it does not work
{
"query": {
"bool": {
"must": [
{
"term": {
"name": "bob"
}
}
],
"filter": [
{
"script": {
"script": "doc['posts'].values.size() > 5"
}
}
]
}
}
}
I get error "reason":"Variable [posts] is not defined."
update posts.id to keyword
{"id": {"type":"text"},"fields":{{"keyword":{"type":"keyword","ignore_above":256}}}}
Same error
"caused_by":{"type":"script_exception","reason":"compile error","script_stack":["doc[posts.id].values.size() > ..."," ^---- HERE"],"script":"doc[posts.id].values.size() > 5","lang":"painless","caused_by":{"type":"illegal_argument_exception","reason":"Variable [posts] is not defined."}}}}]},"status":400
According to https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html this is because one document is missing the field posts (I suppose).
You could add a filter: "if field posts exists" or use in the script the condition "if doc.containsKey('posts')...."

How to add new element to the existing array field in Elasticsearch

Hi Below is my document
"catid": [
514500
],
"studentid": 5282439,
In this catid field , I want to add new element 543 , where studentid = 5282439
I am trying the below query but it's giving me an exception
POST /parts/_update_by_query
{
"query": {
"match": {
"studentid": 5282439
}
},
"script" : "ctx._source.catid+= [543 ]"
}
I am getting the below exception:
"root_cause": [
{
"type": "class_cast_exception",
"reason": "java.lang.String cannot be cast to java.util.Map"
}
]
--> If the above answer doesn't work try this one
POST /parts/_update_by_query
{
"query": {
"match": {
"studentid": 5282439
}
},
"script" : {
"lang":"painless",
"inline": "ctx._source.catid.add(params.newsupp)",
"params":{
"newsupp":5302
}
}
}
The script part is not correct, change it to this instead (i.e. move the script to the script.inline property):
POST /parts/_update_by_query
{
"query": {
"match": {
"studentid": 5282439
}
},
"script" : {
"inline": "ctx._source.catid += [543 ]"
}
}

How to delete a field in elasticsearch 2.3.0? I am using sense to do my queries

I have inserted a new field wiki_collection and set its value to true in my document by using:
POST /pibtest1/_update_by_query
{
"script": {
"inline": "ctx._source.wiki_collection=true"
},
"query": {
"match": {
"url":"http%3A%2F%2Fwww.searchtechnologies.com%2Fbundles%2Fmodernizr%3Fv%3D7-yR01kdRVQ7W1RQzMBVKYLDhCt0itEATWHixfzE8Os1"
}
}
}
But now I want to delete this field. I am trying to do this:
POST /pibtest1/_update
{
"script" : "ctx._source.remove(\"wiki_collection\")"
}
But I am getting an error which says:
{
"error": {
"root_cause": [
{
"type": "invalid_type_name_exception",
"reason": "Document mapping type name can't start with '_'"
}
],
"type": "invalid_type_name_exception",
"reason": "Document mapping type name can't start with '_'"
},
"status": 400
}
Is there any other way to delete a field in elasticsearch?
Edit: I have updated by query:
POST /pibtest1/_update_by_query
{
"script": {
"inline": "ctx._source.remove(\"wiki_collection\")"
},
"query": {
"match": {
"url": "http://www.searchtechnologies.com/bundles/modernizr?v=7-yR01kdRVQ7W1RQzMBVKYLDhCt0itEATWHixfzE8Os1"
}
}
}
But now I am getting a new filed called "remove='wiki_collection'" in my documents which looks like this: http://i.stack.imgur.com/QvxIa.png
I want to remove/delete this wiki_collection field from my documents.
Your update is wrong. It should either be (you specify the complete path - index/type/ID):
POST /pibtest1/test/234/_update
{
"script": {
"inline": "ctx._source.remove(\"wiki_collection\")"
}
}
Or you use the same _update_by_query:
POST /pibtest1/_update_by_query
{
"script": {
"inline": "ctx._source.remove(\"wiki_collection\")"
},
"query": {
"match": {
"url": "whatever"
}
}
}

Resources