I have the following set-up:
mapping:
esClient.indices.putMapping({
index: 'tests',
body: {
properties: {
name: {
type: 'text',
},
lastName: {
type: 'text',
},
},
},
});
this is the result when I post an entry:
this is the result when I query the entries:
curl -X GET "localhost:9200/tests/_search?pretty" -H 'Content-Type: application/json' -d'
{
"size": 1000,
"query" : {
"match_all" : {}
}
}
'
{
"took" : 18,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "tests",
"_type" : "_doc",
"_id" : "KJbtj3kBRlqnip7VJLLI",
"_score" : 1.0,
"_source" : {
"lastName" : 1,
"name" : "lucas"
}
}
]
}
}
I tried to update the entry's last name with the following curl:
curl -X POST "localhost:9200/tests/_update_by_query?pretty" -H 'Content-Type: application/json' -d'
{
"script": {
"source": "ctx._source.lastName='johnson'",
"lang": "painless"
},
"query": {
"term": {
"name": "lucas"
}
}
}
'
AND THIS IS THE ERROR I'M GETTIN:
{
"error" : {
"root_cause" : [
{
"type" : "script_exception",
"reason" : "compile error",
"script_stack" : [
"ctx._source.lastName=johnson",
" ^---- HERE"
],
"script" : "ctx._source.lastName=johnson",
"lang" : "painless",
"position" : {
"offset" : 21,
"start" : 0,
"end" : 28
}
}
],
"type" : "script_exception",
"reason" : "compile error",
"script_stack" : [
"ctx._source.lastName=johnson",
" ^---- HERE"
],
"script" : "ctx._source.lastName=johnson",
"lang" : "painless",
"position" : {
"offset" : 21,
"start" : 0,
"end" : 28
},
"caused_by" : {
"type" : "illegal_argument_exception",
"reason" : "cannot resolve symbol [johnson]"
}
},
"status" : 400
}
If I put an integer instead of a string it updates it otherwise I keep getting that error.
Thanks a lot for your help.
You need to surround the new lastName field value with ' '.
Adding a working example
Index Data:
{
"name":"lucas",
"lastName":"erla"
}
Query:
POST _update_by_query
{
"script": {
"source": "ctx._source.lastName='johnson'",
"lang": "painless"
},
"query": {
"term": {
"name": "lucas"
}
}
}
After hitting the update by query API, the document will be updated to
GET /_doc/1
{
"_index": "67641538",
"_type": "_doc",
"_id": "1",
"_version": 3,
"_seq_no": 2,
"_primary_term": 1,
"found": true,
"_source": {
"lastName": "johnson",
"name": "lucas"
}
}
You need to surround the new lastName field value with " ".
I faced similar problem and I solved by adding " " instead of ' '. My ES version is 8.4.
QUERY.
curl -X POST "localhost:9200/tests/_update_by_query?pretty" -H 'Content-Type: application/json' -d'
{
"script": {
"source": "ctx._source.lastName=\"johnson\"",
"lang": "painless"
},
"query": {
"term": {
"name": "lucas"
}
}
}
'
Related
Trying to apply sum aggregation in ES 7.14 and get unexpected result
1. prepare dataset
$cat products.json
{"index":{"_id":"1"}}
{"productId": 10,"shopId": 45,"prices": {"retailPrice": 525000000.02,"sumRetailPrice": 5250000000.2},"count": 10}
{"index":{"_id":"2"}}
{"productId": 10,"shopId": 48,"prices": {"retailPrice": 26250000004,"sumRetailPrice": 5250000000.8},"count": 20}
2. bulk insert
curl -XPOST localhost:9200/25products/_bulk -H "Content-Type: application/x-ndjson" --data-binary #./products.json
3. view mapping
curl -XGET "http://localhost:9200/25products/_mapping?pretty"
{
"25products" : {
"mappings" : {
"properties" : {
"count" : {
"type" : "long"
},
"prices" : {
"properties" : {
"retailPrice" : {
"type" : "float"
},
"sumRetailPrice" : {
"type" : "float"
}
}
},
"productId" : {
"type" : "long"
},
"shopId" : {
"type" : "long"
}
}
}
}
}
4. Sum field "prices.sumRetailPrice" in Painless
curl --location --request POST 'http://localhost:9200/25products/_search?pretty' \
--header 'Content-Type: application/json' \
--data-raw '{
"aggs": {"sumSupplyPrice": {
"sum": {"script": {
"source": "(!doc.containsKey('\''prices.sumRetailPrice'\'') ? 0 : (doc['\''prices.sumRetailPrice'\''].size() == 0 ? 0: doc['\''prices.sumRetailPrice'\''].value))"
}}
}},
"query": {"bool": {
"filter": [
{"terms": {"shopId": [45]}},
{"terms": {"productId": [10]}}
]
}},
"from": 0, "size": 10
}'
result is
{
"took" : 2,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 0.0,
"hits" : [
{
"_index" : "25products",
"_type" : "_doc",
"_id" : "1",
"_score" : 0.0,
"_source" : {
"productId" : 10,
"shopId" : 45,
"prices" : {
"retailPrice" : 5.2500000002E8,
"sumRetailPrice" : 5.2500000002E9
},
"count" : 10
}
}
]
},
"aggregations" : {
"sumSupplyPrice" : {
"value" : 5.249999872E9
}
}
}
4. Expectation
as well as I have a single record, expecting to have the same value as sumRetailPrice
"aggregations" : {
"sumSupplyPrice" : {
"value" : **5.2500000002E9**
}
}
But, actual result is not as expected.
"aggregations" : {
"sumSupplyPrice" : {
"value" : **5.249999872E9**
}
}
Where am I wrong?
Thanks!
Hi I am having an instance of elastic search running on my machine . it has an index named mep-reports. when i do a query using curl command it is giving an error . the following is the curl command.
curl -X GET "10.10.9.1:9200/mep-reports*/_search?pretty&size=0" -H 'Content-Type: application/json' -d'{
"size": 0,
"query": {
"bool": {
"must": [
{
"range": {
"#timestamp": {
"from": "2019-01-31T23:59:59Z",
"to": "2020-02-17T23:59:59Z",
"include_lower": true,
"include_upper": false,
"format": "yyyy-MM-dd'T'HH:mm:ssZ",
"boost": 1.0
}
}
},
{
"term": {
"account_id": {
"value": "270d13e6-2f4f-4d51-99d5-92ffba5f0cb6",
"boost": 1.0
}
}
}
],
"adjust_pure_negative": true,
"boost": 1.0
}
},
"aggregations": {
"performance_over_time": {
"date_histogram": {
"field": "#timestamp",
"format": "yyyy-MM-dd'T'HH:mm:ssZ",
"interval": "1M",
"offset": 0,
"order": {
"_key": "asc"
},
"keyed": false,
"min_doc_count": 0
}
}
}
}'
Response
{
"error" : {
"root_cause" : [
{
"type" : "illegal_argument_exception",
"reason" : "Invalid format: [yyyy-MM-ddTHH:mm:ssZ]: Illegal pattern component: T"
}
],
"type" : "illegal_argument_exception",
"reason" : "Invalid format: [yyyy-MM-ddTHH:mm:ssZ]: Illegal pattern component: T",
"caused_by" : {
"type" : "illegal_argument_exception",
"reason" : "Illegal pattern component: T"
}
},
"status" : 400
}
The following a sample from my elastic search index
{
"took" : 14,
"timed_out" : false,
"_shards" : {
"total" : 12,
"successful" : 12,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : 1073013,
"max_score" : 1.0,
"hits" : [
{
"_index" : "mep-reports-2019.09.11",
"_type" : "doc",
"_id" : "68e8e03f-baf8-4bfc-a920-58e26edf835c-353899837500",
"_score" : 1.0,
"_source" : {
"account_id" : "270d13e6-2f4f-4d51-99d5-92ffba5f0cb6",
"inventory" : "SMS",
"flight_name" : "test flight 001",
"status" : "ENROUTE",
"msg_text" : "Test !!!!!!!!!!!!!!1 elastic searchY",
"flight_id" : "68e8e03f-baf8-4bfc-a920-58e26edf835c",
"submission_ts" : "1568197286",
"recipient" : "353899837500",
"o_error" : null,
"nof_segments" : "-1",
"campaign_id" : "0fae8662-bee9-46ac-9b3e-062f4ba55966",
"campaign_name" : "Index search petri11",
"#version" : "1",
"sender" : "800111",
"delivery_ts" : "0",
"#timestamp" : "2019-09-11T10:21:26.000Z"
}
}
]
}
}
it something related to date format as i am trying to do a search on #timestamp field
really appreciate if you can help
thank you
The problem is because the JSON query is enclosed into single quotes, i.e. the same characters around the T in your date format.
What I suggest you to do is to store the query inside a file named query.json and then send it in binary-mode like this:
curl -X GET "10.10.9.1:9200/mep-reports*/_search?pretty&size=0" -H 'Content-Type: application/json' --data-binary #query.json
That should solve your issue
I am trying to work with update_by_query but cannot make it work.
Following is a simple query,
curl -X GET "172.17.0.3:9200/useripvsuserid/_search?pretty" -H 'Content-Type: application/json' -d'
{
"_source":"userid","query": {
"term": {
"userip": "10.0.30.181"
}
}
}
'
{
"took" : 3,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 10.803431,
"hits" : [
{
"_index" : "useripvsuserid",
"_type" : "_doc",
"_id" : "PhfBW3AB8mhGfmGvIs-j",
"_score" : 10.803431,
"_source" : {
"userid" : "hasan1855"
}
}
]
}
}
Following is the update_by_query that is not working. I am trying to replace userid value hasan1855 to arif. Where is the problem?
curl -X POST "172.17.0.3:9200/useripvsuserid/_update_by_query?pretty" -H 'Content-Type: application/json' -d'
{
"script": {
"source": "ctx._source.userid='arif';",
"lang": "painless"
},
"query": {
"term": {
"userip": "10.0.30.181"
}
}
}
'
{
"error" : {
"root_cause" : [
{
"type" : "script_exception",
"reason" : "compile error",
"script_stack" : [
"ctx._source.userid=arif;",
" ^---- HERE"
],
"script" : "ctx._source.userid=arif;",
"lang" : "painless"
}
],
"type" : "script_exception",
"reason" : "compile error",
"script_stack" : [
"ctx._source.userid=arif;",
" ^---- HERE"
],
"script" : "ctx._source.userid=arif;",
"lang" : "painless",
"caused_by" : {
"type" : "illegal_argument_exception",
"reason" : "Variable [arif] is not defined."
}
},
"status" : 400
}
It's the same issue as described here, i.e. the single quotes around arif conflict with the single quotes around the JSON query.
So you can either send your query in binary mode as explained in the link above, or escape the quotes, like this:
curl -X POST "172.17.0.3:9200/useripvsuserid/_update_by_query?pretty" -H 'Content-Type: application/json' -d'
{
"script": {
"source": "ctx._source.userid = \"arif\";", <---- escape quotes
"lang": "painless"
},
"query": {
"term": {
"userip": "10.0.30.181"
}
}
}
'
According to the documentation you can run ElasticSearch aggregations on fields that are type keyword or not a text field or which have fielddata set to true in the index mapping.
I am trying to count city_names in an nginx log. It works fine with the int field result. But it does not work with the field city_name even when I updated the index mapping for that to put fielddata=true. The should have been not required as it was of type keyword.
To say it does not work means that:
"aggregations" : {
"cities" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [ ]
}
}
Here is the field mapping:
"city_name" : {
"type" : "text",
"fielddata" : true
},
And here is the aggression query:
curl -XGET --user $pwd --header 'Content-Type: application/json' https://58571402f5464923883e7be42a037917.eu-central-1.aws.cloud.es.io:9243/logstash/_search?pretty -d '{
"aggs" : {
"cities": {
"terms" : { "field": "city_name"}
}
}
}'
If you don't get any error when executing your search it seems that is more like a problem with the data. Are you sure you have, at least, one document with the field city_name filled?
I tried to reproduce your issue with ElasticSearch 6.6.2.
I created an index
PUT cities
{
"mappings": {
"city": {
"dynamic": "true",
"properties": {
"id": {
"type": "long"
},
"city_name": {
"type": "text",
"fielddata": true
}
}
}
}
}
I added one document without the city_name
PUT cities/city/1
{
"id": "1"
}
When i performed the search:
GET cities/_search
{
"aggs": {
"cities": {
"terms" : { "field": "city_name"}
}
}
}
I got no buckets in the cities aggregation. But when I added one document with the city name filled:
PUT cities/city/2
{
"id": "2",
"city_name": "London"
}
I got the expected result:
{
"took" : 3,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : 2,
"max_score" : 1.0,
"hits" : [
{
"_index" : "cities",
"_type" : "city",
"_id" : "2",
"_score" : 1.0,
"_source" : {
"id" : "2",
"city_name" : "london"
}
},
{
"_index" : "cities",
"_type" : "city",
"_id" : "1",
"_score" : 1.0,
"_source" : {
"id" : "1"
}
}
]
},
"aggregations" : {
"cities" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "london",
"doc_count" : 1
}
]
}
}
}
I have set up an elastic Search index which includes different _type mapping for every country.
So there is a mapping for "us" "au" "uk" etc.
Each mapping includes a location mapping of type "geo_point"
prior to adding different _types
My query sort would look like:
"sort" : [
{
"_geo_distance" : {
"postcode.location" : [' . $mylocation_long . ',' . $mylocation_lat . '],
"order" : "asc",
"unit" : "km"
}
}
],
with adding _types to the data and mapping this no longer works, instead I specify it like:
"sort" : [
{
"_geo_distance" : {
"$country.location" : [' . $mylocation_long . ',' . $mylocation_lat . '],
"order" : "asc",
"unit" : "km"
}
}
],
this works fine.
However there are times when queries need to be done beyond a single country. So setting it to "us.location" isn't correct, and wont work.
In this case, how do I make this sorting work, when I don't know the country and I need to sort it by a mapped location.
Or is it a case of this can not be done and all docs must have the same _type in order for this to work?
Sorry if I am missing something obvious, but why cannot you just sort on "location". It seems to work just fine:
curl -XDELETE localhost:9200/test-idx/ && echo
curl -XPUT localhost:9200/test-idx/ -d '
{
"settings":{
"number_of_shards":1,
"number_of_replicas":0
},
"mappings": {
"us": {
"properties": {
"location": {
"type": "geo_point"
}
}
},
"uk": {
"properties": {
"location": {
"type": "geo_point"
}
}
},
"au": {
"properties": {
"location": {
"type": "geo_point"
}
}
}
}
}' && echo
curl -XPUT localhost:9200/test-idx/us/1 -d '
{
"location": "42.3606402,-71.0674569"
}
' && echo
curl -XPUT localhost:9200/test-idx/uk/2 -d '
{
"location": "51.5286416,-0.1017943"
}
' && echo
curl -XPUT localhost:9200/test-idx/au/3 -d '
{
"location": "-33.8471226,151.0594183"
}
' && echo
curl -XPOST localhost:9200/test-idx/_refresh && echo
curl "localhost:9200/test-idx/_search?pretty" -d '{
"query": {
"match_all": {}
},
"sort" : [
{
"_geo_distance" : {
"location" : "52.3712989,4.8937347",
"order" : "asc",
"unit" : "km"
}
}
]
}' && echo
output:
{"ok":true,"acknowledged":true}
{"ok":true,"acknowledged":true}
{"ok":true,"_index":"test-idx","_type":"us","_id":"1","_version":1}
{"ok":true,"_index":"test-idx","_type":"uk","_id":"2","_version":1}
{"ok":true,"_index":"test-idx","_type":"au","_id":"3","_version":1}
{"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
{
"took" : 3,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : null,
"hits" : [ {
"_index" : "test-idx",
"_type" : "uk",
"_id" : "2",
"_score" : null, "_source" : {"location": "51.5286416,-0.1017943"},
"sort" : [ 355.2735714686373 ]
}, {
"_index" : "test-idx",
"_type" : "us",
"_id" : "1",
"_score" : null, "_source" : {"location": "42.3606402,-71.0674569"},
"sort" : [ 5563.606078215864 ]
}, {
"_index" : "test-idx",
"_type" : "au",
"_id" : "3",
"_score" : null, "_source" : {"location": "-33.8471226,151.0594183"},
"sort" : [ 16650.926847312003 ]
} ]
}
}
What happens when you point the working query at /index/_search instead of /index/type/_search ?