Upsert document such that it would update the particular item in an array field - elasticsearch

In Elasticsearch, say I have the document like this:
{
"inputs": [
{
"id": "1234",
"value": "ABCD"
},
{
"id": "5678",
"value": "EFGH"
}
]
}
Say, now, I wanted to update value of all items where id is "1234" to "XYZA". How can I do that using script in elasticsearch? I am not sure if I can do some for loop in script?

Mapping:
{
"inputs" : {
"mappings" : {
"properties" : {
"inputs" : {
"type" : "nested",
"properties" : {
"id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"value" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
}
}
}
Query:
You can use _update_by_query api. Query part will filter out documents and script will update the field
<1. When inputs is of nested type
POST inputs/_update_by_query
{
"script": {
"source": "for(a in ctx._source['inputs']){if(a.id=='1234') a.value=params.new_value; }",
"params": {
"new_value": "XYZA"
}
},
"query": {
"nested":{
"path":"inputs",
"query":{
"term":{
"inputs.id":1234
}
}
}
}
}
2. When inputs if of object type
POST inputs/_update_by_query
{
"script": {
"source": "for(a in ctx._source['inputs']){if(a.id=='1234') a.value=params.new_value; }",
"params": {
"new_value": "XYZA"
}
},
"query": {
"term": {
"inputs.id": 1234
}
}
}
Result:
"hits" : [
{
"_index" : "inputs",
"_type" : "_doc",
"_id" : "3uwrwHEBLcdvQ7OTrUmi",
"_score" : 1.0,
"_source" : {
"inputs" : [
{
"id" : "1234",
"value" : "XYZA"
},
{
"id" : "5678",
"value" : "EFGH"
}
]
}
}
]

Related

Elasticsearch Nested query not working as expected

I am bit new to elastic search. I am trying a nested query to get the result soem thing like below sql in query DSL..means I wanna restrict the search to driver last name as well as the vehicle make as well..like below use case.
select driver.last_name,driver.vehicle.make,driver.vehicle.model from drivers
where driver.last_name='Hudson' and driver.vehicle.make"="Miller-Mete;
But this doesn't work in elastic search sql as well as Query DSL...
--> can we do the query like this in ES...like let me clarify..
if department has List[employees] in Elasticsearch denoarmalized data..
and i want to restrict the query to department_name and emp_position..
--> is this use case even possible in elastic search?
select department_name,emp_name,emp_salary,emp_position
where emp_position="Intern" and department.name="devlopment"
--> Below are mappings and search Query DSL...
PUT /drivers
{
"mappings": {
"properties": {
"driver": {
"type": "nested",
"properties": {
"last_name": {
"type": "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"vehicle": {
"type": "nested",
"properties": {
"make": {
"type": "text"
},
"model": {
"type": "text"
}
}
}
}
}
}
}
}
GET /drivers/_mapping
O/P:
{
"drivers" : {
"mappings" : {
"properties" : {
"driver" : {
"type" : "nested",
"properties" : {
"last_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"vehicle" : {
"type" : "nested",
"properties" : {
"make" : {
"type" : "text"
},
"model" : {
"type" : "text"
}
}
}
}
}
}
}
}
}
--> inserting documents..
PUT /drivers/_doc/1
{
"driver" : {
"last_name" : "McQueen",
"vehicle" : [
{
"make" : "Powell Motors",
"model" : "Canyonero"
},
{
"make" : "Miller-Meteor",
"model" : "Ecto-1"
}
]
}
PUT /drivers/_doc/2
{
"driver" : {
"last_name" : "Hudson",
"vehicle" : [
{
"make" : "Mifune",
"model" : "Mach Five"
},
{
"make" : "Miller-Meteor",
"model" : "Ecto-1"
}
]
}
}
--> Below is the search query dsl..this gives 0 results. Even i replace
"term": {
"driver.last_name.keyword": "McQueen"
}
with "match" or "filter" still gives 0 results...
GET /drivers/_search
{
"query": {
"nested": {
"path": "driver",
"query": {
"nested": {
"path": "driver.vehicle",
"query": {
"bool": {
"must": [
{ "match": { "driver.vehicle.make": "Powell Motors" } },
{ "match": { "driver.vehicle.model": "Canyonero" } },
{
"term": {
"driver.last_name.keyword": "McQueen"
}
}
]
}
}
}
}
}
}
}
==> below Query DSL gives 2 results...
GET /drivers/_search
{
"query": {
"nested": {
"path": "driver",
"query": {
"nested": {
"path": "driver.vehicle",
"query": {
"bool": {
"must": [
{ "match": { "driver.vehicle.make": "Miller-Meteor" } }
]
}
}
}
}
}
}
}
O/P:
{
"took" : 1,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 2,
"relation" : "eq"
},
"max_score" : 1.3097506,
"hits" : [
{
"_index" : "drivers",
"_type" : "_doc",
"_id" : "1",
"_score" : 1.3097506,
"_source" : {
"driver" : {
"last_name" : "McQueen",
"vehicle" : [
{
"make" : "Powell Motors",
"model" : "Canyonero"
},
{
"make" : "Miller-Meteor",
"model" : "Ecto-1"
}
]
}
}
},
{
"_index" : "drivers",
"_type" : "_doc",
"_id" : "2",
"_score" : 1.3097506,
"_source" : {
"driver" : {
"last_name" : "Hudson",
"vehicle" : [
{
"make" : "Mifune",
"model" : "Mach Five"
},
{
"make" : "Miller-Meteor",
"model" : "Ecto-1"
}
]
}
}
}
]
}
}
==> this gives "parsing_exception",
"reason" : "[bool] malformed query, expected [END_OBJECT] but found [FIELD_NAME]",
==> even replacing 1st query bool to "match" also gives this error...as below
GET /drivers/_search
{
"query": {
"nested": {
"path": "driver",
"query": {
"bool": {
"must": [
{"match": {
"driver.last_name.keyword": "Hudson"
}}
]
},
"nested": {
"path": "driver.vehicle",
"query": {
"bool": {
"must": [
{
"match": {
"driver.vehicle.make": "Miller-Meteor"
}
}
]
}
}
}
}
}
}

update a nested object in Elasticsearch

i hope i can find some help
my problem is that i am triying to update a nasted object that i have.
here 's my index architecture
{
"grille_badge" : {
"mappings" : {
"properties" : {
"theme" : {
"type" : "nested",
"properties" : {
"categories" : {
"type" : "nested",
"properties" : {
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"niveaux" : {
"type" : "nested",
"properties" : {
"grille_defis" : {
"type" : "nested",
"properties" : {
"defis" : {
"type" : "nested",
"properties" : {
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
}
}
}
and here is exemple of my document that i wand to edit
{
"_index" : "grille_badge",
"_type" : "_doc",
"_id" : "1",
"_version" : 1,
"_seq_no" : 0,
"_primary_term" : 1,
"found" : true,
"_source" : {
"theme" : [
{
"name" : "foot",
"categories" : [
{
"name" : "categorie1",
"niveaux" : [
{
"name" : "bronze",
"grille_defis" : [
{
"name" : "grille1",
"defis" : [ ]
}
]
}
]
},
{
"name" : "categorie2",
"niveaux" : [
{
"name" : "argent",
"grille_defis" : [
{
"name" : "grille1",
"defis" : [ ]
}
]
}
]
},
{
"name" : "categorie3",
"niveaux" : [
{
"name" : "or",
"grille_defis" : [
{
"name" : "grille1",
"defis" : [ ]
}
]
}
]
}
]
}
]
}
}
And i am triying to add a Defis to my empty list, like the code below that i found in the documentation
POST index_name/_update/doc_id
{
"script": {
"source": "ctx._source.cats.add(params.cat)",
"params": {
"cat": {
"colors": 4,
"name": "Leon",
"breed": "Persian"
}
}
}
}
i tried to make my own script that can go deep into my defis list but still don't know how to do it.
here's what i tried to do
POST grille_badge/_update/1
{
"script": {
"source": "def theme_target = ctx._source.theme.findAll(our_theme theme.name == params.current_theme); def categorie_target = theme_target.findAll(our_categorie categories.name == params.current_categorie); def niveau_target = categorie_target.findAll(our_niveau niveaux.name == params.current_niveau); def grille_target = niveau_target.findAll(our_grille grille_defis.name == params.current_grille); grille_target.defis.add(params.defis_ajouter)",
"params": {
"current_theme": "foot",
"current_categorie": "categorie1",
"current_niveau": "niveau1",
"current_grille": "grille1",
"defis_ajouter": "i am a defis"
}
}
}
please could anyone help me !!
thanks for advance
Welcome to SO. Your question is quite confusing but here's how you update a doc by a script where your params are an instance of HashMap:
Index the original document w/ an id of 1 -- collapsed for brevity:
POST grille_badge/_doc/1
{"theme":[{"name":"foot","categories":[{"name":"categorie1","niveaux":[{"name":"bronze","grille_defis":[{"name":"grille1","defis":[]}]}]},{"name":"categorie2","niveaux":[{"name":"argent","grille_defis":[{"name":"grille1","defis":[]}]}]},{"name":"categorie3","niveaux":[{"name":"or","grille_defis":[{"name":"grille1","defis":[]}]}]}]}]}
Update doc#1 (note that you can also use a wide-ranging _update_by_query too). Also note that you need to verify whether cats exist before add-ing a hashmap in there.
POST grille_badge/_update/1
{
"script": {
"source": "if (ctx._source.cats instanceof List) { ctx._source.cats.add(params.cat) } else { ctx._source.cats = [params.cat] }",
"params": {
"cat": {
"colors": 4,
"name": "Leon",
"breed": "Persian"
}
}
}
}
Verify that cats are there:
GET grille_badge/_search
EDIT & ELABORATION
Here's how you do it:
POST grille_badge/_update/1
{
"script": {
"source": """
def theme_target = ctx._source.theme.find(theme -> theme.name == params.current_theme);
def categorie_target = theme_target.categories.find(categories -> categories.name == params.current_categorie);
def niveau_target = categorie_target.niveaux.find(niveaux -> niveaux.name == params.current_niveau);
def grille_target = niveau_target.grille_defis.find(grille_defis -> grille_defis.name == params.current_grille);
grille_target['defis'].add(["name" : params['defis_ajouter']]);
""",
"params": {
"current_theme": "foot",
"current_categorie": "categorie1",
"current_niveau": "bronze",
"current_grille": "grille1",
"defis_ajouter": "i am a defis"
}
}
}
Note the proper arrow notation in the findAll methods.
Also note that your own mapping defined the innermost defis as a nested array of key-value pairs -- this means you cannot push a plain string using the .add method -- you need to use a HashMap. You instatiate one with the one liner ["your_key" : "your_value"]. Good luck.

How to query in parent-child relation in elasticsearch

I have parent-child relation for Customer(Parent) and PromotionCustomer(Child).
Customer data:
{
"id": "b7818d4d-566e-24f5-89d2-3995bb97fd5e",
**"externalId": "9200191",**
"name": "LOBLAW NFLD",
"fullName": "LOBLAW NFLD",
"businessSystem": "EBJ/001"
}
PromotionCustomer data:
{
"id" : "31f2e065-a046-9c3a-808b-83545ddb07d1",
"externalId" : "T-000195542",
"businessSystem" : "EBJ/001",
"promotionDescription" : "PM-RT-LOBLAW NFLD-BB",
"promotionType" : "Bill Back",
"promotionStatus" : "Approved",
**"promotionCustomer" : "9200191",**
"validFrom" : "02/28/2019",
"validTo" : "03/20/2019",
"promotionDateTypeCode" : "1"
}
This the mapping details(schema)
{
"promotionsearch" : {
"mappings" : {
"properties" : {
"_class" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"customer" : {
"type" : "nested",
"include_in_parent" : true,
"properties" : {
"businessSystem" : {
"type" : "keyword"
},
"externalId" : {
"type" : "keyword"
},
"fullName" : {
"type" : "text"
},
"id" : {
"type" : "text"
},
"name" : {
"type" : "text"
}
}
},
"id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"promotioncustomer" : {
"type" : "nested",
"include_in_parent" : true,
"properties" : {
"businessSystem" : {
"type" : "keyword"
},
"externalId" : {
"type" : "keyword"
},
"id" : {
"type" : "text"
},
"promotionCustomer" : {
"type" : "text"
},
"promotionDateTypeCode" : {
"type" : "keyword"
},
"promotionDescription" : {
"type" : "text"
},
"promotionProducts" : {
"type" : "text"
},
"promotionStatus" : {
"type" : "keyword"
},
"promotionType" : {
"type" : "keyword"
},
"validFrom" : {
"type" : "date",
"format" : "MM/dd/yyyy"
},
"validTo" : {
"type" : "date",
"format" : "MM/dd/yyyy"
}
}
},
"promotionjoin" : {
"type" : "join",
"eager_global_ordinals" : true,
"relations" : {
"customer" : "promotioncustomer"
}
},
"routing" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
}
I need to get the get the promotionCustomer data based on the externalId which is the customerId which maps to promotionCustomer along with the date range.
I've written a query as below.
GET /promotionsearch/_search
{
"query": {
"has_parent": {
"parent_type": "customer",
"inner_hits": {},
"query": {
"has_child": {
"type": "promotioncustomer",
"query": {
"bool": {
"must": [
{
"match": {
"customer.externalId": "9200191"
}
},
{
"range": {
"promotioncustomer.validFrom": {
"gte": "02/28/2019"
}
}
},
{
"range": {
"promotioncustomer.validTo": {
"lte": "03/20/2019"
}
}
}
]
}
}
}
}
}
}
}
But it's not yielding the result. I know the reason as well. In the has_child clause i'm making use of parent field i.e "customer.externalId". Is there a way to include/add this condition in the parent_type query and then based on the result apply range condition inside has_child
I was able to get the solution. Hope it helps others as well. No need make multiple query call to ES inorder get the desired result.
GET /promotionsearch/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"promotioncustomer.promotionDescription": "PM-RT"
}
},
{
"range": {
"promotioncustomer.validFrom": {
"gte": "02/28/2019"
}
}
},
{
"range": {
"promotioncustomer.validTo": {
"lte": "03/28/2019"
}
}
},
{
"has_parent": {
"parent_type": "customer",
"query": {
"match": {
"customer.fullName": "sdjhfb"
}
}
}
}
]
}
}
}

Elasticsearch: filter documents by array passed in request contains all document array elements

My documents stored in elasticsearch have following structure:
{
"id": 1,
"test": "name",
"rules": [
{
"id": 2,
"name": "rule1",
"ruleDetails": [
{
"id": 3,
"requiredAnswerId": 1
},
{
"id": 4,
"requiredAnswerId": 2
},
{
"id": 5,
"requiredAnswerId": 3
}
]
}
]
}
where, rules property has nested type.
I need to query documents by checking that array of requiredAnswerId passed in the search request (provided terms) contains all rules.ruleDetails.requiredAnswerId stored in the document.
Does anyone know which elasticsearch option I can use to build such specific query? Or maybe, it is better to fetch the whole document and perform filtering on the application level.
UPDATED
Adding mapping
{
"my_index": {
"mappings": {
"properties": {
"id": {
"type": "long"
},
"test": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"rules": {
"type": "nested",
"properties": {
"id": {
"type": "long"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ruleDetails": {
"properties": {
"id": {
"type": "long"
},
"requiredAnswerId": {
"type": "long"
}
}
}
}
}
}
}
}
}
Mapping:
{
"index4" : {
"mappings" : {
"properties" : {
"id" : {
"type" : "integer"
},
"rules" : {
"type" : "nested",
"properties" : {
"id" : {
"type" : "integer"
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword"
}
}
},
"ruleDetails" : {
"properties" : {
"id" : {
"type" : "long"
},
"requiredAnswerId" : {
"type" : "long"
}
}
}
}
},
"test" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword"
}
}
}
}
}
}
}
Query: This will need use of scripts which are not good from performance perspective. I am looping through all documents and checking if field is present is passed parameters
{
"query": {
"nested": {
"path": "rules",
"query": {
"script": {
"script": {
"source": "for(a in doc['rules.ruleDetails.requiredAnswerId']){if(!params.Ids.contains((int)a)) return false; } return true;",
"params": {
"Ids": [
1,
2,
3
]
}
}
}
},
"inner_hits": {}
}
}
}
Result:
"hits" : [
{
"_index" : "index4",
"_type" : "_doc",
"_id" : "TxOpvnEBf42mOjxvvLQB",
"_score" : 4.0,
"_source" : {
"id" : 1,
"test" : "name",
"rules" : [
{
"id" : 2,
"name" : "rule1",
"ruleDetails" : [
{
"id" : 3,
"requiredAnswerId" : 1
},
{
"id" : 4,
"requiredAnswerId" : 2
},
{
"id" : 5,
"requiredAnswerId" : 3
}
]
},
{
"id" : 3,
"name" : "rule3",
"ruleDetails" : [
{
"id" : 3,
"requiredAnswerId" : 1
},
{
"id" : 4,
"requiredAnswerId" : 2
}
]
}
]
},
"inner_hits" : {
"rules" : {
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 4.0,
"hits" : [
{
"_index" : "index4",
"_type" : "_doc",
"_id" : "TxOpvnEBf42mOjxvvLQB",
"_nested" : {
"field" : "rules",
"offset" : 0
},
"_score" : 4.0,
"_source" : {
"id" : 2,
"name" : "rule1",
"ruleDetails" : [
{
"id" : 3,
"requiredAnswerId" : 1
},
{
"id" : 4,
"requiredAnswerId" : 2
},
{
"id" : 5,
"requiredAnswerId" : 3
}
]
}
}
]
}
}
}
}
]
EDIT 1
Terms_set can be used as an alternative. It will be faster compared to script query
Returns documents that contain a minimum number of exact terms in a
provided field.
minimum_should_match_script- size of array can be used to match the minimum number of passed values.
Query:
{
"query": {
"nested": {
"path": "rules",
"query": {
"bool": {
"filter": {
"terms_set": {
"rules.ruleDetails.requiredAnswerId": {
"terms": [
1,
2,
3
],
"minimum_should_match_script": {
"source": "doc['rules.ruleDetails.requiredAnswerId'].size()"
}
}
}
}
}
},
"inner_hits": {}
}
}
}
After some time playing with ES and reading its documentation, I found that you should keep in mind that provided script should be compiled and applied for the document, hence it will be slower, if you just know the required number elements that should match in advance.
Therefore, I created a separate field requiredMatches that stores the number of rules.ruleDetails.requiredAnswerId elements for every document and calculate it before indexing document. Then, instead of using minimum_should_match_script in my search query, I am using minimum_should_match_field:
{
"query": {
"nested": {
"path": "rules",
"query": {
"bool": {
"filter": {
"terms_set": {
"rules.ruleDetails.requiredAnswerId": {
"terms": [
1,
2,
3
],
"minimum_should_match_field": "requiredMatches"
}
}
}
}
},
"inner_hits": {}
}
}
}
I used, following example, as a reference

Elasticsearch - Conditional nested fetching

I have index mapping:
{
"dev.directory.3" : {
"mappings" : {
"profile" : {
"properties" : {
"email" : {
"type" : "string",
"index" : "not_analyzed"
},
"events" : {
"type" : "nested",
"properties" : {
"id" : {
"type" : "integer"
},
"name" : {
"type" : "string",
"index" : "not_analyzed"
},
}
}
}
}
}
}
}
with data:
"hits" : [ {
"_index" : "dev.directory.3",
"_type" : "profile",
"_id" : "1",
"_score" : 1.0,
"_source" : {
"email" : "test#dummy.com",
"events" : [
{
"id" : 111,
"name" : "ABC",
},
{
"id" : 222,
"name" : "DEF",
}
],
}
}]
I'd like to filter only matched nested elements instead of returning all events array - is this possible in ES?
Example query:
{
"nested" : {
"path" : "events",
"query" : {
"bool" : {
"filter" : [
{ "match" : { "events.id" : 222 } },
]
}
}
}
}
Eg. If I query for events.id=222 there should be only single element on the result list returned.
What strategy for would be the best to achieve this kind of requirement?
You can use inner_hits to only get the nested records which matched the query.
{
"query": {
"nested": {
"path": "events",
"query": {
"bool": {
"filter": [
{
"match": {
"events.id": 222
}
}
]
}
},
"inner_hits": {}
}
},
"_source": false
}
I am also excluding the source to get only nested hits

Resources