elasticsearch:use script to update nested field? - elasticsearch

I want to add a object into the nested field every update time.
For example,I have a doc:
{
"test":[{"remark":"remark1"}]
}
Next time,i want to add a remark object into test field and save the old remark objects.And the result is :
{
"test":[{"remark":"remark1"},{"remark":"remark2"}]
}
How to achieve it?
Edit
I use the script:
{
"script": "ctx._source.test= ((ctx._source.test?: []) += remarkItem)",
"params": {
"remarkItem": {
"remark": "addd"
}
}
}
But,i get the exception:
{
"error": {
"root_cause": [
{
"type": "remote_transport_exception",
"reason": "[es77][10.14.84.77:9300][indices:data/write/update[s]]"
}
],
"type": "illegal_argument_exception",
"reason": "failed to execute script",
"caused_by": {
"type": "script_exception",
"reason": "Failed to compile inline script [ctx._source.test= ((ctx._source.test?: []) += remarkItem)] using lang [groovy]",
"caused_by": {
"type": "script_exception",
"reason": "failed to compile groovy script",
"caused_by": {
"type": "multiple_compilation_errors_exception",
"reason": "startup failed:\na8220b2cf14b8b7ebeead7f068416882d04fa25d: 1: \nclass org.codehaus.groovy.ast.expr.ElvisOperatorExpression, with its value '(ctx._source.test) ? ctx._source.test: []', is a bad expression as the left hand side of an assignment operator at line: 1 column: 82. File: a8220b2cf14b8b7ebeead7f068416882d04fa25d # line 1, column 82.\n CILastCallResultRemark ?: []) += remarkI\n ^\n\n1 error\n"
}
}
}
},
"status": 400
}
edit
Now,i want to add a field to ensure update or insert the object.
For example:
{
"test":[{"remark":"remark1","id":"1"}]
}
When i update the field,when the id exist,i will update the object.On the contrary,i will insert the object.

I suggest to try a script like this, which takes two parameters in argument. It will check if any of the nested objects already contains the given id:
if yes, it will update the given remark
if not, it will insert a new nested object in the test array.
The script goes like this:
def updated = false
ctx._source.test?.each { obj ->
if (obj.id == item.id) {
obj.remark = item.remark
updated = true
}
}
if (!updated) {
ctx._source.test = ((ctx._source.test ?: []) + item)
}
After being inlined and with proper semicolons, the script looks like this:
{
"script": "def updated = false; ctx._source.test?.each { obj -> if (obj.id == item.id) { obj.remark = item.remark; updated = true } }; if (!updated) { ctx._source.test = ((ctx._source.test ?: []) + item)}",
"params": {
"item": {
"remark": "addd",
"id": "1"
}
}
}

Related

Elasticsearch ingest pipeline: how to recursively modify values in a HashMap

Using an ingest pipeline, I want to iterate over a HashMap and remove underscores from all string values (where underscores exist), leaving underscores in the keys intact. Some values are arrays that must further be iterated over to do the same modification.
In the pipeline, I use a function to traverse and modify the values of a Collection view of the HashMap.
PUT /_ingest/pipeline/samples
{
"description": "preprocessing of samples.json",
"processors": [
{
"script": {
"tag": "remove underscore from sample_tags values",
"source": """
void findReplace(Collection collection) {
collection.forEach(element -> {
if (element instanceof String) {
element.replace('_',' ');
} else {
findReplace(element);
}
return true;
})
}
Collection samples = ctx.samples;
samples.forEach(sample -> { //sample.sample_tags is a HashMap
Collection sample_tags = sample.sample_tags.values();
findReplace(sample_tags);
return true;
})
"""
}
}
]
}
When I simulate the pipeline ingestion, I find the string values are not modified. Where am I going wrong?
POST /_ingest/pipeline/samples/_simulate
{
"docs": [
{
"_index": "samples",
"_id": "xUSU_3UB5CXFr25x7DcC",
"_source": {
"samples": [
{
"sample_tags": {
"Entry_A": [
"A_hyphentated-sample",
"sample1"
],
"Entry_B": "A_multiple_underscore_example",
"Entry_C": [
"sample2",
"another_example_with_underscores"
],
"Entry_E": "last_example"
}
}
]
}
}
]
}
\\Result
{
"docs" : [
{
"doc" : {
"_index" : "samples",
"_type" : "_doc",
"_id" : "xUSU_3UB5CXFr25x7DcC",
"_source" : {
"samples" : [
{
"sample_tags" : {
"Entry_E" : "last_example",
"Entry_C" : [
"sample2",
"another_example_with_underscores"
],
"Entry_B" : "A_multiple_underscore_example",
"Entry_A" : [
"A_hyphentated-sample",
"sample1"
]
}
}
]
},
"_ingest" : {
"timestamp" : "2020-12-01T17:29:52.3917165Z"
}
}
}
]
}
Here is a modified version of your script that will work on the data you provided:
PUT /_ingest/pipeline/samples
{
"description": "preprocessing of samples.json",
"processors": [
{
"script": {
"tag": "remove underscore from sample_tags values",
"source": """
String replaceString(String value) {
return value.replace('_',' ');
}
void findReplace(Map map) {
map.keySet().forEach(key -> {
if (map[key] instanceof String) {
map[key] = replaceString(map[key]);
} else {
map[key] = map[key].stream().map(this::replaceString).collect(Collectors.toList());
}
});
}
ctx.samples.forEach(sample -> {
findReplace(sample.sample_tags);
return true;
});
"""
}
}
]
}
The result looks like this:
{
"samples" : [
{
"sample_tags" : {
"Entry_E" : "last example",
"Entry_C" : [
"sample2",
"another example with underscores"
],
"Entry_B" : "A multiple underscore example",
"Entry_A" : [
"A hyphentated-sample",
"sample1"
]
}
}
]
}
You were on the right path but you were working on copies of values and weren't setting the modified values back onto the document context ctx which is eventually returned from the pipeline. This means you'll need to keep track of the current iteration indexes -- so for the array lists, as for the hash maps and everything in between -- so that you can then target the fields' positions in the deeply nested context.
Here's an example taking care of strings and (string-only) array lists. You'll need to extend it to handle hash maps (and other types) and then perhaps extract the whole process into a separate function. But AFAIK you cannot return multiple data types in Java so it may be challenging...
PUT /_ingest/pipeline/samples
{
"description": "preprocessing of samples.json",
"processors": [
{
"script": {
"tag": "remove underscore from sample_tags values",
"source": """
ArrayList samples = ctx.samples;
for (int i = 0; i < samples.size(); i++) {
def sample = samples.get(i).sample_tags;
for (def entry : sample.entrySet()) {
def key = entry.getKey();
def val = entry.getValue();
def replaced_val;
if (val instanceof String) {
replaced_val = val.replace('_',' ');
} else if (val instanceof ArrayList) {
replaced_val = new ArrayList();
for (int j = 0; j < val.length; j++) {
replaced_val.add(val[j].replace('_',' '));
}
}
// else if (val instanceof HashMap) {
// do your thing
// }
// crucial part
ctx.samples[i][key] = replaced_val;
}
}
"""
}
}
]
}

How to update a List of Maps in DynamoDB table via boto3?

is there any way to update/change value of a attribute as map nested in a list. My table looks something like this
{
"MainId": {
"S": "8981d9d0-373d-4f30-8084-cc4e18f9855f"
},
"date": {
"S": "8/11/2020"
},
"user_id": {
"S": "xyz#gmail.com"
},
"role": {
"S": "normal"
},
"finale": {
"L": [
{
"M": {
"Hatch_type": {
"S": "P7684"
}
I would like to change the update the value of 'Hatch Type' map in the 'finale' list. Please note that the provided table is not complete.
My current code looks like
import json
import boto3
from datetime import datetime
#TABLE_NAME = "table1"
def lambda_handler(event, context):
dynamodb = boto3.resource('dynamodb', region_name="us-east-2")
client = boto3.client('dynamodb', region_name="us-east-2")
TABLE_NAME = dynamodb.Table('table1')
response = TABLE_NAME.update_item(
Key={
'MainId':'8981d9d0-373d-4f30-8084-cc4e18f9855f'
}
UpdateExpression='SET #finale[0].#Hatch_type = :newvalue",
ExpressionAttributeNames = {
??
},
ExpressionAttributeValues = {
???
},
ReturnValues="UPDATED_NEW"
)
I'm stuck at this point and not sure how to proceed further. Please help.
ExpressionAttributeNames = {
':newvalue': "P7686 //whatever you want to update it io"
},
ExpressionAttributeValues = {
'#Hatch_type': " Hatch_type"
},

Painless scripting initialize new array

I'm trying to add or update a nested object in Elasticsearch using a script. Below script works fine if integrations is already an array, but when it is null, below script throws a null pointer exception. How do I initialize ctx._source.integrations to be an empty array if its value is null? (Something like the equivalent of JavaScript's myObject.integrations = myObject.integrations ?? [])
POST /products/_update/VFfrnQrKlC5bwdfdeaQ7
{
"script": {
"source": """
ctx._source.integrations.removeIf(i -> i.id == params.integration.id);
ctx._source.integrations.add(params.integration);
ctx._source.integrationCount = ctx._source.integrations.length;
""",
"params": {
"integration": {
"id": "dVTV8GjHj8pXFnlYUUlI",
"from": true,
"to": false,
"vendor": "sfeZWDpZXlF5Qa8mUsiF",
"targetProduct": {
"id": "nyILhphvCrGYm53cfaOx",
"name": "Test Product",
"categoryIds": []
}
}
}
}
}
ok i think this does the trick:
if (ctx._source.integrations == null) {
ctx._source.integrations = new ArrayList();
}
is there a short hand to this like in the JS example?

loopback REST API filter by nested data

I would like to filter from REST API by nested data. For example this object:
[
{
"name": "Handmade Soft Fish",
"tags": "Rubber, Rubber, Salad",
"categories": [
{
"name": "women",
"id": 2,
"parent_id": 0,
"permalink": "/women"
},
{
"name": "kids",
"id": 3,
"parent_id": 0,
"permalink": "/kids"
}
]
},
{
"name": "Tasty Rubber Soap",
"tags": "Granite, Granite, Chair",
"categories": [
{
"name": "kids",
"id": 3,
"parent_id": 0,
"permalink": "/kids"
}
]
}
]
is comming by GET /api/products?filter[include]=categories
and i would like to get only products which has category name "women". How do this?
LoopBack does not support filters based on related models.
This is a limitation that we have never had bandwidth to solve, unfortunately :(
For more details, see the discussion and linked issues here:
Filter on level 2 properties: https://github.com/strongloop/loopback/issues/517
Filter by properties of related models (use SQL JOIN in queries): https://github.com/strongloop/loopback/issues/683
Maybe you want to get this data by the Category REST API. For example:
GET /api/categories?filter[include]=products&filter[where][name]=woman
The result will be a category object with all products related. To this, will be necessary declare this relation on the models.
Try like this.It has worked for me.
const filter = {
where: {
'categories.name': {
inq: ['women']**strong text**
}
}
};
Pass this filter to request as path parameters and the request would be like bellow
GET /api/categoriesfilter=%7B%22where%22:%7B%categories.name%22:%7B%22inq%22:%5B%women%22%5D%7D%7D%7D
Can you share how it looks like without filter[include]=categorie, please ?
[edit]
after a few questions in comment, I'd build a remote method : in common/models/myModel.js (inside the function) :
function getItems(filter, categorieIds = []) {
return new Promise((resolve, reject) => {
let newInclude;
if (filter.hasOwnProperty(include)){
if (Array.isArray(filter.include)) {
newInclude = [].concat(filter.include, "categories")
}else{
if (filter.include.length > 0) {
newInclude = [].concat(filter.include, "categories");
}else{
newInclude = "categories";
}
}
}else{
newInclude = "categories";
}
myModel.find(Object.assign({}, filter, {include: newInclude}))
.then(data => {
if (data.length <= 0) return resolve(data);
if (categoriesIds.length <= 0) return resolve(data);
// there goes your specific filter on categories
const tmp = data.filter(
item => item.categories.findIndex(
categorie => categorieIds.indexOf(categorie.id) > -1
) > -1
);
return resolve(tmp);
})
}
}
myModel.remoteMethod('getItems', {
accepts: [{
arg: "filter",
type: "object",
required: true
}, {
arg: "categorieIds",
type: "array",
required: true
}],
returns: {arg: 'getItems', type: 'array'}
});
I hope it answers your question...

Loopback : Validate model from another model is not returning proper error message

I am validating model from another model like below
Model.addFavorite = function (data, callbackFn) {
if (data) {
var faviroteModel = this.app.models.Favorite;
var objFavorite = new faviroteModel(data);
objFavorite.isValid(function (isValid) {
if (isValid) {
callbackFn(null, objFavorite);
}
else {
callbackFn(objFavorite.errors);
}
});
}
else callbackFn("Post data required", {});
}
If i do this then I am getting error like below
{
"error": {
"statusCode": 500,
"t": [
"is not a valid date"
]
}
}
It should be with error message like below
{
"error": {
"statusCode": 422,
"name": "ValidationError",
"message": "The `Favorite` instance is not valid. Details: `t` is not a valid date (value: Invalid Date).",
"details": {
"context": "Favorite",
"codes": {
"t": [
"date"
]
},
"messages": {
"t": [
"is not a valid date"
]
}
}
}
}
Can anyone tell me what am i missing here.
How can i achieve this.
https://github.com/strongloop/loopback-datasource-juggler/blob/master/lib/validations.js#L843
You might run into situations where you need to raise a validation
error yourself, for example in a "before" hook or a custom model
method.
if (model.isValid()) {
return callback(null, { success: true });
}
// This line shows how to create a ValidationError
var err = new MyModel.ValidationError(model);
callback(err);
}

Resources