Copy field value to a new field in existing index - elasticsearch

I have a document that has the structure with an field object with a nested field internally. The nested field is responsible for storing all interactions that occurred in an internal communication.
It happens that I need to create a new field inside the nested field, with a new type that will now be used to store the old field with a new parser.
How can I copy the data from the old field to the new field inside the nested field?
My document:
curl -XPUT 'localhost:9200/problems?pretty' -H 'Content-Type: application/json' -d '
{
"settings": {
"number_of_shards": 1
},
"mappings": {
"problem": {
"properties": {
"problemid": {
"type": "long"
},
"subject": {
"type": "text",
"index": true
},
"usermessage": {
"type": "object",
"properties": {
"content": {
"type": "nested",
"properties": {
"messageid": {
"type": "long",
"index": true
},
"message": {
"type": "text",
"index": true
}
}
}
}
}
}
}
}
}'
My New Field:
curl -XPUT 'localhost:9200/problems/_mapping/problem?pretty' -H 'Content-Type: application/json' -d '
{
"properties": {
"usermessage": {
"type": "object",
"properties": {
"content": {
"type": "nested",
"properties": {
"message_accents" : {
"type" : "text",
"analyzer" : "ignoreaccents"
}
}
}
}
}
}
}
'
Data Example:
{
"problemid": 1,
"subject": "Test",
"usermessage": [
{
"messageid": 1
"message": "Hello"
},
{
"messageid": 2
"message": "Its me"
},
]
}'
My script to copy fields:
curl -XPOST 'localhost:9200/problems/_update_by_query' -H 'Content-Type: application/json' -d '
{
"query": {
"match_all": {
}
},
"script": "ctx._source.usermessage.content.message_accents = ctx._source.usermessage.content.message"
}'
I tried the code below but it didn't work, it returns an error.
curl -XPOST 'localhost:9200/problems/_update_by_query' -H 'Content-Type: application/json' -d '
{
"query": {
"match_all": {
}
},
"script": "ctx._source.usermessage.content.each { elm -> elm.message_accents = elm.message }"
}
'
Error:
"script":"ctx._source.usermessage.content.each { elm -> elm.message_accents = elm.message }","lang":"painless","caused_by":{"type":"illegal_argument_exception","reason":"unexpected token ['{'] was expecting one of [{, ';'}]."}},"status":500}%

Related

error in elasticserach while importing single line via json

ubuntu#ip-172-31-5-121:~$ curl -XPUT localhost:9200/movies -d'
{
"mapping": {
"properties": {
"year": {
"type": "date"
}
}
}
}'
{"error":{"root_cause":[{"type":"parse_exception","reason":"unknown key [mapping] for create index"}],"type":"parse_exception","reason":"unknown key [mapping] for create index"},"status":400}
Tldr;
Be careful with typos. "mappings" takes an S
To solve
curl -XPUT localhost:9200/movies -d'
{
"mappings": {
"properties": {
"year": {
"type": "date"
}
}
}
}
'

Elassandra: UDT List Match Query- No Results

I am using Elassandra. In Cassandra, I have a UDT:
CREATE TYPE test.entity_attributes (
attribute_key text,
attribute_value text
);
It is used in table
CREATE TABLE test.attributes_test (
id text PRIMARY KEY,
attr list<frozen<entity_attributes>>
)
I mapped the attributes_test using:
curl --location --request PUT 'localhost:9200/attr_index' \
--header 'Content-Type: application/json' \
--data-raw '{
"settings": { "keyspace": "test" },
"mappings": {
"attributes_test" : {
"discover":".*"
}
}
}'
(copied from postman)
It returns the following as mapping:
{
"attr_index": {
"aliases": {},
"mappings": {
"attributes_test": {
"properties": {
"attr": {
"type": "nested",
"cql_udt_name": "entity_attributes",
"properties": {
"attribute_key": {
"type": "keyword",
"cql_collection": "singleton"
},
"attribute_value": {
"type": "keyword",
"cql_collection": "singleton"
}
}
},
"id": {
"type": "keyword",
"cql_collection": "singleton",
"cql_partition_key": true,
"cql_primary_key_order": 0
}
}
}
},
"settings": {
"index": {
"keyspace": "test",
"number_of_shards": "1",
"provided_name": "attr_index",
"creation_date": "1615291749532",
"number_of_replicas": "0",
"uuid": "Oua1ACLbRvCATC-kcGPoQg",
"version": {
"created": "6020399"
}
}
}
}
}
This is what I have in the table:
id | attr
----+----------------------------------------------------------------------------------------------
2 | [{attribute_key: 'abc', attribute_value: '2'}, {attribute_key: 'def', attribute_value: '1'}]
1 | [{attribute_key: 'abc', attribute_value: '1'}]
The problem now is, when I run the following query, it does not return any result.
curl --location --request POST 'localhost:9200/attr_index/_search' \
--header 'Content-Type: application/json' \
--data-raw '{
"query": {
"match": {
"attr.attribute_key": "abc"
}
}
}'
https://www.elastic.co/guide/en/elasticsearch/reference/current/nested.html - describes the way to search in nested objects.
How can I search in the list of nested objects?
It was a mistake in my query. The correct query would be:
{
"query": {
"nested": {
"path": "attr",
"query": {
"match": {
"attr.attribute_key": "abc"
}
}
}
}
}

Kibana No results found for my index pattern

I insert this pattern to find my object when I search in kibana
curl -XPUT 'localhost:9200/exception_index?pretty' -H 'Content-Type: application/json' -d'
{
"mappings": {
"siam_exception": {
"dynamic": "true",
"properties": {
"Title": { "type": "text" },
"Date&Time": { "type": "text" },
"Tags": { "type": "text" },
"Level": { "type": "text" },
"Message": {
"dynamic": "true",
"properties": {
"Root": { "type": "text" },
"ExceptionList": { "type": "text" }
}
}
}
}
}
}
'
and this is my exception.log
"siam_exception":{
"Title": "exception",
"Date&Time": "2018-01-21 09:52:20.759950",
"Tags": "SiamCore",
"Level": "Fatal",
"Message":[
{
"Root": "( ['MQROOT' : 0x7f0a902b2d80]
(0x01000000:Name ):Properties = ( ['MQPROPERTYPARSER' : 0x7f0a902bffa0]
(0x03000000:NameValue):MessageSet = 'SIAMCOMMONMSG' (CHARACTER)",
"ExceptionList": "(0x03000000:NameValue):ReplyIdentifier = X'000000000000000000000000000000000000000000000000' (BLOB)","
}
]
}
but kibana find nothing when I search *
even if I omit ' "siam_exception": ' from first line of exception.log nothing change
what is my problem

Elasticsearch: Parent-child relationship after rollover

Suppose there is a simple blog index which contains two types: blog and comment. One blog can have multiple comments. The index is created like this
curl -X PUT \
'http://localhost:9200/%3Cblog-%7Bnow%2Fd%7D-000001%3E?pretty=' \
-H 'content-type: application/json' \
-d '{
"mappings": {
"comment": {
"_parent": { "type": "blog" },
"properties": {
"name": { "type": "keyword" },
"comment": { "type": "text" }
}
},
"blog": {
"properties": {
"author": { "type": "keyword" },
"subject": { "type": "text" },
"content": { "type": "text" }
}
}
}
}'
The index %3Cblog-%7Bnow%2Fd%7D-000001%3E is equal to <blog-{now/d}-000001> (see here for more about date math).
We're going to add 'blog-active' alias to this index. This alias is going to be used for storing data.
curl -X POST 'http://localhost:9200/_aliases?pretty=' \
-H 'content-type: application/json' \
-d '{ "actions" : [ { "add" : { "index" : "blog-*", "alias" : "blog-active" } } ] }'
Now if we do the following actions:
1.Add a blog using blog-active alias
curl -X POST http://localhost:9200/blog-active/blog/1 \
-H 'content-type: application/json' \
-d '{
"author": "author1",
"subject": "subject1",
"content": "content1"
}'
2.Add a comment to the blog
curl -X POST \
'http://localhost:9200/blog-active/comment/1?parent=1' \
-H 'content-type: application/json' \
-d '{
"name": "commenter1",
"comment": "new comment1"
}'
3.Do a rollover with max_docs = 2
curl -X POST \
http://localhost:9200/blog-active/_rollover \
-H 'content-type: application/json' \
-d '{
"conditions": {
"max_docs": 2
},
"mappings": {
"comment": {
"_parent": { "type": "blog" },
"properties": {
"name": { "type": "keyword" },
"comment": { "type": "text" }
}
},
"blog": {
"properties": {
"author": { "type": "keyword" },
"subject": { "type": "text" },
"content": { "type": "text" }
}
}
}
}'
4.And add another comment to the blog
curl -X POST \
'http://localhost:9200/blog-active/comment/1?parent=1' \
-H 'content-type: application/json' \
-d '{
"name": "commenter2",
"comment": "new comment2"
}'
Now if we search all blog indices for all comments on 'author1' blogs with (blog-%2A is blog-*)
curl -X POST \
http://localhost:9200/blog-%2A/comment/_search \
-H 'content-type: application/json' \
-d '{
"query": {
"has_parent" : {
"query" : {
"match" : { "author" : { "query" : "author1" } }
},
"parent_type" : "blog"
}
}
}'
the result only contains first comment.
This is due to the fact that second comment is in the second index which does not have parent blog document in itself. So it doesn't know about the author of the blog.
So, my question is how do I approach parent-child relations when rollover is used?
Is the relationship even possible in that case?
Similar question: ElasticSearch parent/child on different indexes
All documents that form part of a parent-child relationship need to live in the same index, more preciously same shard. Therefore it's not possible to have parent-child relationship if rollover is used, since it creates new indices.
One solution for the problem above could be to denormalize data by adding filed blog_author and blog_id in comment type. The mapping in that case will look like this (notice that parent-child relationship has been removed):
"mappings": {
"comment": {
"properties": {
"blog_id": { "type": "keyword" },
"blog_author": { "type": "keyword" },
"name": { "type": "keyword" },
"comment": { "type": "text" }
}
},
"blog": {
"properties": {
"author": { "type": "keyword" },
"subject": { "type": "text" },
"content": { "type": "text" }
}
}
}
and the query to fetch comments by blog author is:
curl -X POST \
http://localhost:9200/blog-%2A/comment/_search \
-H 'cache-control: no-cache' \
-H 'content-type: application/json' \
-d '{
"query": {
"match": {
"blog_author": "user1"
}
}
}'

accessing _id or _parent fields in script query in elasticsearch

when writing a search query with a script, I can access fields using "doc['myfield']"
curl -XPOST 'http://localhost:9200/index1/type1/_search' -d '
{
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"script": {
"script": "doc[\"myfield\"].value>0",
"params": {},
"lang":"python"
}
}
}
}
}'
how do I go about accessing the _id or _parent fields?
The "ctx" object does not seem to be available in a search query (while it is accessible in an update API request, why?).
Mind you, I am using the python language instead of mvel, but both of them pose the same question.
By default, both document id and parent id are indexed in uid format: type#id. Elasticsearch provides a few methods that can be used to extract type and id from uid string. Here is an example of using these methods in MVEL:
curl -XDELETE localhost:9200/test
curl -XPUT localhost:9200/test -d '{
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 0
},
"mappings": {
"doc": {
"properties": {
"name": {
"type": "string"
}
}
},
"child_doc": {
"_parent": {
"type": "doc"
},
"properties": {
"name": {
"type": "string"
}
}
}
}
}'
curl -XPUT "localhost:9200/test/doc/1" -d '{"name": "doc 1"}'
curl -XPUT "localhost:9200/test/child_doc/1-1?parent=1" -d '{"name": "child 1-1 of doc 1"}'
curl -XPOST "localhost:9200/test/_refresh"
echo
curl "localhost:9200/test/child_doc/_search?pretty=true" -d '{
"script_fields": {
"uid_in_script": {
"script": "doc[\"_uid\"].value"
},
"id_in_script": {
"script": "org.elasticsearch.index.mapper.Uid.idFromUid(doc[\"_uid\"].value)"
},
"parent_uid_in_script": {
"script": "doc[\"_parent\"].value"
},
"parent_id_in_script": {
"script": "org.elasticsearch.index.mapper.Uid.idFromUid(doc[\"_parent\"].value)"
},
"parent_type_in_script": {
"script": "org.elasticsearch.index.mapper.Uid.typeFromUid(doc[\"_parent\"].value)"
}
}
}'
echo

Resources