ElasticSearch Dynamic Template for nested - elasticsearch

I try to create dynamic template for a nested object.
Here is a document to index :
{
"title": "My title",
"attributes": {
"color": {
"id": 42,
"label": "red"
},
"material": {
"id": 43,
"label": "textile"
}
}
}
This is the template i tried, without success
{
"dynamic": "false",
"dynamic_templates": [
{
"attributes": {
"path_match": "attributes",
"mapping": {
"type": "nested"
}
}
},
{
"attributes_nested": {
"path_match": "attributes.*",
"mapping": {
"properties": {
"id": {
"type": "integer"
},
"value": {
"type": "string"
}
}
}
}
}
],
"properties": {
"title": {
"type": "string"
}
}
}
I'd like to be able to make aggregations on attributes.color.id, and attributes.material.id


Nevermind, the problem was that i had
{ "dynamic": false}
The correct mapping is
{
"dynamic": "false",
"dynamic_templates": [
{
"attributes_nested": {
"path_match": "attributes.*",
"mapping": {
"properties": {
"id": {
"type": "integer"
},
"value": {
"type": "string"
}
}
}
}
}
],
"properties": {
"title": {
"type": "string"
},
"attributes": {
"type": "nested",
"dynamic": true
}
}
}

Related

Reindex parent/child relationship from 2.x to 6.x in elasticsearch

I need way to re-index parent/child data from 2.4 to 6.8 Join. The old index has multiple entities.
old-index mapping:
{
"mappings": {
"parentType": {
"properties": {
"field_1": {
"type": "text"
},
"field_2": {
"type": "text"
},
"field_3": {
"type": "keyword"
},
"filed_4": {
"type": "long"
}
}
},
"child_type_1": {
"_parent": {
"type": "case"
},
"_routing": {
" required": true
},
"child1_field_1": {
"type": "text"
},
"child1_field_2": {
"type": "text"
},
"child1_field_3": {
"type": "keyword"
}
},
"child_type_2": {
"_parent": {
"type": "parentType"
},
"_routing": {
"required": true
},
"child2_field_1": {
"type": "text"
},
"child2_field_2": {
"type": "text"
}
}
}
}
I want to transform it into to the following 6.8 mapping:
{
"mappings": {
"doc": {
"properties": {
"parentType": {
"properties": {
"field_1": {
"type": "text"
},
"field_2": {
"type": "text"
},
"field_3": {
"type": "keyword"
},
"filed_4": {
"type": "long"
}
}
},
"child_type_1": {
"child1_field_1": {
"type": "text"
},
"child1_field_2": {
"type": "text"
},
"child1_field_3": {
"type": "keyword"
}
},
"child_type_2": {
"child2_field_1": {
"type": "text"
},
"child2_field_2": {
"type": "text"
}
},
"join_field": {
"type": "join",
"relations": {
"parentType": [
"child_type_1",
"child_type_2"
]
}
}
}
}
}
}
I know I am supposed to use the re-index API but I am not sure how exactly the script has to be written. I want to re-index all the parent and children documents into the new index where the _type is "doc"

Unable to apply new index template

I am currently trying to update an index template on Elastic Search 6.7/6.8.
Templates are stored in the code and are applied each time my API starts.
There are no errors, the request returns 200.
For example, here is a template i am currently using:
{
"index_patterns": [ "*-ec2-reports" ],
"version": 11,
"mappings": {
"ec2-report": {
"properties": {
"account": {
"type": "keyword"
},
"reportDate": {
"type": "date"
},
"reportType": {
"type": "keyword"
},
"instance": {
"properties": {
"id": {
"type": "keyword"
},
"region": {
"type": "keyword"
},
"state": {
"type": "keyword"
},
"purchasing": {
"type": "keyword"
},
"keyPair": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"platform": {
"type": "keyword"
},
"tags": {
"type": "nested",
"properties": {
"key": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"costs": {
"type": "object"
},
"stats": {
"type": "object",
"properties": {
"cpu": {
"type": "object",
"properties": {
"average": {
"type": "double"
},
"peak": {
"type": "double"
}
}
},
"network": {
"type": "object",
"properties": {
"in": {
"type": "double"
},
"out": {
"type": "double"
}
}
},
"volumes": {
"type": "nested",
"properties": {
"id": {
"type": "keyword"
},
"read": {
"type": "double"
},
"write": {
"type": "double"
}
}
}
}
},
"recommendation": {
"type": "object",
"properties": {
"instancetype": {
"type": "keyword"
},
"reason": {
"type": "keyword"
},
"newgeneration": {
"type": "keyword"
}
}
}
}
}
},
"_all": {
"enabled": false
},
"numeric_detection": false,
"date_detection": false
}
}
}
I'd like to add a new keyword field under the properties object like this :
"exampleField": {
"type": "keyword"
}
but it seems the template is not applied to existing indexes.
When data is inserted into a specific index which use the template, it is stored like this:
"exampleField": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
because the template has not been updated beforehand.
I would expect it to be like:
"exampleField": {
"type": "keyword"
}
in the index and in the template.
Does someone have any idea on how to have this result?
Thank you, Alexandre.

Completion Suggester in Elasticsearch not working

(Using ES 6.7)
I have an index and want to support search-as-you-type feature. For that, I want to try completion suggester but I'm having trouble in reindexing to change the mappings old index.
Here's the old index mappings
{
"old-index": {
"mappings": {
"doc": {
"properties": {
"content": {
"type": "text"
},
"project": {
"type": "keyword"
},
"title": {
"type": "text"
},
"version": {
"type": "keyword"
}
}
}
}
}
}
Here's the new test index mappings
PUT test-completion
{
"mappings": {
"doc": {
"properties": {
"content": {
"type": "text",
"fields": {
"autocomplete": {
"type": "completion",
"contexts": [
{
"name": "project",
"type": "category",
"path": "project"
},
{
"name": "version",
"type": "category",
"path": "version"
}
]
}
}
},
"title": {
"type": "text"
},
"project": {
"type": "keyword"
},
"version": {
"type": "keyword"
}
}
}
}
}
Here's the reindexing query
POST _reindex
{
"source": {
"index": "old-index"
},
"dest": {
"index": "test-completion"
}
}
And here's the query which returns no results
POST test-completion/_search
{
"suggest": {
"autocompletion_suggest": {
"prefix": "part of documentation",
"completion": {
"field": "content.autocomplete",
"fuzzy": {
"fuzziness": "AUTO"
},
"contexts": {
"project": "xyz-project",
"version": "abc-version"
}
}
}
}
}
If the prefix is set to a or b, it returns results outside of context.
Where I'm doing wrong?
https://discuss.elastic.co/t/problem-with-completion-suggester/181695

Kibana don't store the right data into the right fiels

Ok so now I've my mapping into kibana.
Here's my mapping:
PUT logstash-2019.05.09
{
"mappings": {
"doc": {
"properties": {
"index": {
"_index": {
"type": "keyword"
},
"_type": {
"type": "text"
}
},
"#timestamp": {
"type": "date"
},
"ip": {
"type": "ip"
},
"extension": {
"type": "text"
},
"response": {
"type": "text"
},
"geo": {
"coordinates": {
"type": "geo_point"
},
"src": {
"type": "text"
},
"dest": {
"type": "text"
},
"srcdest": {
"type": "text"
}
},
"tags": {
"type": "text"
},
"utc_time": {
"type": "date"
},
"referer": {
"type": "text"
},
"agent": {
"type": "text"
},
"clientip": {
"type": "ip"
},
"bytes": {
"type": "integer"
},
"host": {
"type": "text"
},
"request": {
"type": "text"
},
"url": {
"type": "text"
},
"#message": {
"type": "text"
},
"spaces": {
"type": "text"
},
"xss": {
"type": "text"
},
"links": {
"type": "text"
},
"relatedContent": {
"url": {
"type": "text"
},
"og:type": {
"type": "text"
},
"og:title": {
"type": "text"
},
"og:description": {
"type": ""
},
"og:url": {
"type": ""
},
"article:published_time": {
"type": "date"
},
"article:modified_time": {
"type": "date"
},
"article:section": {
"type": "keyword"
},
"article:tag": {
"type": "text"
},
"og:image": {
"type": "text"
},
"og:image:height": {
"type": "integer"
},
"og:image:width": {
"type": "integer"
},
"og:site_name": {
"type": "text"
},
"twitter:title": {
"type": "text"
},
"twitter:description": {
"type": "text"
},
"twitter:card": {
"type": "keyword"
},
"twitter:image": {
"type": "text"
},
"twitter:site": {
"type": "keyword"
}
},
"machine": {
"os": {
"type": "text"
},
"ram": {
"type": "integer"
}
},
"#version": {
"type": "integer"
}
}
}
}
}
But I don't know why, Kibana don't store the right information into the right field. He just put all the information into a message field. I think it's because I've a dynamic mapping by default, I'm not really sure. Here's the result :
Result (table)
result (json):
{
"_index": "logstash-2019.05.09",
"_type": "doc",
"_id": "9zfam2oBWngGU4Wy3Id5",
"_version": 1,
"_score": null,
"_source": {
"#version": "1",
"#timestamp": "2019-05-09T09:09:32.167Z",
"path": "/home/secunix/logs/TestLogPourMapping_09_05.json",
"message": "{\"#timestamp\":\"2019-05-07T09:56:33.996Z\",\"ip\":\"181.144.250.19\",\"extension\":\"jpg\",\"response\":\"200\",\"geo\":{\"coordinates\":{\"lat\":44.12326,\"lon\":-123.2186856},\"src\":\"IN\",\"dest\":\"CN\",\"srcdest\":\"IN:CN\"},\"#tags\":[\"success\",\"info\"],\"utc_time\":\"2019-05-07T09:56:33.996Z\",\"referer\":\"http://www.slate.com/success/thomas-marshburn\",\"agent\":\"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24\",\"clientip\":\"181.144.250.19\",\"bytes\":2553,\"host\":\"media-for-the-masses.theacademyofperformingartsandscience.org\",\"request\":\"/uploads/fyodor-yurchikhin.jpg\",\"url\":\"https://media-for-the-masses.theacademyofperformingartsandscience.org/uploads/fyodor-yurchikhin.jpg\",\"#message\":\"181.144.250.19 - - [2019-05-07T09:56:33.996Z] \\\"GET /uploads/fyodor-yurchikhin.jpg HTTP/1.1\\\" 200 2553 \\\"-\\\" \\\"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24\\\"\",\"spaces\":\"this is a thing with lots of spaces wwwwoooooo\",\"xss\":\"<script>console.log(\\\"xss\\\")</script>\",\"headings\":[\"<h3>ulrich-walter</h5>\",\"http://www.slate.com/success/susan-still-kilrain\"],\"links\":[\"viktor-m-afanasyev#twitter.com\",\"http://twitter.com/security/stephen-oswald\",\"www.twitter.com\"],\"relatedContent\":[],\"machine\":{\"os\":\"win xp\",\"ram\":6442450944},\"#version\":\"1\"}\r",
"host": "qvisbcld0051"
},
"fields": {
"#timestamp": [
"2019-05-09T09:09:32.167Z"
]
},
"sort": [
1557392972167
]
}
And that's what I have when I check my mapping:
{
"mapping": {
"doc": {
"dynamic_templates": [
{
"message_field": {
"path_match": "message",
"match_mapping_type": "string",
"mapping": {
"norms": false,
"type": "text"
}
}
},
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"norms": false,
"type": "text"
}
}
}
],
"properties": {
"#message": {
"type": "text"
},
"#timestamp": {
"type": "date"
},
"#version": {
"type": "integer"
},
"agent": {
"type": "text"
},
"bytes": {
"type": "integer"
},
"clientip": {
"type": "ip"
},
"extension": {
"type": "text"
},
"geo": {
"properties": {
"coordinates": {
"type": "geo_point"
},
"dest": {
"type": "text"
},
"src": {
"type": "text"
},
"srcdest": {
"type": "text"
}
}
},
"geoip": {
"dynamic": "true",
"properties": {
"ip": {
"type": "ip"
},
"latitude": {
"type": "half_float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "half_float"
}
}
},
"host": {
"type": "text"
},
"ip": {
"type": "ip"
},
"links": {
"type": "text"
},
"machine": {
"properties": {
"os": {
"type": "text"
},
"ram": {
"type": "integer"
}
}
},
"message": {
"type": "text",
"norms": false
},
"path": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"referer": {
"type": "text"
},
"relatedContent": {
"properties": {
"article:modified_time": {
"type": "date"
},
"article:published_time": {
"type": "date"
},
"article:section": {
"type": "keyword"
},
"article:tag": {
"type": "text"
},
"og:description": {
"type": "text"
},
"og:image": {
"type": "text"
},
"og:image:height": {
"type": "integer"
},
"og:image:width": {
"type": "integer"
},
"og:site_name": {
"type": "text"
},
"og:title": {
"type": "text"
},
"og:type": {
"type": "text"
},
"og:url": {
"type": "text"
},
"twitter:card": {
"type": "keyword"
},
"twitter:description": {
"type": "text"
},
"twitter:image": {
"type": "text"
},
"twitter:site": {
"type": "keyword"
},
"twitter:title": {
"type": "text"
},
"url": {
"type": "text"
}
}
},
"request": {
"type": "text"
},
"response": {
"type": "text"
},
"spaces": {
"type": "text"
},
"tags": {
"type": "text"
},
"url": {
"type": "text"
},
"utc_time": {
"type": "date"
},
"xss": {
"type": "text"
}
}
},
"_default_": {
"dynamic_templates": [
{
"message_field": {
"path_match": "message",
"match_mapping_type": "string",
"mapping": {
"norms": false,
"type": "text"
}
}
},
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"norms": false,
"type": "text"
}
}
}
],
"properties": {
"#timestamp": {
"type": "date"
},
"#version": {
"type": "keyword"
},
"geoip": {
"dynamic": "true",
"properties": {
"ip": {
"type": "ip"
},
"latitude": {
"type": "half_float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "half_float"
}
}
}
}
}
}
}
I send my data thanks to logstash, so here's the conf of the input:
input {
beats {
port => 5044
tags => "fromBeats"
}
file {
path => [
"/home/secunix/logs/*",
"/tech/*"
]
start_position => "beginning"
sincedb_path => "/dev/null"
}
tcp {
port => 5514
type => "syslog"
tags => "from Syslog-ng"
}
}
filter {
if [type] == "syslog"{
grok {
match => ["message", "<(?<sys_priority>\d+?)>(?<syslog_timestamp>%{CISCOTIMESTAMP})\s(?<logsource>%{URIHOST})(\s(?:(?<application>.*?)(%(?<project>.*?))?))?:(?:\s)?(?<logmessage>.*$)"]
}
if [logmessage] {
mutate {
replace => [ "message", "%{logmessage}" ]
remove_field => [ "logmessage" ]
}
}
if [project] {
mutate {
replace => [ "type", "%{project}" ]
remove_field => [ "project" ]
}
}else if [application] {
mutate {
lowercase => [ "application" ]
}
mutate {
gsub => [ "application", " ", "_" ]
}
mutate {
replace => [ "type", "%{application}" ]
}
}else {
mutate {
replace => [ "type", "uknapp" ]
add_field => { "application" => "uknapp" }
}
}
}
}
and the output:
output {
elasticsearch {
hosts => ["localhost:9200"]
index => "logstash-%{+YYYY.MM.dd}"
#+++ sa Added by scr-sop-af-config-elksandbox. Do not remove this line.
user => "logstash"
#--- sa Added by scr-sop-af-config-elksandbox. Do not remove this line.
#+++ sa Added by scr-sop-af-config-elksandbox. Do not remove this line.
password => "logstash"
#--- sa Added by scr-sop-af-config-elksandbox. Do not remove this line.
}
}
Can someone tell me how I can fix this please ?

Filter elasticsearch by range of date on a string property

I need to query elasticsearch & filter the result to be in a range of dates.
the thing is the date property is mapped as a string.
is it possible to do so ?
this is the search query i'm using:
{
"size": 1,
"from": 0,
"query": {
"bool": {
"must": [
{ "match": { "status": "active" }},
{ "match": { "last_action_state": "accepted" }}
],
"filter": [
{"missing" : { "field" : "store_id" }},
{ "range": { "list_time": { "gte": "2017/01/01 00:00:00", "lte": "2017/03/01 23:59:59", "format": "yyyy/MM/dd HH:mm:ss"}}}
]
}
}
}
the thing is i have no control over the mapping since it's created automatically by another program which index the documents, and i can't change the mapping once it's created.
ps: elasticsearch version: 2.3
UPDATE:
index info:
{
"avindex_v3": {
"aliases": {
"avindex": {}
},
"mappings": {
"ads": {
"properties": {
"account_id": {
"type": "long"
},
"ad_id": {
"type": "long"
},
"ad_params": {
"type": "string"
},
"body": {
"type": "string"
},
"category": {
"type": "long"
},
"city": {
"type": "long"
},
"company_ad": {
"type": "boolean"
},
"email": {
"type": "string"
},
"images": {
"type": "string"
},
"lang": {
"type": "string"
},
"last_action_state": {
"type": "string"
},
"list_date": {
"type": "long"
},
"list_id": {
"type": "long"
},
"list_time": {
"type": "string"
},
"modified_at": {
"type": "string"
},
"modified_ts": {
"type": "double"
},
"name": {
"type": "string"
},
"orig_date": {
"type": "long"
},
"orig_list_time": {
"type": "string"
},
"phone": {
"type": "string"
},
"phone_hidden": {
"type": "boolean"
},
"price": {
"type": "long"
},
"region": {
"type": "long"
},
"status": {
"type": "string"
},
"store_id": {
"type": "long"
},
"subject": {
"type": "string"
},
"type": {
"type": "string"
},
"user_id": {
"type": "long"
}
}
}
},
"settings": {
"index": {
"creation_date": "1493216710928",
"number_of_shards": "5",
"number_of_replicas": "1",
"uuid": "WEHGLF8iRyGk3Xgbmo7H8Q",
"version": {
"created": "2040499"
}
}
},
"warmers": {}
}
}

You can try to give it as a keyword like this :
{
"range": {
"list_time.keyword": {
"gte": "2020-08-12 22:24:55.56",
"lte": "2020-08-12 22:24:56.56"
}
}
}

Resources