ElasticSearch - doc_value in murmur3 hash - elasticsearch

Continuing a previous question.
What am I trying to do?
Thanks to #AndreiStefan I'm trying to put a murmur3 hash off the heap, using doc_values:
"dynamic_templates": [
{
"murmur3_hashed": {
"mapping": {
"index": "not_analyzed",
"norms": {
"enabled": false
},
"fielddata": {
"format": "doc_values"
},
"doc_values": true,
"type": "string",
"fields": {
"hash": {
"index": "no",
"doc_values": true,
"type": "murmur3"
}
}
},
"match_mapping_type": "string",
"match": "my_prop"
}
}
]
I used stream2es for reindexing.
What is the result?
After a reindexing, the result property is:
"my_prop": {
"index": "not_analyzed",
"fielddata": {
"format": "doc_values"
},
"doc_values": true,
"type": "string",
"fields": {
"hash": {
"null_value": -1,
"precision_step": 2147483647,
"type": "murmur3"
}
}
},
What is the problem?
Why is the "index": "no", "doc_values": true missing in the result property?

This is the list of commands that I tested in ES 1.6.0:
PUT /test
{
"mappings": {
"_default_": {
"dynamic_templates": [
{
"murmur3_hashed": {
"mapping": {
"index": "not_analyzed",
"norms": {
"enabled": false
},
"fielddata": {
"format": "doc_values"
},
"doc_values": true,
"type": "string",
"fields": {
"hash": {
"index": "no",
"doc_values": true,
"type": "murmur3"
}
}
},
"match_mapping_type": "string",
"match": "my_prop"
}
}
]
}
}
}
POST /test/test_type/1
{
"my_prop": "xxx"
}
GET /test/test_type/_mapping
And I got this as output:
{
"test": {
"mappings": {
"test_type": {
"dynamic_templates": [
{
"murmur3_hashed": {
"mapping": {
"fielddata": {
"format": "doc_values"
},
"norms": {
"enabled": false
},
"index": "not_analyzed",
"type": "string",
"fields": {
"hash": {
"index": "no",
"type": "murmur3",
"doc_values": true
}
},
"doc_values": true
},
"match": "my_prop",
"match_mapping_type": "string"
}
}
],
"properties": {
"my_prop": {
"type": "string",
"index": "not_analyzed",
"doc_values": true,
"fielddata": {
"format": "doc_values"
},
"fields": {
"hash": {
"type": "murmur3",
"index": "no",
"doc_values": true,
"precision_step": 2147483647,
"null_value": -1
}
}
}
}
}
}
}
}

Related

How to migrate data from ES2.1 to ES7 by logstash

[ERROR][logstash.outputs.elasticsearch][main] Failed to install
template {:message=>"Got response code '400' contacting Elasticsearch
at URL 'http://127.0.0.1:9200/_template/ecs-logstash'",
:exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError
original ES version: 2.1.2 new ES version: 7.13 logstash version: 8.1.1
I have below index on my ES2
"designs_v1": {
"mappings": {
"design": {
"dynamic": "false",
"_all": {
"enabled": false
},
"_id": {
"store": true,
"index": "not_analyzed"
},
"_timestamp": {
"enabled": true,
"store": true
},
"properties": {
"createDate": {
"type": "date",
"fielddata": {
"__comment": "Used for sorting",
"loading": "eager"
},
"format": "epoch_millis||date_time"
},
"designId": {
"type": "long",
"fielddata": {
"__comment": "Used for sorting to break ties and accessed by our custom scorer",
"loading": "eager"
}
},
"editorialTags": {
"type": "string",
"norms": {
"loading": "eager"
},
"analyzer": "standard_with_stopwords",
"fields": {
"shingles": {
"type": "string",
"norms": {
"loading": "eager"
},
"analyzer": "shingle"
},
"stemmed": {
"type": "string",
"norms": {
"loading": "eager"
},
"analyzer": "kstem"
}
}
},
"isPersonalizable": {
"type": "boolean"
},
"legalBlockTags": {
"type": "string",
"norms": {
"enabled": false
},
"analyzer": "standard_with_stopwords"
},
"memberId": {
"type": "long"
},
"pixel_height": {
"type": "integer"
},
"pixel_width": {
"type": "integer"
},
"products": {
"type": "nested",
"properties": {
"caption": {
"type": "string",
"norms": {
"enabled": false
},
"analyzer": "standard_with_stopwords"
},
"createDate": {
"type": "date",
"format": "epoch_millis||date_time"
},
"defaultThumbnail": {
"type": "integer"
},
"description": {
"type": "string",
"norms": {
"enabled": false
},
"analyzer": "standard_with_stopwords"
},
"hasPersonalizableSvg": {
"type": "boolean"
},
"imageOneId": {
"type": "long"
},
"imageTwoId": {
"type": "long"
},
"locations": {
"type": "string",
"norms": {
"enabled": false
},
"analyzer": "keyword"
},
"productId": {
"type": "long"
},
"productTypeId": {
"type": "integer",
"fielddata": {
"__comment": "Used during aggregations",
"loading": "eager"
}
},
"showColorId": {
"type": "integer"
},
"storeId": {
"type": "long"
}
}
},
"sellerTags": {
"type": "string",
"norms": {
"loading": "eager"
},
"analyzer": "standard_with_stopwords",
"fields": {
"shingles": {
"type": "string",
"norms": {
"loading": "eager"
},
"analyzer": "shingle"
},
"stemmed": {
"type": "string",
"norms": {
"loading": "eager"
},
"analyzer": "kstem"
}
}
}
}
}
}
}
I created new index in ES7:
{
"mappings": {
// "_id": {
// "store": true,
// "index": "not_analyzed"
// },
// "_timestamp": {
// "enabled": true,
// "store": true
// },
"properties": {
"createDate": {
"type": "date",
"format": "epoch_millis||date_time"
},
"designId": {
"type": "long"
},
"editorialTags": {
"type": "text",
"norms": true,
"analyzer": "standard_with_stopwords",
"fields": {
"shingles": {
"type": "text",
"norms": true,
"analyzer": "shingle"
},
"stemmed": {
"type": "text",
"norms": true,
"analyzer": "kstem"
}
}
},
"isPersonalizable": {
"type": "boolean"
},
"legalBlockTags": {
"type": "text",
"norms": false,
"analyzer": "standard_with_stopwords"
},
"memberId": {
"type": "long"
},
"pixel_height": {
"type": "integer"
},
"pixel_width": {
"type": "integer"
},
"products": {
"type": "nested",
"properties": {
"caption": {
"type": "text",
"norms": false,
"analyzer": "standard_with_stopwords"
},
"createDate": {
"type": "date",
"format": "epoch_millis||date_time"
},
"defaultThumbnail": {
"type": "integer"
},
"description": {
"type": "text",
"norms": false,
"analyzer": "standard_with_stopwords"
},
"hasPersonalizableSvg": {
"type": "boolean"
},
"imageOneId": {
"type": "long"
},
"imageTwoId": {
"type": "long"
},
"locations": {
"type": "text",
"norms": false,
"analyzer": "keyword"
},
"productId": {
"type": "long"
},
"productTypeId": {
"type": "integer"
},
"showColorId": {
"type": "integer"
},
"storeId": {
"type": "long"
}
}
},
"sellerTags": {
"type": "text",
"norms": true,
"analyzer": "standard_with_stopwords",
"fields": {
"shingles": {
"type": "text",
"norms": true,
"analyzer": "shingle"
},
"stemmed": {
"type": "text",
"norms": true,
"analyzer": "kstem"
}
}
}
}
}
}
I want to migrate data to ES7 by logstash, below is my logstash conf file:
input {
elasticsearch {
hosts => ["http://xxx:9200"]
index => "designs_v1"
type => "design"
size => 10
scroll => "1m"
}
}
filter {
}
output {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "designs_v1"
#document_type => "%{[#metadata][_type]}"
document_id => "%{[#metadata][_id]}"
}
}
But I can't fix issue:
[ERROR][logstash.outputs.elasticsearch][main] Failed to install
template {:message=>"Got response code '400' contacting Elasticsearch
at URL 'http://127.0.0.1:9200/_template/ecs-logstash'",
:exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError
What can I do now?

Elasticsearch match or term always failing

I have a problem with a query that return no result. When I execute the following query either with match or term :
{
"size": 1,
"query": {
"bool": {
"must": [
{ "term": { "ALERT_TYPE.raw": "ERROR" }}
],
"filter": [
{ "range": {
"#timestamp": {
"gte": "2018-02-01T00:00:01.000Z",
"lte": "2018-02-28T23:55:55.000Z"
}
}}
]
}
}
}
I always got the following response, :
{
"took": 92,
"timed_out": false,
"_shards": {
"total": 215,
"successful": 215,
"failed": 0
},
"hits": {
"total": 0,
"max_score": null,
"hits": []
}
}
But i'm sure the element is present because when i do a match_all query, the first hit is the following :
{
"took": 269,
"timed_out": false,
"_shards": {
"total": 210,
"successful": 210,
"failed": 0
},
"hits": {
"total": 68292,
"max_score": 1,
"hits": [
{
"_index": "logstash-2018.02.22",
"_type": "alert",
"_id": "AWEdVphtJjppDZ0FiAz-",
"_score": 1,
"_source": {
"#version": "1",
"#timestamp": "2018-02-22T10:07:41.549Z",
"path": "/something",
"host": "host.host",
"type": "alert",
"SERVER_TYPE": "STANDALONE",
"LOG_FILE": "log.log",
"DATE": "2018-02-22 11:02:02,367",
"ALERT_TYPE": "ERROR",
"MESSAGE": "There is an error"
}
}
]
}
}
Here I can see the field is the value that I am expecting. And from the mapping I know the field is analyzed by the default analyser and the raw field is not analysed (Thanks to the answer of Glenn Van Schil). The mapping is generated dynamically by logstash but it looks like this for the type i'm looking into:
"alert": {
"_all": {
"enabled": true,
"omit_norms": true
},
"dynamic_templates": [
{
"message_field": {
"mapping": {
"index": "analyzed",
"omit_norms": true,
"fielddata": { "format": "disabled" },
"type": "string"
},
"match": "message",
"match_mapping_type": "string"
}
},
{
"string_fields": {
"mapping": {
"index": "analyzed",
"omit_norms": true,
"fielddata": { "format": "disabled" },
"type": "string",
"fields": {
"raw": {
"index": "not_analyzed",
"ignore_above": 256,
"type": "string"
}
}
},
"match": "*",
"match_mapping_type": "string"
}
}
],
"properties": {
"#timestamp": { "type": "date", "format": "strict_date_optional_time||epoch_millis" },
"#version": { "type": "string", "index": "not_analyzed" },
"ALERT_TYPE": {
"type": "string",
"norms": { "enabled": false },
"fielddata": { "format": "disabled" },
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"DATE": {
"type": "string",
"norms": { "enabled": false },
"fielddata": { "format": "disabled" },
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"LOG_FILE": {
"type": "string",
"norms": { "enabled": false },
"fielddata": { "format": "disabled" },
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"MESSAGE": {
"type": "string",
"norms": { "enabled": false },
"fielddata": { "format": "disabled" },
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"SERVER_TYPE": {
"type": "string",
"norms": { "enabled": false },
"fielddata": { "format": "disabled" },
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"geoip": {
"dynamic": "true",
"properties": {
"ip": { "type": "ip" },
"latitude": { "type": "float" },
"location": { "type": "geo_point" },
"longitude": { "type": "float" }
}
},
"host": {
"type": "string",
"norms": { "enabled": false },
"fielddata": { "format": "disabled" },
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"path": {
"type": "string",
"norms": { "enabled": false },
"fielddata": { "format": "disabled" },
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
},
"type": {
"type": "string",
"norms": { "enabled": false },
"fielddata": { "format": "disabled" },
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
}
Does anyone have a clue about why this query keep returning nothing ? Maybe there is something in the mapping that i am missing which explain why the match or term query keep failing ? I'm running out of idea about what is happenning and i'm quite new to elasticsearch and logstash.
Versions of tools and environment :
OS: RHEL Server 6.5 (Santiago)
Java: 1.7.0_91
Elasticsearch: 2.4.6
Lucene: 5.5.4
Logstash: 2.4.1
This is not really an answer, but it was to complicated to write this as a comment.
from the mapping i know the field is not analysed.
You are searching for ALERT_TYPE, but this one is in fact analyzed with the default analyzer since you did not specify any analyzer directly under your ALERT_TYPE's mapping.
However, your ALERT_TYPE has an internal field named raw that is not analyzed. If you want to search documents using the raw field you'll need to change the query from
"must": [
{ "term": { "ALERT_TYPE": "ERROR" }}
]
to
"must": [
{ "term": { "ALERT_TYPE.raw": "ERROR" }}
]

Error while creating an index on Elasticsearch with custom analyzer

I am trying to create an index with a custom default analyzer.
I already checked the following questions:
Analyzer not found exception while creating an index with mapping and settings
How to specify an analyzer while creating an index in ElasticSearch
mapper_parsing_exception for a custom analyzer while creating index in elasticsearch?
but they didn't solve the issue.
Here is my schema:
put /emails
{
"mappings": {
"email": {
"analyzer": "lkw",
"properties": {
"createdOn": {
"type": "date",
"store": true,
"format": "strict_date_optional_time||epoch_millis"
},
"data": {
"type": "object",
"dynamic": "true"
},
"from": {
"type": "string",
"store": true
},
"id": {
"type": "string",
"store": true
},
"sentOn": {
"type": "date",
"store": true,
"format": "strict_date_optional_time||epoch_millis"
},
"sesId": {
"type": "string",
"store": true
},
"subject": {
"type": "string",
"store": true,
"analyzer": "standard"
},
"templates": {
"properties": {
"html": {
"type": "string",
"store": true
},
"plainText": {
"type": "string",
"store": true
}
}
},
"to": {
"type": "string",
"store": true
},
"type": {
"type": "string",
"store": true
}
}
},
"event": {
"_parent": {
"type": "email"
},
"analyzer": "lkw",
"properties": {
"id": {
"type": "string",
"store": true
},
"origin": {
"type": "string",
"store": true
},
"time": {
"type": "date",
"store": true,
"format": "strict_date_optional_time||epoch_millis"
},
"type": {
"type": "string",
"store": true
},
"userAgent": {
"type": "string",
"store": true
}
}
}
},
"settings": {
"analysis": {
"analyzer": {
"lkw": {
"tokenizer": "keyword",
"filter": [
"lowercase"
],
"type": "custom"
}
}
}
}
}
When I execute the command above, I get this error:
{
"error": {
"root_cause": [
{
"type": "mapper_parsing_exception",
"reason": "Root mapping definition has unsupported parameters: [analyzer : lkw]"
}
],
"type": "mapper_parsing_exception",
"reason": "Failed to parse mapping [event]: Root mapping definition has unsupported parameters: [analyzer : lkw]",
"caused_by": {
"type": "mapper_parsing_exception",
"reason": "Root mapping definition has unsupported parameters: [analyzer : lkw]"
}
},
"status": 400
}
Since you have only a few string fields, I suggest you simply specify your lkw analyzer where you need it, just like you did for the standard one:
PUT /emails
{
"mappings": {
"email": {
"properties": {
"createdOn": {
"type": "date",
"store": true,
"format": "strict_date_optional_time||epoch_millis"
},
"data": {
"type": "object",
"dynamic": "true"
},
"from": {
"type": "string",
"store": true,
"analyzer": "lkw"
},
"id": {
"type": "string",
"store": true,
"analyzer": "lkw"
},
"sentOn": {
"type": "date",
"store": true,
"format": "strict_date_optional_time||epoch_millis"
},
"sesId": {
"type": "string",
"store": true,
"analyzer": "lkw"
},
"subject": {
"type": "string",
"store": true,
"analyzer": "standard"
},
"templates": {
"properties": {
"html": {
"type": "string",
"store": true,
"analyzer": "lkw"
},
"plainText": {
"type": "string",
"store": true,
"analyzer": "lkw"
}
}
},
"to": {
"type": "string",
"store": true,
"analyzer": "lkw"
},
"type": {
"type": "string",
"store": true,
"analyzer": "lkw"
}
}
},
"event": {
"_parent": {
"type": "email"
},
"properties": {
"id": {
"type": "string",
"store": true,
"analyzer": "lkw"
},
"origin": {
"type": "string",
"store": true,
"analyzer": "lkw"
},
"time": {
"type": "date",
"store": true,
"format": "strict_date_optional_time||epoch_millis"
},
"type": {
"type": "string",
"store": true,
"analyzer": "lkw"
},
"userAgent": {
"type": "string",
"store": true,
"analyzer": "lkw"
}
}
}
},
"settings": {
"analysis": {
"analyzer": {
"lkw": {
"tokenizer": "keyword",
"filter": [
"lowercase"
],
"type": "custom"
}
}
}
}
}

ES NGram with multi-languages

I'm trying to implement an auto-suggest control powered by an ES index. The index has multiple fields (Multi-language - Arabic and English) and I want to be able to search in all languages.
The easiest way to do that is NGram with the "_all" field, as long as some care is taken in the mapping definition. The issue we have now how to accomplish this using multi-language.
PS: We are looking to separate field for all the possible languages (Using one index).
I tried to use the nGram tokenizer and filter and it's working good for one language (English).
{
"template": "index_com",
"settings": {
"number_of_shards": 5,
"number_of_replicas": 1,
"analysis": {
"filter": {
"edgeNGram_filter": {
"type": "edgeNGram",
"min_gram": 2,
"max_gram": 20
}
},
"analyzer": {
"edgeNGram_analyzer": {
"type": "custom",
"tokenizer": "whitespace",
"filter": [
"lowercase",
"asciifolding",
"edgeNGram_filter"
]
}
}
}
},
"mappings": {
"product": {
"_all": {
"enabled": true,
"index_analyzer": "edgeNGram_analyzer",
"search_analyzer": "standard"
},
"properties": {
"id": {
"type": "string",
"index": "no",
"include_in_all": false
},
"uuid": {
"type": "string",
"index": "no",
"include_in_all": false
},
"name": {
"type": "string",
"include_in_all": true
},
"description": {
"type": "string",
"include_in_all": true
},
"brand": {
"type": "string",
"include_in_all": true
},
"made_id": {
"type": "string",
"include_in_all": true
},
"category": {
"type": "string",
"include_in_all": true
},
"category_id": {
"type": "integer",
"include_in_all": false
},
"keywords": {
"type": "string",
"include_in_all": true
},
"colors": {
"type": "string",
"index": "not_analyzed"
},
"colors_name": {
"type": "string",
"include_in_all": true
},
"quality": {
"type": "string",
"index": "not_analyzed"
},
"vendor_name": {
"type": "string",
"include_in_all": false
},
"vendor_location" : {
"type" : "geo_point",
"include_in_all": false
},
"price": {
"type": "double",
"include_in_all": false
},
"price_before_discount": {
"type": "double",
"include_in_all": false
},
"is_deal": {
"type": "integer",
"include_in_all": false
},
"is_best_seller": {
"type": "integer",
"include_in_all": false
},
"views": {
"type": "integer",
"include_in_all": false
},
"rating": {
"type": "integer",
"include_in_all": false
},
"updated_at": {
"type": "date",
"format": "dateOptionalTime"
},
"created_at": {
"type": "date",
"format": "dateOptionalTime"
},
"image_link": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
Arabic analyzer:
{
"settings": {
"analysis": {
"filter": {
"arabic_stop": {
"type": "stop",
"stopwords": "_arabic_"
},
"arabic_keywords": {
"type": "keyword_marker",
"keywords": []
},
"arabic_stemmer": {
"type": "stemmer",
"language": "arabic"
}
},
"analyzer": {
"arabic": {
"tokenizer": "standard",
"filter": [
"lowercase",
"arabic_stop",
"arabic_normalization",
"arabic_keywords",
"arabic_stemmer"
]
}
}
}
}
}
can someone suggest any solution? Thanks!
Your second snippet defines the arabic analyzer, which is already available so you shouldn't need to add it.
What you are missing is to tell elasticsearch to also use the arabic analyzer. So you want to analyze each field twice, in english and arabic. To do that, add
"fields": {
"ar": {
"type": "string",
"analyzer": "arabic"
},
"en": {
"type": "string",
"analyzer": "english"
}
}
to all your fields that have "include_in_all": true. That makes your mappings look like this:
{
"template": "index_com",
"settings": {
"number_of_shards": 5,
"number_of_replicas": 1,
"analysis": {
"filter": {
"edgeNGram_filter": {
"type": "edgeNGram",
"min_gram": 2,
"max_gram": 20
}
},
"analyzer": {
"edgeNGram_analyzer": {
"type": "custom",
"tokenizer": "whitespace",
"filter": [
"lowercase",
"asciifolding",
"edgeNGram_filter"
]
}
}
}
},
"mappings": {
"product": {
"_all": {
"enabled": true,
"index_analyzer": "edgeNGram_analyzer",
"search_analyzer": "standard"
},
"properties": {
"id": {
"type": "string",
"index": "no",
"include_in_all": false
},
"uuid": {
"type": "string",
"index": "no",
"include_in_all": false
},
"name": {
"type": "string",
"include_in_all": true,
"fields": {
"ar": {
"type": "string",
"analyzer": "arabic"
},
"en": {
"type": "string",
"analyzer": "english"
}
}
},
"description": {
"type": "string",
"include_in_all": true,
"fields": {
"ar": {
"type": "string",
"analyzer": "arabic"
},
"en": {
"type": "string",
"analyzer": "english"
}
}
},
"brand": {
"type": "string",
"include_in_all": true,
"fields": {
"ar": {
"type": "string",
"analyzer": "arabic"
},
"en": {
"type": "string",
"analyzer": "english"
}
}
},
"made_id": {
"type": "string",
"include_in_all": true,
"fields": {
"ar": {
"type": "string",
"analyzer": "arabic"
},
"en": {
"type": "string",
"analyzer": "english"
}
}
},
"category": {
"type": "string",
"include_in_all": true,
"fields": {
"ar": {
"type": "string",
"analyzer": "arabic"
},
"en": {
"type": "string",
"analyzer": "english"
}
}
},
"category_id": {
"type": "integer",
"include_in_all": false
},
"keywords": {
"type": "string",
"include_in_all": true,
"fields": {
"ar": {
"type": "string",
"analyzer": "arabic"
},
"en": {
"type": "string",
"analyzer": "english"
}
}
},
"colors": {
"type": "string",
"index": "not_analyzed"
},
"colors_name": {
"type": "string",
"include_in_all": true,
"fields": {
"ar": {
"type": "string",
"analyzer": "arabic"
},
"en": {
"type": "string",
"analyzer": "english"
}
}
},
"quality": {
"type": "string",
"index": "not_analyzed"
},
"vendor_name": {
"type": "string",
"include_in_all": false
},
"vendor_location": {
"type": "geo_point",
"include_in_all": false
},
"price": {
"type": "double",
"include_in_all": false
},
"price_before_discount": {
"type": "double",
"include_in_all": false
},
"is_deal": {
"type": "integer",
"include_in_all": false
},
"is_best_seller": {
"type": "integer",
"include_in_all": false
},
"views": {
"type": "integer",
"include_in_all": false
},
"rating": {
"type": "integer",
"include_in_all": false
},
"updated_at": {
"type": "date",
"format": "dateOptionalTime"
},
"created_at": {
"type": "date",
"format": "dateOptionalTime"
},
"image_link": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}

elastic: HOW-TO search a nested field ? [not working]

if i have a JSON document indexed into Elasticsearch, like the following:
"_source": {
"pid_no": 19321,
"aggregator_id": null,
"inet_family": "ipv4-unicast",
"origin_code": "igp",
"extended_community": null,
"atomic_aggregate": null,
"adv_type": "announce",
"local_preference": 250,
"med_metric": 0,
"time_stamp": 1447534931,
"net_mask": "23",
"prefix4_": {
"last": 222,
"first": 111
},
"counter_no": 69668,
"confederation_path": "",
"as_set": null,
and i have tried successfully to filter all of the keys of the doc,
but, except the nested ones.
the query looks like:
GET /SNIP!/SNIP!/_search?routing=SNIP!
{
"query": {
"bool": {
"must": {
"query": {
"match_all": {}
}
},
"filter": {
"bool": {
"filter": [
{
"range": {
"local_preference": {
"gt": 150,
"lte": 250
}
}
},
>>> if i remove the filter below, matches the document.
>>> when i apply the filter, i get 0 hits
{
"and": [
{
"range": {
"prefix4_.first": {
"lte": 200
}
}
},
{
"range": {
"prefix4_.last": {
"gte": 200
}
}
}
]
}
]
}
}
}
}
}
it goes without saying that the mapping is done using integers in the corresponding fields (prefix4_.first,prefix4_.last)
could you please advise on why the filtering does not work ?
EDIT: the mapping looks like this
{
"mappings": {
"_default_": {
"_all": { "enabled": False },
"dynamic": True,
"_routing": { "required": True },
"properties": {
"pid_no": { "type": "string", "index": "not_analyzed", "store": "no" },
"counter_no": { "type": "long", "store": "no" },
"time_stamp": { "type": "date", "format": "epoch_second", "store": "no" },
"host_name": { "type": "string", "index": "not_analyzed", "store": "no" },
"local_ip": { "type": "ip", "store": "no" },
"peer_ip": { "type": "ip", "store": "no" },
"local_asn": { "type": "string", "index": "not_analyzed", "store": "no" },
"peer_asn": { "type": "string", "index": "not_analyzed", "store": "no" },
"inet_family": { "type": "string", "index": "not_analyzed", "store": "no" },
"next_hop": { "type": "ip", "store": "no" },
"net_block": { "type": "string", "index": "analyzed", "store": "no" },
"as_path": { "type": "string", "index": "analyzed", "store": "no" },
"cluster_list": { "type": "string", "index": "not_analyzed", "store": "no" },
"confederation_path": { "type": "string", "index": "not_analyzed", "store": "no" },
"local_preference": { "type": "integer", "store": "no" },
"originator_ip": { "type": "ip", "store": "no" },
"origin_code": { "type": "string", "index": "not_analyzed", "store": "no" },
"community_note": { "type": "string", "index": "analyzed", "store": "no" },
"med_metric": { "type": "long", "store": "no" },
"atomic_aggregate": { "type": "boolean", "store": "no" },
"aggregator_id": { "type": "string", "index": "analyzed", "store": "no" },
"as_set": { "type": "string", "index": "analyzed", "store": "no" },
"extended_community": { "type": "string", "index": "not_analyzed", "store": "no" },
"adv_type": { "type": "string", "index": "not_analyzed", "store": "no" },
"prefix_": { "type": "string", "index": "not_analyzed", "store": "no" },
"net_mask": { "type": "integer", "store": "no" },
"prefix4_": {
"type": "nested",
"properties": {
"first": { "type": "integer", "store": "no" },
"last": { "type": "integer", "store": "no" }
}
},
"prefix6_": {
"type": "nested",
"properties": {
"lofirst": { "type": "long", "store": "no" },
"lolast": { "type": "long", "store": "no" },
"hifirst": { "type": "long", "store": "no" },
"hilast": { "type": "long", "store": "no" }
}
}
}
}
},
"settings" : {
"number_of_shards": 1,
"number_of_replicas": 0,
"index": {
"store.throttle.type": "none",
"memory.index_buffer_size": "20%",
"refresh_interval": "1m",
"merge.async": True,
"merge.scheduler.type": "concurrent",
"merge.policy.type": "log_byte_size",
"merge.policy.merge_factor": 15,
"cache.query.enable": True,
"cache.filter.type": "node",
"fielddata.cache.type": "node",
"cache.field.type": "soft"
}
}
}
Elasticsearch provides multiple ways of mapping nested documents. You are using nested which indexes nested documents as separate documents behind the scenes and as such querying them requires the use of a nested query.
The simplest way of indexing nested JSON like you've shown is using the object type mapping. This would allow you to query the field the way you were expecting, however Elasticsearch flattens the hierarchy which may not be acceptable for you.
use nested filters to filter your documents on nested fields.
https://www.elastic.co/guide/en/elasticsearch/reference/1.4/query-dsl-nested-filter.html
{
"query": {
"filtered": {
"filter": {
"bool": {
"must": [
{
"term": {
"peer_ip": "pqr",
"_cache": true
}
},
{
"nested": {
"filter": {
"bool": {
"must": [
{
"terms": {
"first": [
"xyz"
],
"_cache": true
}
}
]
}
},
"path": "prefix4_",
"inner_hits": {}
}
},
{
"terms": {
"pid_no": [
"yyu"
],
"_cache": true
}
}
]
}
}
}
}
}

Resources