Elasticsearch NEST highlighting the attachment content - elasticsearch

In Elasticsearch 5.6.3 I installed ingest-attachment plugin. I am trying to set the term vector property to WithPositionOffsets in attachment's content. Where should I set this property to see highligted result in my search?
The Document POCO is below:
public class Document
{
public int Id { get; set; }
public string Path { get; set; }
public string Content { get; set; }
public Attachment Attachment { get; set; }
}
Here is the CreateIndex function with mappings.
var indexResponse = client.CreateIndex(documentsIndex, c => c
.Settings(s => s
.Analysis(a => a
.Analyzers(ad => ad
.Custom("windows_path_hierarchy_analyzer", ca => ca
.Tokenizer("windows_path_hierarchy_tokenizer")
)
)
.Tokenizers(t => t
.PathHierarchy("windows_path_hierarchy_tokenizer", ph => ph
.Delimiter('\\')
)
)
)
)
.Mappings(m => m
.Map<Document>(mp => mp
.AutoMap()
.AllField(all => all
.Enabled(false)
)
.Properties(ps => ps
.Text(s => s
.Name(n => n.Path)
.Analyzer("windows_path_hierarchy_analyzer")
)
.Attachment(at => at.
Name(n => n.Attachment.Content)
.FileField(ff => ff
.Name("Content")
.TermVector(TermVectorOption.WithPositionsOffsets)
.Store()))
//.Text(s => s
// .Name(n => n.Attachment.Content)
// .TermVector(TermVectorOption.WithPositionsOffsets)
// .Store(true)
//)
.Object<Attachment>(a => a
.Name(n => n.Attachment)
.AutoMap()
)
)
)
)
);
into the .Mappings part I used a FileField to set termvector and store property. But the result is below:
{
"documents": {
"mappings": {
"document": {
"_all": {
"enabled": false
},
"properties": {
"attachment": {
"properties": {
"author": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"content": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"content_length": {
"type": "long"
},
"content_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"date": {
"type": "date"
},
"detect_language": {
"type": "boolean"
},
"indexed_chars": {
"type": "long"
},
"keywords": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"language": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"content": {
"type": "attachment",
"fields": {
"content": {
"type": "text",
"store": true,
"term_vector": "with_positions_offsets"
},
"author": {
"type": "text"
},
"title": {
"type": "text"
},
"name": {
"type": "text"
},
"date": {
"type": "date"
},
"keywords": {
"type": "text"
},
"content_type": {
"type": "text"
},
"content_length": {
"type": "integer"
},
"language": {
"type": "text"
}
}
},
"id": {
"type": "integer"
},
"path": {
"type": "text",
"analyzer": "windows_path_hierarchy_analyzer"
}
}
}
}
}
}
So I can't see the highlight property in my query result. How should I do this?

To work with the ingest-attachment plugin, don't map the Attachment property as an attachment data type; this mapping is for the mapper-attachment plugin which is deprecated in 5.x and removed in 6.x.
I wrote a blog post about using ingest-attachment with .NET which includes an example mapping for Attachment; essentially, map it as an object data type and map the Content property on it as a text data type with .TermVector(TermVectorOption.WithPositionsOffsets) applied. You'll need to create this mapping in an index before indexing any documents with attachments into it.

Related

Elasticsearch using special characters

I'm indexing pdf's into Elasticsearch.
When I search for content like "§ 123", than the "§" is ignored.
What do I have to do so that "§" is also included in the search?
Here the Indices.Create:
CreateIndexResponse createIndexResponse = elasticClient.Indices.Create(sep.DefaultIndex, c => c
.Settings(s => s
.Analysis(a => a
.Analyzers(ad => ad
.Custom("windows_path_hierarchy_analyzer", ca => ca
.Tokenizer("windows_path_hierarchy_tokenizer")
)
)
.Tokenizers(t => t
.PathHierarchy("windows_path_hierarchy_tokenizer", ph => ph
.Delimiter('\\')
)
.NGram("ngram_tokenizer", td => td
.MinGram(2)
.MaxGram(20)
.TokenChars(
TokenChar.Letter,
TokenChar.Digit,
TokenChar.Symbol)
)
)
)
)
.Map<ElasticDocument>(mp => mp
.AutoMap()
.Properties(ps => ps
.Text(s => s
.Name(n => n.Path)
.Analyzer("windows_path_hierarchy_analyzer")
)
.Object<Attachment>(a => a
.Name(n => n.Attachment)
.AutoMap()
)
)
)
);
Here the mappings:
{
"attachments": {
"mappings": {
"properties": {
"attachment": {
"properties": {
"author": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"content": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"content_length": {
"type": "long"
},
"content_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"creator_tool": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"date": {
"type": "date"
},
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"format": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"keywords": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"language": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"metadata_date": {
"type": "date"
},
"modified": {
"type": "date"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"id": {
"type": "long"
},
"instance": {
"type": "long"
},
"path": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
Tldr;
As pointed out by #llermaly in the mapping you do not seems to be using the analyser you are creating with your code.
Be default text fields are going to be analysed with the standard analyser.
Solution
You will need to specify the analyser you want to be used on text fields
{
"mappings": {
"properties": {
"title": {
"type": "text",
"analyzer": "windows_path_hierarchy_analyzer"
}
}
}
}
else the standard analyser is used.
Which is going to delete special char.
POST _analyze
{
"analyzer": "standard",
"text": "§ 123"
}
gives you:
{
"tokens": [
{
"token": "123",
"start_offset": 2,
"end_offset": 5,
"type": "<NUM>",
"position": 0
}
]
}
The § has been removed.

Elasticsearch query for multiple terms

I am trying to create a search query that allows to search by name and type.
I have indexed the values, and my record in Elasticsearch look like this:
{
_index: "assets",
_type: "asset",
_id: "eAOEN28BcFmQazI-nngR",
_score: 1,
_source: {
name: "test.png",
mediaType: "IMAGE",
meta: {
content-type: "image/png",
width: 3348,
height: 1890,
},
createdAt: "2019-12-24T10:47:15.727Z",
updatedAt: "2019-12-24T10:47:15.727Z",
}
}
so how would I create for example, a query that finds all assets that have the name "test' and are images?
I tried multi_mach query but that did not return the correct results:
{
"query": {
"multi_match" : {
"query": "*test* IMAGE",
"type": "cross_fields",
"fields": [ "name", "mediaType" ],
"operator": "and"
}
}
}
The query above returns 0 results, and if I change the operator to "or" it returns all this assets of type IMAGE.
Any suggestions would be greatly appreciated. TIA!
EDIT: Added Mapping
Below is the mapping:
{
"assets": {
"aliases": {},
"mappings": {
"properties": {
"__v": {
"type": "long"
},
"createdAt": {
"type": "date"
},
"deleted": {
"type": "date"
},
"mediaType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"meta": {
"properties": {
"content-type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"width": {
"type": "long"
},
"height": {
"type": "long"
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"originalName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"updatedAt": {
"type": "date"
}
}
},
"settings": {
"index": {
"creation_date": "1575884312237",
"number_of_shards": "1",
"number_of_replicas": "1",
"uuid": "nSiAoIIwQJqXQRTyqw9CSA",
"version": {
"created": "7030099"
},
"provided_name": "assets"
}
}
}
}
You are unnecessary using the wildcard expression for this simple query.
First, change your analyzer on name field.
You need to create a custom analyzer which replaces . with space as default standard analyzer doesn't do that, so that you when searching for test you get test.png as there will be both test and png in the inverted index. The main benefit of doing this is to avoid the regex queries which are very costly.
Updated mapping with custom analyzer which would do the work for you. Just update your mapping and re-index again all the doc.
{
"aliases": {},
"mappings": {
"properties": {
"__v": {
"type": "long"
},
"createdAt": {
"type": "date"
},
"deleted": {
"type": "date"
},
"mediaType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"meta": {
"properties": {
"content-type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"width": {
"type": "long"
},
"height": {
"type": "long"
}
}
},
"name": {
"type": "text",
"analyzer" : "my_analyzer"
},
"originalName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"updatedAt": {
"type": "date"
}
}
},
"settings": {
"analysis": {
"analyzer": {
"my_analyzer": {
"tokenizer": "standard",
"char_filter": [
"replace_dots"
]
}
},
"char_filter": {
"replace_dots": {
"type": "mapping",
"mappings": [
". => \\u0020"
]
}
}
},
"index": {
"number_of_shards": "1",
"number_of_replicas": "1"
}
}
}
Second, you should change your query to bool query as below:
{
"query": {
"bool": {
"must": [
{
"match": {
"name": "test"
}
},
{
"match": {
"mediaType.keyword": "IMAGE"
}
}
]
}
}
}
Which is using must with 2 match queries means, that it would return docs only when there is a match in all the clauses of must query.
I already tested my solution by creating the index, inserting a few sample docs and query them, let me know if you need any help.
Did you tried with best_fields ?
{
"query": {
"multi_match" : {
"query": "Will Smith",
"type": "best_fields",
"fields": [ "name", "mediaType" ],
"operator": "and"
}
}
}

Range filter not working for "gt" operator but working for "lt"

I am using elastic search to index my documents and wanted to filter documents based on a specific property. Here is my code:
filter.push({
range: {"audience.ethnicity.asian" : { gt: 50 } }
})
It doesn't work with gt operator (sends back results that are not consistent) but works with lt operator
filter.push({
range: {"audience.ethnicity.asian" : { lt: 50 } }
})
This works surprisingly. I have double checked that the property value IS an integer. I have tried many different ways but can't seem to figure out what am I missing.
Mapping:
"audience": {
"properties": {
"age": {
"properties": {
"13-17": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"18-24": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"25-34": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"35-44": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"45-64": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"eighteen": {
"type": "long"
},
"fortyfive": {
"type": "long"
},
"thirteen": {
"type": "long"
},
"thirtyfive": {
"type": "long"
},
"twentyfive": {
"type": "long"
}
}
},
"ages": {
"properties": {
"13-17": {
"type": "float"
},
"18-24": {
"type": "float"
},
"25-34": {
"type": "float"
},
"35-44": {
"type": "float"
},
"45-64": {
"type": "float"
}
}
},
"ethnicity": {
"properties": {
"African American": {
"type": "float"
},
"Asian": {
"type": "float"
},
"Hispanic": {
"type": "float"
},
"White / Caucasian": {
"type": "float"
},
"african": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"asian": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hispanic": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"white": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"gender": {
"properties": {
"female": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"male": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"genders": {
"properties": {
"FEMALE": {
"type": "float"
},
"MALE": {
"type": "float"
}
}
},
"gendersPerAge": {
"properties": {
"13-17": {
"properties": {
"FEMALE": {
"type": "float"
},
"MALE": {
"type": "float"
}
}
},
"18-24": {
"properties": {
"FEMALE": {
"type": "float"
},
"MALE": {
"type": "float"
}
}
},
"25-34": {
"properties": {
"FEMALE": {
"type": "float"
},
"MALE": {
"type": "float"
}
}
},
"35-44": {
"properties": {
"FEMALE": {
"type": "float"
},
"MALE": {
"type": "float"
}
}
},
"45-64": {
"properties": {
"FEMALE": {
"type": "float"
},
"MALE": {
"type": "float"
}
}
}
}
}
}
} "audience": {
"properties": {
"age": {
"properties": {
"13-17": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"18-24": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"25-34": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"35-44": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"45-64": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"eighteen": {
"type": "long"
},
"fortyfive": {
"type": "long"
},
"thirteen": {
"type": "long"
},
"thirtyfive": {
"type": "long"
},
"twentyfive": {
"type": "long"
}
}
},
"ages": {
"properties": {
"13-17": {
"type": "float"
},
"18-24": {
"type": "float"
},
"25-34": {
"type": "float"
},
"35-44": {
"type": "float"
},
"45-64": {
"type": "float"
}
}
},
"ethnicity": {
"properties": {
"African American": {
"type": "float"
},
"Asian": {
"type": "float"
},
"Hispanic": {
"type": "float"
},
"White / Caucasian": {
"type": "float"
},
"african": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"asian": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hispanic": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"white": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"gender": {
"properties": {
"female": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"male": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"genders": {
"properties": {
"FEMALE": {
"type": "float"
},
"MALE": {
"type": "float"
}
}
},
"gendersPerAge": {
"properties": {
"13-17": {
"properties": {
"FEMALE": {
"type": "float"
},
"MALE": {
"type": "float"
}
}
},
"18-24": {
"properties": {
"FEMALE": {
"type": "float"
},
"MALE": {
"type": "float"
}
}
},
"25-34": {
"properties": {
"FEMALE": {
"type": "float"
},
"MALE": {
"type": "float"
}
}
},
"35-44": {
"properties": {
"FEMALE": {
"type": "float"
},
"MALE": {
"type": "float"
}
}
},
"45-64": {
"properties": {
"FEMALE": {
"type": "float"
},
"MALE": {
"type": "float"
}
}
}
}
}
}
}
In your mapping, the audience.ethnicity.asian field is of type text, hence the range query is doing a lexical range comparison instead of a numerical range comparison.
Numerically, 123 > 50, but lexically 123 < 50 because ASCII character 1 comes before 5, that's exactly the issue you're suffering from here.
I'm not sure if you can use the audience.ethnicity.Asian field, which is a float. If not, you'll need to change the mapping of audience.ethnicity.asian to a numeric type (integer, float, etc) and reindex your data.

elasticsearch dynamic field nested detection

Hi im trying to create an index in my elastic search without defining the mapping so what i did was this.
PUT my_index1/my_type/1
{
"group" : "fans",
"user" : [
{
"first" : "John",
"last" : "Smith",
"age" : "1",
"enabled": false
},
{
"first" : "Alice",
"last" : "White",
"age" : "10",
"enabled": true
}
]
}
if did this elastic search will create a mapping for this index which is the result is
{
"my_index1": {
"mappings": {
"my_type": {
"properties": {
"group": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user": {
"properties": {
"age": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"enabled": {
"type": "boolean"
},
"first": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"last": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
}
}
if you would notice the property user didn't have a type of nested other properties has their own type defined by elastic search is there a way to it automatically the mapping should be look like this for the user property
"user": {
type:"nested"
"properties": {
"age": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"enabled": {
"type": "boolean"
},
"first": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"last": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
which is missing. im currently using nest
is there a way to define a dynamic mapping to detect if the newly added data on index is nested?
By default, Elasticsearch/Lucene has no concept of inner objects. Therefore, it flattens object hierarchies into a simple list of field names and values.
The above document would be converted internally into a document that looks more like this: (See Nested field type for more details)
{
"group" : "fans",
"user.first" : [ "alice", "john" ],
"user.last" : [ "smith", "white" ]
}
There is no beautiful answer here. A common approach might be using dynamic template to convert object to nested (however, a side effect is that all fields of object type would be changed to nested type),
{
"mappings": {
"dynamic_templates": [
{
"objects": {
"match": "*",
"match_mapping_type": "object",
"mapping": {
"type": "nested"
}
}
}
]
}
}
Another approach is specify mapping for the field before inserting data.
PUT <your index>
{
"mappings": {
"properties": {
"user": {
"type": "nested"
}
}
}
}
You can define a dynamic template where you can define your own custom mapping which can be used later when you index documents in the index.
Adding a step by step procedure, with the help of which automatically the mapping of the user field would be mapped to that of nested type
First, you need to define a dynamic template for the index as shown below, which have a match parameter which will match the field name having pattern similar to user* and map it to nested type
PUT /<index-name>
{
"mappings": {
"dynamic_templates": [
{
"nested_users": {
"match": "user*",
"mapping": {
"type": "nested"
}
}
}
]
}
}
After creating this template, you need to index the documents into it
POST /<index-name>/_doc/1
{
"group": "fans",
"user": [
{
"first": "John",
"last": "Smith",
"age": "1",
"enabled": false
},
{
"first": "Alice",
"last": "White",
"age": "10",
"enabled": true
}
]
}
Now when you see the mapping of the index documents, using the Get Mapping API, the mapping would be similar to what you expect to see
GET /<index-name>/_mapping?pretty
{
"index-name": {
"mappings": {
"dynamic_templates": [
{
"nested_users": {
"match": "user*",
"mapping": {
"type": "nested"
}
}
}
],
"properties": {
"group": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user": {
"type": "nested", // note this
"properties": {
"age": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"enabled": {
"type": "boolean"
},
"first": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"last": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
}
Or as #Jacky1205 mentioned, if it is not field-specific then you can use the below template that will match all object type fields to be of nested type
{
"mappings": {
"dynamic_templates": [
{
"nested_users": {
"match": "*",
"match_mapping_type": "object",
"mapping": {
"type": "nested"
}
}
}
]
}
}

query on a date range in elasticsearch

I want to get documents from last 30 days in elastic search but it returns empty.
it is my mapping:
PUT /books
{
"mappings": {
"impressions": {
"properties": {
"booksCreated" : {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis",
"index": true
}
}
}
}
}
and it is my query:
POST /books/_search?size=0
{
"aggs": {
"range": {
"date_range": {
"field": "booksCreated",
"format": "yyyy-MM-dd",
"ranges": [
{ "to": "now" },
{ "from": "now-1M/M" }
]
}
}
}
}
I've tried all possible ways but it returns empty.
but i can query on #timestamp field
the problem is that logstash changes the field type from date to string. my json is :
{
"index":"books",
"type":"book",
"body":{
"impressions":{
"_source":{
"enabled":true
},
"properties":{
"BookCreated":"2017-09-18 12:18:39"
}
}
}
}
and my logstash config:
input {
file {
path => "E:\data2\log\logstash.log"
start_position => "beginning"
sincedb_path => "/dev/null"
codec => json
}
}
filter {
mutate {
strip => ["message"]
}
}
output {
elasticsearch {
hosts => "localhost"
index => "books"
document_type => "book"
}
}
i will log the json to a log file and logstash send them to elasticsearch
after adding json the mapping chasnges to this:
{
"Books": {
"mappings": {
"Books": {
"properties": {
"#timestamp": {
"type": "date"
},
"#version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"BookCreated": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"body": {
"properties": {
"Books": {
"properties": {
"_source": {
"properties": {
"enabled": {
"type": "boolean"
}
}
},
"properties": {
"properties": {
"BookCreated": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
},
"host": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"index": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"path": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
it has two BookCreated one isdate and the other is text
You need to put from and to in the same range, like this:
POST /books/_search?size=0
{
"aggs": {
"range": {
"date_range": {
"field": "BookCreated",
"format": "yyyy-MM-dd",
"ranges": [
{
"from": "now-1M/M",
"to": "now"
}
]
}
}
}
}
I'm pretty sure there is an issue with your mapping. First of all, make sure the bookCreated field is named consistently, both in regards to naming as well as capitalization!
Secondly, I believe the reason you have two bookCreated is because your mapping contains a bookCreated property. Your JSON however contains a nested structure: body => properties => bookCreated. Either flatten/transform the book in logstash to the required index structure, or model your index according to your json, which could look something like this?
"mappings": {
"properties": {
"body": {
"type": "object",
"properties": {
"properties": {
"type": "object",
"properties": {
"bookCreated": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis",
"index": true
}
}
}
}
}
}
}
Either way, I recommend you to set "dynamic": "strict" so you will actually see when you make a mistake in the mapping rather than just new fields being created

Resources