I'm trying to setup a mapping for an object that looks like this:
class TestObject
{
public long TestID { get; set; }
[ElasticProperty(Type = FieldType.Object)]
public Dictionary<long, List<DateTime>> Items { get; set; }
}
I use the following mapping code (where Client is IElasticClient):
this.Client.Map<TestObject>(m => m.MapFromAttributes());
I get the following mapping result:
{
"mappings": {
"testobject": {
"properties": {
"items": {
"properties": {
"comparer": {
"type": "object"
},
"count": {
"type": "integer"
},
"item": {
"type": "date",
"format": "dateOptionalTime"
},
"keys": {
"properties": {
"count": {
"type": "integer"
}
}
},
"values": {
"properties": {
"count": {
"type": "integer"
}
}
}
}
},
"testID": {
"type": "long"
}
}
}
}
This becomes a problem when I want to do a search like this:
{
"query_string": {
"query": "[2015-06-03T00:00:00.000 TO 2015-06-05T23:59:59.999]",
"fields": [
"items.*"
]
}
}
This causes exceptions, that I guess are because of all the fields in the items object are not of the same type. What is the proper mapping to searches of this type?
I was able to fix this by using the following mapping:
this.Client.Map<TestObject>(m => m.MapFromAttributes())
.Properties(p => p
.Object<Dictionary<long, List<DateTime>>>(o => o.Name("items")));
Related
I Have an index with the following mappings:
{
"winnings": {
"mappings": {
"properties": {
"handId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"playerId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"value": {
"type": "float"
}
}
}
}
}
generated from the class:
public class ElasticWinnings
{
public Guid Id { get; set; }
public Guid HandId { get; set; }
public Guid PlayerId { get; set; }
public decimal Value { get; set; }
}
I created that in nest with the ConnectionSettings:
.DefaultMappingFor<ElasticWinnings>(u =>
u.IndexName("winnings")
.IdProperty(x => x.Id)
);
when I try and run the following query:
var result = _client.Search<ElasticWinnings>(s =>
s.Aggregations(a =>
a.Terms("term_Agg", t =>
t.Field(f => f.PlayerId)
.Aggregations(aa =>
aa.Sum("sum", sum =>
sum.Field(f => f.Value))
)
))
);
I get a 400 back, with the error:
type: illegal_argument_exception Reason: "Fielddata is disabled on text fields by default
It creates this query:
{
"aggs":{
"term_Agg":{
"aggs":{
"sum":{
"sum":{
"field":"value"
}
}
},
"terms":{
"field":"playerId"
}
}
}
}
If I changed that query to:
{
"aggs":{
"term_Agg":{
"aggs":{
"sum":{
"sum":{
"field":"value"
}
}
},
"terms":{
"field":"playerId.keyword"
}
}
}
}
and used that in postman, it works.
I am not sure why it is not putting the .keyword into the query. Is it the way the nest client is configured, the indicies or the query?
You need to change your query a little bit to tell NEST to use keyword field instead of text, you can do this with .Suffix extension method. Link to docs.
var result = _client.Search<ElasticWinnings>(s =>
s.Aggregations(a =>
a.Terms("term_Agg", t =>
t.Field(f => f.PlayerId.Suffix("keyword"))
.Aggregations(aa =>
aa.Sum("sum", sum =>
sum.Field(f => f.Value))
)
))
);
Hope that helps.
The solution I found was to add [Keyword] to the PlayerId property in ElasticWinnings class.
I kept the .DefaultMappingFor<ElasticWinnings>(u => u.IndexName("winnings") in the creation of the ConnectionSettings class, but added this before the Elastic client is returned:
var client = new ElasticClient(settings);
client.Indices.Create("winnings", c =>
c.Map<ElasticWinnings>(m => m.AutoMap())
);
Without adding the section above, it did not apply the attributes. This changed my mappings (http://localhost:9200/winnings/_mappings) to
{
"winnings": {
"mappings": {
"properties": {
"handId": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"playerId": {
"type": "keyword"
},
"value": {
"type": "double"
}
}
}
}
}
This is the docs about setting up the mappings https://www.elastic.co/guide/en/elasticsearch/client/net-api/current/fluent-mapping.html
I have an index with mappings that look like this:
"mappings": {
"default": {
"_all": {
"enabled": false
},
"properties": {
"Foo": {
"properties": {
"Bar": {
"type": "keyword"
}
}
}
}
}
I am trying to change the mapping to introduce a sub-field of Bar, called Code, whilst migrating the string currently in Bar into Bar.Code. Here is the new mapping:
"mappings": {
"default": {
"_all": {
"enabled": false
},
"properties": {
"Foo": {
"properties": {
"Bar": {
"properties": {
"Code": {
"type": "keyword"
}
}
}
}
}
}
}
In order to do this, I think I need to do a _reindex and specify a pipeline. Is that correct? If so, how does my pipeline access the original data?
I have tried variations on the following code, but without success:
PUT _ingest/pipeline/transformFooBar
{
"processors": [
{
"set": {
"field": "Bar.Code",
"value": "{{_source.Bar}}"
}
}
]
}
POST _reindex
{
"source": {
"index": "foo_v1"
},
"dest": {
"index": "foo_v2",
"pipeline": "transformFooBar"
}
}
Ah, I almost had the syntax right. The _source is not required:
// Create a pipeline with a SET processor
PUT _ingest/pipeline/transformFooBar
{
"processors": [
{
"set": {
"field": "Bar.Code",
"value": "{{Bar}}"
}
}
]
}
// Reindex using the above pipeline
POST _reindex
{
"source": {
"index": "foo_v1"
},
"dest": {
"index": "foo_v2",
"pipeline": "transformFooBar"
}
}
How do you count objects of a nested field (which is a nested objects list) which meet a certain condition in ElasticSearch?
EXAMPLE
Having Customer index, with type Customer which has a Services nested field with following structure:
public class Customer
{
public int Id;
public List<Service> Services;
}
public class Service
{
public int Id;
public DateTime Date;
public decimal Rating;
}
How do I count all services which happened in June 2017 and got a rating higher than 5?
Good question :)
In order to get what you want you should define your mapping upfront and nested property mapping works well.
The nested type is a specialized version of the object datatype that allows arrays of objects to be indexed and queried independently of each other.
https://www.elastic.co/guide/en/elasticsearch/reference/2.4/nested.html
Please find below full example :)
Mapping
PUT example_nested_test
{
"mappings": {
"nested": {
"properties": {
"Id": {
"type": "long"
},
"Services": {
"type": "nested",
"properties": {
"Id": {
"type": "long"
},
"Date": {
"type": "date"
},
"Rating": {
"type": "long"
}
}
}
}
}
}
}
POST Data
POST example_nested_test/nested/100
{
"Id" : 1,
"Services": [
{
"Id": 1,
"Date" :"2017-05-10",
"Rating" : 5
},
{
"Id": 2,
"Date" :"2013-05-10",
"Rating" : 2
},
{
"Id": 4,
"Date" :"2017-05-10",
"Rating" : 6
}]
}
Query
GET example_nested_test/_search
{
"size":0,
"aggs": {
"Services": {
"nested": {
"path": "Services"
},
"aggs": {
"Rating": {
"filter": {
"bool": {
"must": [
{
"range": {
"Services.Date": {
"gt": "2017",
"format": "yyyy"
}
}
},
{
"range": {
"Services.Rating": {
"gt": "5"
}
}
}
]
}
}
}
}
}
}
}
Result :
"aggregations": {
"Services": {
"doc_count": 3,
"Rating": {
"doc_count": 1
}
}
}
I'm trying use a dynamic template in ES such that all string fields are multifields. I also want to apply some specific mappings to certain fields.
Take the following example class:
[ElasticsearchType(Name = "sample1")]
public class Sample1
{
public string ID { get; set; }
[String(Index = FieldIndexOption.No)]
public string DoNotIndex { get; set; }
public string MultiField1 { get; set; }
public string MultiField2 { get; set; }
}
I want to then create the dynamic template and apply the mapping to DoNotIndex using the following command:
_client.Map<Sample1>(m => m
.AutoMap()
.DynamicTemplates(dt=> dt
.DynamicTemplate("all_strings_multifields", t => t
.MatchMappingType("string")
.Mapping(tm => tm
.String(mf => mf
.Index(FieldIndexOption.Analyzed)
.Fields(mff => mff
.String(s => s
.Name("raw")
.Index(FieldIndexOption.NotAnalyzed)
)
)
)
)
)
)
)
.VerifySuccessfulResponse();
The result is:
{
"test1": {
"mappings": {
"sample1": {
"dynamic_templates": [
{
"all_strings_multifields": {
"match_mapping_type": "string",
"mapping": {
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
},
"index": "analyzed",
"type": "string"
}
}
}
],
"properties": {
"doNotIndex": {
"type": "keyword",
"index": false
},
"iD": {
"type": "text"
},
"multiField1": {
"type": "text"
},
"multiField2": {
"type": "text"
}
}
}
}
}
}
Results
you'll see that the DoNotIndex property is indeed correct, but the multifield1 and multifield2 are not correct (they are not multi fields).
Workaround
I know I can "fix" this by NOT doing the AutoMap() and instead specifying each of the special indexes but there are a lot of fields and that isn't as clean of a solution.
Can I do AutoMap with DynamicTemplates?
Dynamic templates only apply to fields that are dynamically added to a mapping, so properties explicitly mapped with .AutoMap() will not be affected by dynamic mapping.
There is however a way to apply conventions to explicit mappings with NEST using the visitor pattern. It looks like you're using Elasticsearch 5.0, so you should use the text and keyword mappings.
First define a visitor
[ElasticsearchType(Name = "sample1")]
public class Sample1
{
public string ID { get; set; }
[Keyword(Index = false)]
public string DoNotIndex { get; set; }
public string MultiField1 { get; set; }
public string MultiField2 { get; set; }
}
public class AllStringsMultiFieldsVisitor : NoopPropertyVisitor
{
public override void Visit(ITextProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute)
{
// if a custom attribute has been applied, let it take precedence
if (propertyInfo.GetCustomAttribute<ElasticsearchPropertyAttributeBase>() == null)
{
type.Fields = new Properties
{
{
"raw", new KeywordProperty()
}
};
}
base.Visit(type, propertyInfo, attribute);
}
}
Then pass an instance of the visitor to .AutoMap()
client.Map<Sample1>(m => m
.AutoMap(new AllStringsMultiFieldsVisitor())
.DynamicTemplates(dt => dt
.DynamicTemplate("all_strings_multifields", t => t
.MatchMappingType("text")
.Mapping(tm => tm
.Text(mf => mf
.Index(true)
.Fields(mff => mff
.Keyword(s => s
.Name("raw")
)
)
)
)
)
)
);
produces
{
"dynamic_templates": [
{
"all_strings_multifields": {
"match_mapping_type": "text",
"mapping": {
"type": "text",
"fields": {
"raw": {
"type": "keyword"
}
},
"index": true
}
}
}
],
"properties": {
"iD": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"doNotIndex": {
"type": "keyword",
"index": false
},
"multiField1": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"multiField2": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
}
}
}
I should point out however that the default automapping for a C# string property in NEST 5.0 is to map it as a text field with a keyword sub field with ignore_above:256. NEST 5.0 was released to nuget earlier this week
client.Map<Sample1>(m => m
.AutoMap()
.DynamicTemplates(dt => dt
.DynamicTemplate("all_strings_multifields", t => t
.MatchMappingType("text")
.Mapping(tm => tm
.Text(mf => mf
.Index(true)
.Fields(mff => mff
.Keyword(s => s
.Name("raw")
)
)
)
)
)
)
);
produces
{
"dynamic_templates": [
{
"all_strings_multifields": {
"match_mapping_type": "text",
"mapping": {
"type": "text",
"fields": {
"raw": {
"type": "keyword"
}
},
"index": true
}
}
}
],
"properties": {
"iD": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"doNotIndex": {
"type": "keyword",
"index": false
},
"multiField1": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"multiField2": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
}
In between projects, started to play around with Elastic Search and NEST in preparation for some auto-completion features coming up.
I followed examples here and in the various docs ... and it seems like what I have should work. There must be some subtle difference with what i am doing.
This is the object that represents my document.
[ElasticType(IdProperty="KeyInstn")]
public class Instn
{
[ElasticProperty(Name = "keyinstn", Index = FieldIndexOption.Analyzed, Type = FieldType.Integer)]
public int KeyInstn { get; set; }
public string Type { get; set; }
[ElasticProperty(Name = "name", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Name { get; set; }
[ElasticProperty(Name = "ticker", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Ticker { get; set; }
[ElasticProperty(Name = "street", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Street { get; set; }
[ElasticProperty(Name = "city", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string City { get; set; }
[ElasticProperty(Name = "state", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string State { get; set; }
[ElasticProperty(Name = "zip", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Zip { get; set; }
[ElasticProperty(Name = "country", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Country { get; set; }
[ElasticProperty(Name = "suggest", Type = FieldType.Completion)]
public CompletionField Suggest { get; set; }
}
This is the code that I use to create my index:
var index = client.CreateIndex("snl", c => c
.NumberOfReplicas(1)
.NumberOfShards(5)
.Settings(s => s
.Add("merge.policy.merge_factor", "10")
.Add("search.slowlog.threshold.fetch.warn", "1s")
)
.AddMapping<Instn>(m => m.MapFromAttributes()
.Properties(props => props
.Completion(s => s
.Name(p => p.Suggest)
.IndexAnalyzer("simple")
.SearchAnalyzer("simple")
.MaxInputLength(20)
.Payloads()
.PreservePositionIncrements()
.PreserveSeparators()
)
)
));
I am populating my documents using IndexMany in batches of 1000 documents.
I then perform my query:
var suggestResponse = this._client.Suggest<Instn>(s => s
.Index("snl")
.Completion("suggest", c => c
.Text(this.AutoCompleteText.Text)
.OnField("suggest")));
The error I receive is:
Field [suggest] is not a completion suggest field
Using SENSE, I pulls the mappings and notice that "suggest" doesn't have a type. The examples I followed indicate that the type should be 'completion.:'
Elasticsearch - Autocomplete with NEST
{
"snl": {
"mappings": {
"instn": {
"properties": {
"city": {
"type": "string"
},
"country": {
"type": "string"
},
"keyinstn": {
"type": "long"
},
"name": {
"type": "string"
},
"state": {
"type": "string"
},
"suggest": {
"properties": {
"input": {
"type": "string"
},
"output": {
"type": "string"
},
"payload": {
"type": "long"
},
"weight": {
"type": "double"
}
}
},
"type": {
"type": "string"
},
"zip": {
"type": "string"
}
}
}
}
}
}
In place of the CompletionObject I created myself, I also tried the intrinsic object, "SuggestField."
If anyone could point me in the right direction of what I am doing wrong, I'd appreciate it. FWIW "normal" searches work fine and return appropriate results.
Thanks.
UPDATE 1/26/2016
So, managed to get an index created that looks "better" ... I was able to get around the new error "analyzer on completion field must be set when search_analyzer is set" by omitting the search analyzer during creation and now my index looks like this:
{
"snl": {
"aliases": {},
"mappings": {
"instn": {
"properties": {
"city": {
"type": "string"
},
"country": {
"type": "string"
},
"keyinstn": {
"type": "long"
},
"name": {
"type": "string"
},
"state": {
"type": "string"
},
"suggest": {
"properties": {
"completion": {
"properties": {
"field": {
"type": "string"
}
}
},
"input": {
"type": "string"
},
"output": {
"type": "string"
},
"payload": {
"type": "long"
},
"text": {
"type": "string"
},
"weight": {
"type": "double"
}
}
},
"type": {
"type": "string"
},
"zip": {
"type": "string"
}
}
}
},
"settings": {
"index": {
"creation_date": "1453835225862",
"number_of_shards": "5",
"number_of_replicas": "1",
"uuid": "90QKK1OyRMKtwcyjAwPElA",
"version": {
"created": "2010199"
}
}
},
"warmers": {}
}
}
However, no completion searches return any results. Is it because the search analyzer omission? If so, how do I get around the "analyzer on completion field must be set when search_analyzer is set" error?