Problem in nest query for sorting in aggregation - elasticsearch

I need to get a query from Elasticsearch for last registered records in specified area in range of specified serial numbers.
for this reason I mapped my index in this shape:
{ "settings": {
"index": {
"number_of_shards": 5,
"number_of_replicas": 2
}
},
"mapping": {
"AssetStatus": {
"properties": {
"serialnumber": {
"type": "text",
"fielddata": true
},
"vehiclestate": {
"type": "text",
"fielddata": true
},
"vehiclegeopoint": {
"type": "geo-point",
"fielddata": true
},
"vehiclespeed": {
"type": "number",
"fielddata": true
},
"vehiclefuelpercent": {
"type": "text",
"fielddata": true
},
"devicebatterypercent": {
"type": "text",
"fielddata": true
},
"networklatency": {
"type": "text",
"fielddata": true
},
"satellitescount": {
"type": "number",
"fielddata": true
},
"createdate": {
"type": "date",
"fielddata": true
}
}
}
}
}
and this query works correctly
{
"query": {
"bool": {
"must": [
{
"term": {
"serialnumber.keyword": "2228187d-b1a5-4e18-82bb-4d12438e0ec0"
}
},
{
"range": {
"vehiclegeopoint.lat": {
"gt": "31.287958",
"lt": "31.295485"
}
}
},
{
"range": {
"vehiclegeopoint.lon": {
"gt": "48.639844",
"lt": "48.652032"
}
}
}
],
"must_not": [],
"should": []
}
},
"from": 0,
"size": 0,
"sort": [],
"aggs": {
"SerialNumberGroups": {
"terms": {
"field": "serialnumber.keyword"
},
"aggs": {
"tops": {
"top_hits": {
"sort": [
{
"createdate.keyword": {
"order": "desc"
}
}
],
"size": 1
}
}
}
}
}
}
whereas my nest query has this error
Invalid NEST response built from a unsuccessful low level call on
POST: /fms2/AssetStatus/_search?typed_keys=true
Audit trail of this API call:
[1] BadResponse: Node: http://localhost:9200/ Took: 00:00:00.1917118
OriginalException: Elasticsearch.Net.ElasticsearchClientException: Request failed to execute. Call: Status code 400 from: POST
/fms2/AssetStatus/_search?typed_keys=true. ServerError: Type:
parsing_exception Reason: "Unknown key for a VALUE_STRING in [field]."
Request:
force it to be set on the response.>
Response:
ConnectionSettings to force it to be set on the response.>
my nest query is this
var searchResponse =
Client.Search<AssetStatus>(x => x
.Index(settings.DefaultIndex)
.Type("AssetStatus")
.Query(fq => fq.GeoBoundingBox(c => c.Field(f => f.VehicleGeoPoint).BoundingBox(new GeoLocation(TopLeft.Lat, TopLeft.Lon), new GeoLocation(BottomRight.Lat, BottomRight.Lon))))
.Query(fq =>
fq.Bool(b => b.
Filter(
f => f.Match(m => m.Field(g => g.SerialNumber.Suffix("keyword").Equals(sn)))
)
))
.Aggregations(a => a
.Terms("group_by_SerialNumber", st => st
.Field(o => o.SerialNumber.Suffix("keyword"))
.Size(0)
.Aggregations(b=> b.TopHits("top_hits", lastRegistered => lastRegistered
.Field(bf=> bf.CreateDate.Suffix("keyword"))
.Size(1)))
))
);
This problem is for sorting in aggregation

Problem was in my POCO, as you can see in my nest query I use uppercase letters for naming my properties. I should use Nest library for using data annotation for elastic's POCO.
[ElasticsearchType(Name = "AssetStatus")]
public class AssetStatus
{
[Text]
[PropertyName("serialnumber")]
public string SerialNumber { get; set; }
[Text]
[PropertyName("vehiclestate")]
public string VehicleState { get; set; }
[GeoPoint]
[PropertyName("vehiclegeopoint")]
public GeoPoint VehicleGeoPoint { get; set; }
[Number]
[PropertyName("vehiclespeed")]
public int VehicleSpeed { get; set; }
[Text]
[PropertyName("vehiclefuelpercent")]
public string VehicleFuelPercent { get; set; }
[Text]
[PropertyName("devicebatterypercent")]
public string DeviceBatteryPercent { get; set; }
[Text]
[PropertyName("networklatency")]
public string NetworkLatency { get; set; }
[Number]
[PropertyName("satellitescount")]
public byte SatellitesCount { get; set; }
[Date]
[PropertyName("createdate")]
public string CreateDate { get; set; }
}

Related

elasticsearch 7 nest aggregation text keyword error

I Have an index with the following mappings:
{
"winnings": {
"mappings": {
"properties": {
"handId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"playerId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"value": {
"type": "float"
}
}
}
}
}
generated from the class:
public class ElasticWinnings
{
public Guid Id { get; set; }
public Guid HandId { get; set; }
public Guid PlayerId { get; set; }
public decimal Value { get; set; }
}
I created that in nest with the ConnectionSettings:
.DefaultMappingFor<ElasticWinnings>(u =>
u.IndexName("winnings")
.IdProperty(x => x.Id)
);
when I try and run the following query:
var result = _client.Search<ElasticWinnings>(s =>
s.Aggregations(a =>
a.Terms("term_Agg", t =>
t.Field(f => f.PlayerId)
.Aggregations(aa =>
aa.Sum("sum", sum =>
sum.Field(f => f.Value))
)
))
);
I get a 400 back, with the error:
type: illegal_argument_exception Reason: "Fielddata is disabled on text fields by default
It creates this query:
{
"aggs":{
"term_Agg":{
"aggs":{
"sum":{
"sum":{
"field":"value"
}
}
},
"terms":{
"field":"playerId"
}
}
}
}
If I changed that query to:
{
"aggs":{
"term_Agg":{
"aggs":{
"sum":{
"sum":{
"field":"value"
}
}
},
"terms":{
"field":"playerId.keyword"
}
}
}
}
and used that in postman, it works.
I am not sure why it is not putting the .keyword into the query. Is it the way the nest client is configured, the indicies or the query?
You need to change your query a little bit to tell NEST to use keyword field instead of text, you can do this with .Suffix extension method. Link to docs.
var result = _client.Search<ElasticWinnings>(s =>
s.Aggregations(a =>
a.Terms("term_Agg", t =>
t.Field(f => f.PlayerId.Suffix("keyword"))
.Aggregations(aa =>
aa.Sum("sum", sum =>
sum.Field(f => f.Value))
)
))
);
Hope that helps.
The solution I found was to add [Keyword] to the PlayerId property in ElasticWinnings class.
I kept the .DefaultMappingFor<ElasticWinnings>(u => u.IndexName("winnings") in the creation of the ConnectionSettings class, but added this before the Elastic client is returned:
var client = new ElasticClient(settings);
client.Indices.Create("winnings", c =>
c.Map<ElasticWinnings>(m => m.AutoMap())
);
Without adding the section above, it did not apply the attributes. This changed my mappings (http://localhost:9200/winnings/_mappings) to
{
"winnings": {
"mappings": {
"properties": {
"handId": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"playerId": {
"type": "keyword"
},
"value": {
"type": "double"
}
}
}
}
}
This is the docs about setting up the mappings https://www.elastic.co/guide/en/elasticsearch/client/net-api/current/fluent-mapping.html

How to create bool filter query with nested bool should query using NEST OIS

I'm having trouble with using NEST to create this query:
{
"size": 0,
"query": {
"bool": {
"filter": [
{"term": {"status": 0}},
{"term": {"freeRound": false }},
{"term": {"playerId": 189149 }},
{
"range": {
"eventTime": {
"format": "strict_date_optional_time",
"gte": "2019-05-02T00:00:00",
"lte": "2019-05-02T23:59:59"
}
}
},
{
"bool": {
"should": [
{"term": {"extGameId": 527 }},
{"term": {"extGameId": 804 }},
{"term": {"extGameId": 9503 }}
],
"minimum_should_match": 1
}
},
{"term": {"transactionType": 3 }}
]
}
},
"aggs": {
"byGames": {
"terms": {"field": "extGameId" }
},
"sumAgg": {
"sum": {"field": "amount"}
}
}
}
As can be seen, my query is using bool filter as I don't need scoring. Inside I have those TermQuery with DateRange part which should be met (status, freeRound, playerId and Date range), and then a sub bool should part, saying that one of extGamesId must be met.
Now here is my code using NEST trying to achieve same:
//playerId, gameList, startDate and endDate are passed as a parameters
var filters = new List<QueryContainer>();
filters.Add(new TermQuery { Field = new Field("status"), Value = "0" });//status=0
filters.Add(new TermQuery {Field = new Field("freeRound"), Value = false});//freeRound = false
filters.Add(new TermQuery { Field = new Field("playerId"), Value = playerId });//playerId
filters.Add(endDate != null ?
new DateRangeQuery
{
Field = new Field("eventTime"),
GreaterThanOrEqualTo = startDate,
LessThanOrEqualTo = endDate
} :
new DateRangeQuery
{
Field = new Field("eventTime"),
GreaterThanOrEqualTo = startDate
});
if (gameList.Count > 0)
{
var shouldFilter = new List<QueryContainer>();
gameList.ForEach(g => shouldFilter.Add(new TermQuery { Field = new Field("extGameId"), Value = g }));//extGameIds
filters.Add(new BoolQuery { Should = shouldFilter, MinimumShouldMatch = 1 });
}
filters.Add(new TermQuery { Field = new Field("transactionType"), Value = 3 });//transactionType
var searchRequest = new SearchRequest<SomethingHere>()
{
Query = new BoolQuery
{
Filter = filters
},
Aggregations = new SumAggregation("sum_agg", "amount")
};
var responsesResult = _client.Search<SomethingHere>(searchRequest);
However, I'm getting "Input string was not in a correct format." error.
Please help me understand what I'm doing wrong here. Thanks in advance.
EDIT:
Here are the relevant mappings:
"amount": {
"type": "double"
},
"eventTime": {
"type": "date"
},
"extGameId": {
"type": "keyword"
},
"freeRound": {
"type": "boolean"
},
"playerId": {
"type": "long"
},
"status": {
"type": "keyword"
}
"transactionType": {
"type": "short"
}
And here is POCO:
[Number(NumberType.Double)]
public decimal Amount { get; set; }
[Date]
public DateTime eventTime { get; set; }
[Keyword]
public string ExtGameId { get; set; }
[Boolean]
public decimal FreeRound { get; set; }
[Number(NumberType.Long)]
public long PlayerId { get; set; }
[Keyword]
public string Status { get; set; }
[Number(NumberType.Short)]
public int TransactionType { get; set; }

How to sort an object by it's child object fields?

How could I sort products by GroupModel.PeerOrder but only if GroupModel.ParentGroupId matches some id?
My C# models:
public class ProductModel
{
public int Id { get; set; }
public int Title { get; set; }
public List<GroupModel> Groups { get; set; }
}
public class GroupModel
{
public int Id { get; set; }
public int Title { get; set; }
public int ParentGroupId { get; set; }
public int PeerOrder { get; set; }
}
Use Nested Query to filter matching "GroupModel.ParentGroupId" values and then apply Nested Sort Query to sort results by "GroupModel.PeerOrder".
As per documentation:
Nested Query: Nested query allows to query nested objects / docs (see nested mapping). The query is executed against the nested objects / docs as if they were indexed as separate docs (they are, internally) and resulting in the root parent doc (or parent nested mapping). Here is a sample mapping:
PUT /my_index
{
"mappings": {
"_doc" : {
"properties" : {
"obj1" : {
"type" : "nested"
}
}
}
}
}
GET /_search
{
"query": {
"nested" : {
"path" : "obj1",
"score_mode" : "avg",
"query" : {
"bool" : {
"must" : [
{ "match" : {"obj1.name" : "blue"} },
{ "range" : {"obj1.count" : {"gt" : 5}} }
]
}
}
}
}
}
Nested Sort Query: It is possible to sort by the value of a nested field, even though the value exists in a separate nested document.
GET /_search
{
"query": {
"nested": {
"path": "comments",
"filter": {
"range": {
"comments.date": {
"gte": "2014-10-01",
"lt": "2014-11-01"
}
}
}
}
},
"sort": {
"comments.stars": {
"order": "asc",
"mode": "min",
"nested_filter": {
"range": {
"comments.date": {
"gte": "2014-10-01",
"lt": "2014-11-01"
}
}
}
}
}
}
This is what I ended up with (With the help from #ydrall - thank you very much! :>):
The query part (for us is works just fine without this first part):
"query": {
"bool": {
"must": [
{
"nested": {
"path": "groups",
"query": {
"bool": {
"must": [
{
"match": {
"groups.parentGroupId": 3
}
}
]
}
}
}
}
]
}
}
The sorting part of the query:
"sort": [
{
"groups.peerOrder": {
"order": "asc",
"nested_path": "groups",
"nested_filter": {
"match": {
"groups.parentGroupId": 3
}
}
}
}
Index mappings:
"mappings": {
"productmodel": {
"properties": {
"groups": {
"type": "nested",
"properties": {
"id": {
"type": "integer"
},
"parentGroupId": {
"type": "integer"
},
"peerOrder": {
"type": "integer"
}
}
}
}
}
}

NEST / Elastic Search Auto-Completion 'Field [suggest] is not a completion suggest field'

In between projects, started to play around with Elastic Search and NEST in preparation for some auto-completion features coming up.
I followed examples here and in the various docs ... and it seems like what I have should work. There must be some subtle difference with what i am doing.
This is the object that represents my document.
[ElasticType(IdProperty="KeyInstn")]
public class Instn
{
[ElasticProperty(Name = "keyinstn", Index = FieldIndexOption.Analyzed, Type = FieldType.Integer)]
public int KeyInstn { get; set; }
public string Type { get; set; }
[ElasticProperty(Name = "name", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Name { get; set; }
[ElasticProperty(Name = "ticker", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Ticker { get; set; }
[ElasticProperty(Name = "street", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Street { get; set; }
[ElasticProperty(Name = "city", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string City { get; set; }
[ElasticProperty(Name = "state", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string State { get; set; }
[ElasticProperty(Name = "zip", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Zip { get; set; }
[ElasticProperty(Name = "country", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Country { get; set; }
[ElasticProperty(Name = "suggest", Type = FieldType.Completion)]
public CompletionField Suggest { get; set; }
}
This is the code that I use to create my index:
var index = client.CreateIndex("snl", c => c
.NumberOfReplicas(1)
.NumberOfShards(5)
.Settings(s => s
.Add("merge.policy.merge_factor", "10")
.Add("search.slowlog.threshold.fetch.warn", "1s")
)
.AddMapping<Instn>(m => m.MapFromAttributes()
.Properties(props => props
.Completion(s => s
.Name(p => p.Suggest)
.IndexAnalyzer("simple")
.SearchAnalyzer("simple")
.MaxInputLength(20)
.Payloads()
.PreservePositionIncrements()
.PreserveSeparators()
)
)
));
I am populating my documents using IndexMany in batches of 1000 documents.
I then perform my query:
var suggestResponse = this._client.Suggest<Instn>(s => s
.Index("snl")
.Completion("suggest", c => c
.Text(this.AutoCompleteText.Text)
.OnField("suggest")));
The error I receive is:
Field [suggest] is not a completion suggest field
Using SENSE, I pulls the mappings and notice that "suggest" doesn't have a type. The examples I followed indicate that the type should be 'completion.:'
Elasticsearch - Autocomplete with NEST
{
"snl": {
"mappings": {
"instn": {
"properties": {
"city": {
"type": "string"
},
"country": {
"type": "string"
},
"keyinstn": {
"type": "long"
},
"name": {
"type": "string"
},
"state": {
"type": "string"
},
"suggest": {
"properties": {
"input": {
"type": "string"
},
"output": {
"type": "string"
},
"payload": {
"type": "long"
},
"weight": {
"type": "double"
}
}
},
"type": {
"type": "string"
},
"zip": {
"type": "string"
}
}
}
}
}
}
In place of the CompletionObject I created myself, I also tried the intrinsic object, "SuggestField."
If anyone could point me in the right direction of what I am doing wrong, I'd appreciate it. FWIW "normal" searches work fine and return appropriate results.
Thanks.
UPDATE 1/26/2016
So, managed to get an index created that looks "better" ... I was able to get around the new error "analyzer on completion field must be set when search_analyzer is set" by omitting the search analyzer during creation and now my index looks like this:
{
"snl": {
"aliases": {},
"mappings": {
"instn": {
"properties": {
"city": {
"type": "string"
},
"country": {
"type": "string"
},
"keyinstn": {
"type": "long"
},
"name": {
"type": "string"
},
"state": {
"type": "string"
},
"suggest": {
"properties": {
"completion": {
"properties": {
"field": {
"type": "string"
}
}
},
"input": {
"type": "string"
},
"output": {
"type": "string"
},
"payload": {
"type": "long"
},
"text": {
"type": "string"
},
"weight": {
"type": "double"
}
}
},
"type": {
"type": "string"
},
"zip": {
"type": "string"
}
}
}
},
"settings": {
"index": {
"creation_date": "1453835225862",
"number_of_shards": "5",
"number_of_replicas": "1",
"uuid": "90QKK1OyRMKtwcyjAwPElA",
"version": {
"created": "2010199"
}
}
},
"warmers": {}
}
}
However, no completion searches return any results. Is it because the search analyzer omission? If so, how do I get around the "analyzer on completion field must be set when search_analyzer is set" error?

How to exclude inherited object properties from mappings

I'm trying to setup a mapping for an object that looks like this:
class TestObject
{
public long TestID { get; set; }
[ElasticProperty(Type = FieldType.Object)]
public Dictionary<long, List<DateTime>> Items { get; set; }
}
I use the following mapping code (where Client is IElasticClient):
this.Client.Map<TestObject>(m => m.MapFromAttributes());
I get the following mapping result:
{
"mappings": {
"testobject": {
"properties": {
"items": {
"properties": {
"comparer": {
"type": "object"
},
"count": {
"type": "integer"
},
"item": {
"type": "date",
"format": "dateOptionalTime"
},
"keys": {
"properties": {
"count": {
"type": "integer"
}
}
},
"values": {
"properties": {
"count": {
"type": "integer"
}
}
}
}
},
"testID": {
"type": "long"
}
}
}
}
This becomes a problem when I want to do a search like this:
{
"query_string": {
"query": "[2015-06-03T00:00:00.000 TO 2015-06-05T23:59:59.999]",
"fields": [
"items.*"
]
}
}
This causes exceptions, that I guess are because of all the fields in the items object are not of the same type. What is the proper mapping to searches of this type?
I was able to fix this by using the following mapping:
this.Client.Map<TestObject>(m => m.MapFromAttributes())
.Properties(p => p
.Object<Dictionary<long, List<DateTime>>>(o => o.Name("items")));

Resources