How to create bool filter query with nested bool should query using NEST OIS - elasticsearch

I'm having trouble with using NEST to create this query:
{
"size": 0,
"query": {
"bool": {
"filter": [
{"term": {"status": 0}},
{"term": {"freeRound": false }},
{"term": {"playerId": 189149 }},
{
"range": {
"eventTime": {
"format": "strict_date_optional_time",
"gte": "2019-05-02T00:00:00",
"lte": "2019-05-02T23:59:59"
}
}
},
{
"bool": {
"should": [
{"term": {"extGameId": 527 }},
{"term": {"extGameId": 804 }},
{"term": {"extGameId": 9503 }}
],
"minimum_should_match": 1
}
},
{"term": {"transactionType": 3 }}
]
}
},
"aggs": {
"byGames": {
"terms": {"field": "extGameId" }
},
"sumAgg": {
"sum": {"field": "amount"}
}
}
}
As can be seen, my query is using bool filter as I don't need scoring. Inside I have those TermQuery with DateRange part which should be met (status, freeRound, playerId and Date range), and then a sub bool should part, saying that one of extGamesId must be met.
Now here is my code using NEST trying to achieve same:
//playerId, gameList, startDate and endDate are passed as a parameters
var filters = new List<QueryContainer>();
filters.Add(new TermQuery { Field = new Field("status"), Value = "0" });//status=0
filters.Add(new TermQuery {Field = new Field("freeRound"), Value = false});//freeRound = false
filters.Add(new TermQuery { Field = new Field("playerId"), Value = playerId });//playerId
filters.Add(endDate != null ?
new DateRangeQuery
{
Field = new Field("eventTime"),
GreaterThanOrEqualTo = startDate,
LessThanOrEqualTo = endDate
} :
new DateRangeQuery
{
Field = new Field("eventTime"),
GreaterThanOrEqualTo = startDate
});
if (gameList.Count > 0)
{
var shouldFilter = new List<QueryContainer>();
gameList.ForEach(g => shouldFilter.Add(new TermQuery { Field = new Field("extGameId"), Value = g }));//extGameIds
filters.Add(new BoolQuery { Should = shouldFilter, MinimumShouldMatch = 1 });
}
filters.Add(new TermQuery { Field = new Field("transactionType"), Value = 3 });//transactionType
var searchRequest = new SearchRequest<SomethingHere>()
{
Query = new BoolQuery
{
Filter = filters
},
Aggregations = new SumAggregation("sum_agg", "amount")
};
var responsesResult = _client.Search<SomethingHere>(searchRequest);
However, I'm getting "Input string was not in a correct format." error.
Please help me understand what I'm doing wrong here. Thanks in advance.
EDIT:
Here are the relevant mappings:
"amount": {
"type": "double"
},
"eventTime": {
"type": "date"
},
"extGameId": {
"type": "keyword"
},
"freeRound": {
"type": "boolean"
},
"playerId": {
"type": "long"
},
"status": {
"type": "keyword"
}
"transactionType": {
"type": "short"
}
And here is POCO:
[Number(NumberType.Double)]
public decimal Amount { get; set; }
[Date]
public DateTime eventTime { get; set; }
[Keyword]
public string ExtGameId { get; set; }
[Boolean]
public decimal FreeRound { get; set; }
[Number(NumberType.Long)]
public long PlayerId { get; set; }
[Keyword]
public string Status { get; set; }
[Number(NumberType.Short)]
public int TransactionType { get; set; }

Related

How to sort an object by it's child object fields?

How could I sort products by GroupModel.PeerOrder but only if GroupModel.ParentGroupId matches some id?
My C# models:
public class ProductModel
{
public int Id { get; set; }
public int Title { get; set; }
public List<GroupModel> Groups { get; set; }
}
public class GroupModel
{
public int Id { get; set; }
public int Title { get; set; }
public int ParentGroupId { get; set; }
public int PeerOrder { get; set; }
}
Use Nested Query to filter matching "GroupModel.ParentGroupId" values and then apply Nested Sort Query to sort results by "GroupModel.PeerOrder".
As per documentation:
Nested Query: Nested query allows to query nested objects / docs (see nested mapping). The query is executed against the nested objects / docs as if they were indexed as separate docs (they are, internally) and resulting in the root parent doc (or parent nested mapping). Here is a sample mapping:
PUT /my_index
{
"mappings": {
"_doc" : {
"properties" : {
"obj1" : {
"type" : "nested"
}
}
}
}
}
GET /_search
{
"query": {
"nested" : {
"path" : "obj1",
"score_mode" : "avg",
"query" : {
"bool" : {
"must" : [
{ "match" : {"obj1.name" : "blue"} },
{ "range" : {"obj1.count" : {"gt" : 5}} }
]
}
}
}
}
}
Nested Sort Query: It is possible to sort by the value of a nested field, even though the value exists in a separate nested document.
GET /_search
{
"query": {
"nested": {
"path": "comments",
"filter": {
"range": {
"comments.date": {
"gte": "2014-10-01",
"lt": "2014-11-01"
}
}
}
}
},
"sort": {
"comments.stars": {
"order": "asc",
"mode": "min",
"nested_filter": {
"range": {
"comments.date": {
"gte": "2014-10-01",
"lt": "2014-11-01"
}
}
}
}
}
}
This is what I ended up with (With the help from #ydrall - thank you very much! :>):
The query part (for us is works just fine without this first part):
"query": {
"bool": {
"must": [
{
"nested": {
"path": "groups",
"query": {
"bool": {
"must": [
{
"match": {
"groups.parentGroupId": 3
}
}
]
}
}
}
}
]
}
}
The sorting part of the query:
"sort": [
{
"groups.peerOrder": {
"order": "asc",
"nested_path": "groups",
"nested_filter": {
"match": {
"groups.parentGroupId": 3
}
}
}
}
Index mappings:
"mappings": {
"productmodel": {
"properties": {
"groups": {
"type": "nested",
"properties": {
"id": {
"type": "integer"
},
"parentGroupId": {
"type": "integer"
},
"peerOrder": {
"type": "integer"
}
}
}
}
}
}

Problem in nest query for sorting in aggregation

I need to get a query from Elasticsearch for last registered records in specified area in range of specified serial numbers.
for this reason I mapped my index in this shape:
{ "settings": {
"index": {
"number_of_shards": 5,
"number_of_replicas": 2
}
},
"mapping": {
"AssetStatus": {
"properties": {
"serialnumber": {
"type": "text",
"fielddata": true
},
"vehiclestate": {
"type": "text",
"fielddata": true
},
"vehiclegeopoint": {
"type": "geo-point",
"fielddata": true
},
"vehiclespeed": {
"type": "number",
"fielddata": true
},
"vehiclefuelpercent": {
"type": "text",
"fielddata": true
},
"devicebatterypercent": {
"type": "text",
"fielddata": true
},
"networklatency": {
"type": "text",
"fielddata": true
},
"satellitescount": {
"type": "number",
"fielddata": true
},
"createdate": {
"type": "date",
"fielddata": true
}
}
}
}
}
and this query works correctly
{
"query": {
"bool": {
"must": [
{
"term": {
"serialnumber.keyword": "2228187d-b1a5-4e18-82bb-4d12438e0ec0"
}
},
{
"range": {
"vehiclegeopoint.lat": {
"gt": "31.287958",
"lt": "31.295485"
}
}
},
{
"range": {
"vehiclegeopoint.lon": {
"gt": "48.639844",
"lt": "48.652032"
}
}
}
],
"must_not": [],
"should": []
}
},
"from": 0,
"size": 0,
"sort": [],
"aggs": {
"SerialNumberGroups": {
"terms": {
"field": "serialnumber.keyword"
},
"aggs": {
"tops": {
"top_hits": {
"sort": [
{
"createdate.keyword": {
"order": "desc"
}
}
],
"size": 1
}
}
}
}
}
}
whereas my nest query has this error
Invalid NEST response built from a unsuccessful low level call on
POST: /fms2/AssetStatus/_search?typed_keys=true
Audit trail of this API call:
[1] BadResponse: Node: http://localhost:9200/ Took: 00:00:00.1917118
OriginalException: Elasticsearch.Net.ElasticsearchClientException: Request failed to execute. Call: Status code 400 from: POST
/fms2/AssetStatus/_search?typed_keys=true. ServerError: Type:
parsing_exception Reason: "Unknown key for a VALUE_STRING in [field]."
Request:
force it to be set on the response.>
Response:
ConnectionSettings to force it to be set on the response.>
my nest query is this
var searchResponse =
Client.Search<AssetStatus>(x => x
.Index(settings.DefaultIndex)
.Type("AssetStatus")
.Query(fq => fq.GeoBoundingBox(c => c.Field(f => f.VehicleGeoPoint).BoundingBox(new GeoLocation(TopLeft.Lat, TopLeft.Lon), new GeoLocation(BottomRight.Lat, BottomRight.Lon))))
.Query(fq =>
fq.Bool(b => b.
Filter(
f => f.Match(m => m.Field(g => g.SerialNumber.Suffix("keyword").Equals(sn)))
)
))
.Aggregations(a => a
.Terms("group_by_SerialNumber", st => st
.Field(o => o.SerialNumber.Suffix("keyword"))
.Size(0)
.Aggregations(b=> b.TopHits("top_hits", lastRegistered => lastRegistered
.Field(bf=> bf.CreateDate.Suffix("keyword"))
.Size(1)))
))
);
This problem is for sorting in aggregation
Problem was in my POCO, as you can see in my nest query I use uppercase letters for naming my properties. I should use Nest library for using data annotation for elastic's POCO.
[ElasticsearchType(Name = "AssetStatus")]
public class AssetStatus
{
[Text]
[PropertyName("serialnumber")]
public string SerialNumber { get; set; }
[Text]
[PropertyName("vehiclestate")]
public string VehicleState { get; set; }
[GeoPoint]
[PropertyName("vehiclegeopoint")]
public GeoPoint VehicleGeoPoint { get; set; }
[Number]
[PropertyName("vehiclespeed")]
public int VehicleSpeed { get; set; }
[Text]
[PropertyName("vehiclefuelpercent")]
public string VehicleFuelPercent { get; set; }
[Text]
[PropertyName("devicebatterypercent")]
public string DeviceBatteryPercent { get; set; }
[Text]
[PropertyName("networklatency")]
public string NetworkLatency { get; set; }
[Number]
[PropertyName("satellitescount")]
public byte SatellitesCount { get; set; }
[Date]
[PropertyName("createdate")]
public string CreateDate { get; set; }
}

Elasticsearch 2.3 - Nest - using Phonetic Analyzer

I am using Elasticsearch 2.3 - Nest API to search data. I am using attribute mapping for documents. I would like to know how to use Phonetic Analyzer using attribute mapping.
The document class:
[ElasticsearchType(Name = "contact", IdProperty = nameof(EntityId))]
public class ESContactSearchDocument
{
[String(Name = "entityid")]
public string EntityId { get; set; }
[String(Name = "id")]
public string Id { get; set; }
[String(Name = "name")]
public string Name { get; set; }
[String(Name = "domain")]
public string Domain { get; set; }
[String(Name = "description")]
public string Description { get; set; }
[String(Name = "email")]
public string Email { get; set; }
[String(Name = "firstname", Analyzer = "phonetic")]
public string FirstName { get; set; }
[String(Name = "lastname", Analyzer = "phonetic")]
public string LastName { get; set; }
}
Index creation and insertion:
var createIndexResponse = serviceClient.CreateIndex(indexName, c => c.Mappings(m => m.Map<ESContactSearchDocument>(d => d.AutoMap())));
var indexManyResponse = await serviceClient.IndexManyAsync(ESMapper.Map<TDocument, ESContactSearchDocument>(documentsBatch), indexName, typeName);
The ESMapper is only used to convert from one type to another.
Resultant mapping:
{
"contact-uklwcl072": {
"mappings": {
"contact": {
"properties": {
"description": {
"type": "string"
},
"domain": {
"type": "string"
},
"email": {
"type": "string"
},
"entityid": {
"type": "string"
},
"firstname": {
"type": "string"
},
"id": {
"type": "string"
},
"lastname": {
"type": "string"
},
"name": {
"type": "string"
}
}
}
}
}
}
I have also installed Phonetic Analysis Plugin
Take a look at the automapping documentation; you need to assign the name of the analyzer to the Analyzer property on the attribute
public class Document
{
[String(Analyzer="phonetic")]
public string Name {get; set;}
}
then call AutoMap() when mapping your document type e.g. when you create the index
client.CreateIndex("index-name", c => c
.Mappings(m => m
.Map<Document>(d => d
.AutoMap()
)
)
);
yields
{
"mappings": {
"document": {
"properties": {
"name": {
"type": "string",
"analyzer": "phonetic"
}
}
}
}
}

NEST / Elastic Search Auto-Completion 'Field [suggest] is not a completion suggest field'

In between projects, started to play around with Elastic Search and NEST in preparation for some auto-completion features coming up.
I followed examples here and in the various docs ... and it seems like what I have should work. There must be some subtle difference with what i am doing.
This is the object that represents my document.
[ElasticType(IdProperty="KeyInstn")]
public class Instn
{
[ElasticProperty(Name = "keyinstn", Index = FieldIndexOption.Analyzed, Type = FieldType.Integer)]
public int KeyInstn { get; set; }
public string Type { get; set; }
[ElasticProperty(Name = "name", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Name { get; set; }
[ElasticProperty(Name = "ticker", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Ticker { get; set; }
[ElasticProperty(Name = "street", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Street { get; set; }
[ElasticProperty(Name = "city", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string City { get; set; }
[ElasticProperty(Name = "state", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string State { get; set; }
[ElasticProperty(Name = "zip", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Zip { get; set; }
[ElasticProperty(Name = "country", Index = FieldIndexOption.Analyzed, Type = FieldType.String)]
public string Country { get; set; }
[ElasticProperty(Name = "suggest", Type = FieldType.Completion)]
public CompletionField Suggest { get; set; }
}
This is the code that I use to create my index:
var index = client.CreateIndex("snl", c => c
.NumberOfReplicas(1)
.NumberOfShards(5)
.Settings(s => s
.Add("merge.policy.merge_factor", "10")
.Add("search.slowlog.threshold.fetch.warn", "1s")
)
.AddMapping<Instn>(m => m.MapFromAttributes()
.Properties(props => props
.Completion(s => s
.Name(p => p.Suggest)
.IndexAnalyzer("simple")
.SearchAnalyzer("simple")
.MaxInputLength(20)
.Payloads()
.PreservePositionIncrements()
.PreserveSeparators()
)
)
));
I am populating my documents using IndexMany in batches of 1000 documents.
I then perform my query:
var suggestResponse = this._client.Suggest<Instn>(s => s
.Index("snl")
.Completion("suggest", c => c
.Text(this.AutoCompleteText.Text)
.OnField("suggest")));
The error I receive is:
Field [suggest] is not a completion suggest field
Using SENSE, I pulls the mappings and notice that "suggest" doesn't have a type. The examples I followed indicate that the type should be 'completion.:'
Elasticsearch - Autocomplete with NEST
{
"snl": {
"mappings": {
"instn": {
"properties": {
"city": {
"type": "string"
},
"country": {
"type": "string"
},
"keyinstn": {
"type": "long"
},
"name": {
"type": "string"
},
"state": {
"type": "string"
},
"suggest": {
"properties": {
"input": {
"type": "string"
},
"output": {
"type": "string"
},
"payload": {
"type": "long"
},
"weight": {
"type": "double"
}
}
},
"type": {
"type": "string"
},
"zip": {
"type": "string"
}
}
}
}
}
}
In place of the CompletionObject I created myself, I also tried the intrinsic object, "SuggestField."
If anyone could point me in the right direction of what I am doing wrong, I'd appreciate it. FWIW "normal" searches work fine and return appropriate results.
Thanks.
UPDATE 1/26/2016
So, managed to get an index created that looks "better" ... I was able to get around the new error "analyzer on completion field must be set when search_analyzer is set" by omitting the search analyzer during creation and now my index looks like this:
{
"snl": {
"aliases": {},
"mappings": {
"instn": {
"properties": {
"city": {
"type": "string"
},
"country": {
"type": "string"
},
"keyinstn": {
"type": "long"
},
"name": {
"type": "string"
},
"state": {
"type": "string"
},
"suggest": {
"properties": {
"completion": {
"properties": {
"field": {
"type": "string"
}
}
},
"input": {
"type": "string"
},
"output": {
"type": "string"
},
"payload": {
"type": "long"
},
"text": {
"type": "string"
},
"weight": {
"type": "double"
}
}
},
"type": {
"type": "string"
},
"zip": {
"type": "string"
}
}
}
},
"settings": {
"index": {
"creation_date": "1453835225862",
"number_of_shards": "5",
"number_of_replicas": "1",
"uuid": "90QKK1OyRMKtwcyjAwPElA",
"version": {
"created": "2010199"
}
}
},
"warmers": {}
}
}
However, no completion searches return any results. Is it because the search analyzer omission? If so, how do I get around the "analyzer on completion field must be set when search_analyzer is set" error?

Combining Attribute / Fluent Mapping for Copy_To with Nest

I want to use the copy_to functionality with Nest. I've read that I need to use the fluent mapping (Elasticsearch Nest and CopyTo).
Is it possible to use the attribute based mapping then fluent mapping on top of that to add copy_to? If so, are there any examples? I'm having difficulty finding the answer.
The field I want to copy to does not exist in my model class. I just want to search on it in elasticsearch.
[ElasticType(IdProperty = "CustomerId", Name = "customer_search")]
public class CustomerSearchResult : BindableBase
{
[ElasticProperty(Name = "customer_id", Index = FieldIndexOption.NotAnalyzed, Type = FieldType.String)]
public int CustomerId { get; set; }
[ElasticProperty(Name = "account_type", Index = FieldIndexOption.NotAnalyzed, Type = FieldType.String)]
public string AccountType { get; set; }
[ElasticProperty(Name = "short_name", Index = FieldIndexOption.NotAnalyzed, Type = FieldType.String)]
public string ShortName { get; set; }
[ElasticProperty(Name = "legacy_name", Index = FieldIndexOption.NotAnalyzed, Type = FieldType.String)]
public string LegacyName { get; set; }
[ElasticProperty(Name = "legacy_contact_name", Index = FieldIndexOption.NotAnalyzed, Type = FieldType.String)]
public string LegacyContactName { get; set; }
[ElasticProperty(Name = "city", Index = FieldIndexOption.NotAnalyzed, Type = FieldType.String)]
public string City { get; set; }
[ElasticProperty(Name = "state_abbreviation", Index = FieldIndexOption.NotAnalyzed, Type = FieldType.String)]
public string StateAbbreviation { get; set; }
[ElasticProperty(Name = "country", Index = FieldIndexOption.NotAnalyzed, Type = FieldType.String)]
public string Country { get; set; }
[ElasticProperty(Name = "postal_code", Index = FieldIndexOption.NotAnalyzed, Type = FieldType.String)]
public string PostalCode { get; set; }
}
In the above class I want to use ShortName, LegacyName, and LegacyContactName and copy_to a field called "search" which will be an analyzed field.
Something like the following should do it
void Main()
{
var settings = new ConnectionSettings(new Uri("http://localhost:9200"));
var connection = new InMemoryConnection(settings);
var client = new ElasticClient(connection: connection);
var indexResponse = client.CreateIndex("customer_searches", c => c
.AddMapping<CustomerSearchResult>(m => m
.MapFromAttributes()
.Properties(p => p
.String(s => s.Name("short_name").CopyTo("search").Index(FieldIndexOption.NotAnalyzed))
.String(s => s.Name("legacy_name").CopyTo("search").Index(FieldIndexOption.NotAnalyzed))
.String(s => s.Name("legacy_contact_name").CopyTo("search").Index(FieldIndexOption.NotAnalyzed))
.String(s => s.Name("search").Index(FieldIndexOption.Analyzed))
)
)
);
Console.WriteLine(Encoding.UTF8.GetString(indexResponse.RequestInformation.Request));
}
Which outputs
{
"settings": {
"index": {}
},
"mappings": {
"customer_search": {
"properties": {
"customer_id": {
"index": "not_analyzed",
"type": "string"
},
"account_type": {
"index": "not_analyzed",
"type": "string"
},
"short_name": {
"index": "not_analyzed",
"copy_to": [
"search"
],
"type": "string"
},
"legacy_name": {
"index": "not_analyzed",
"copy_to": [
"search"
],
"type": "string"
},
"legacy_contact_name": {
"index": "not_analyzed",
"copy_to": [
"search"
],
"type": "string"
},
"city": {
"index": "not_analyzed",
"type": "string"
},
"state_abbreviation": {
"index": "not_analyzed",
"type": "string"
},
"country": {
"index": "not_analyzed",
"type": "string"
},
"postal_code": {
"index": "not_analyzed",
"type": "string"
},
"search": {
"index": "analyzed",
"type": "string"
}
}
}
}
}
The call to Properties() overrides the default conventions and attribute mappings so you need to specify that the fields are not_analyzed there as well.

Resources