creating elastic search query - elasticsearch

I have this json doc in my elasticsearch:
{
"personId": "5b564b6a0c000b622a55",
"name": "Jake Harper",
"country": "US",
"socialSecurityNumber": 7634904,
"personAddress": {
"city": "Los Angeles",
"street": "Sunset BLVD",
"streetNumber": 149,
},
"additionalAddresses": [
{
"addressType": "office",
"additionalAddress": {
"city": "Santa Monica",
"street": "3rd street",
"streetNumber": 13
}
},
{
"addressType": "property",
"additionalAddress": {
"city": "mxkwUcc branch city",
"street": "mxkwUcc BLVD",
"streetNumber": 255
}
}
]
}
and I want to create an elastic query that will help me to find people by:
personId
socialSecurityNumber
personAddress(all fields)
additionalAddresses(all fields in th array docs)
and im having trouble with creating the query specially with personAddress and additionalAddresses...
can anyone give me some kind of direction here..? thanks!
currently my query looks like :
{
"query": {
"bool": {
"should": [
{
"match": {
"personId": "5b564b6a0c000b622a"
}
},
{
"match": {
"name": "Harper"
}
}
]
}
}
}
im using multiple query cause I will get a term input and I want to check if its part of any of the above fields.
my mappings:
{
"peopledb": {
"mappings": {
"person": {
"properties": {
"additionalAddresses": {
"properties": {
"additionalAddress": {
"properties": {
"city": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"street": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"streetNumber": {
"type": "long"
},
"zipCode": {
"type": "long"
}
}
},
"addressType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"country": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"personAddress": {
"properties": {
"city": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"street": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"streetNumber": {
"type": "long"
},
"zipCode": {
"type": "long"
}
}
},
"personId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}

You might need to make additionalAddresses of type nested, but first let's see if multi_match gets you a bit further:
{
"query": {
"bool": {
"should": [
{
"match": {
"personId": "5b564b6a0c000b622a"
}
},
{
"match": {
"name": "Harper"
}
},
{
"match": {
"personAddress.city": "Los"
}
},
{
"multi_match": {
"fields": ["additionalAddresses.additionalAddress.city", "additionalAddresses.additionalAddress.street", "additionalAddresses.additionalAddress.streetNumber"],
"query": "123 Main Street"
}
}
]
}
}
}

Related

Elastic Search fetch records if matched the multiple fields with different values in an array

I have below documents in ElasticSearch
[
{
"class": " Grade 1",
"subject": [
"Mathematics",
"German",
"Geometry",
"Arts",
"Physical Education"
],
"student": [
{
"name": "George",
"id": "ORT-823FT"
},
{
"name": "Travis",
"id": "ORT-873FT"
},
{
"name": "Scott",
"id": "ORT-883FT"
}
]
},
{
"class": " Grade 2",
"subject": [
"Mathematics",
"German",
"Geometry",
"French",
"Arts",
"Physical Education"
],
"student": [
{
"name": "Gibbs",
"id": "ORT-923DG"
},
{
"name": "Elizabeth",
"id": "ORT-973DG"
},
{
"name": "Michale",
"id": "ORT-983DG"
}
]
}
]
I need to fetch the document only when the student name and id are matching, for eg: if the student name is George and the id is ORT-823FT, then the first document should be returned. On the other hand if the student name is Gibbs and the id is ORT-923DG then second document must be returned.
The below query works for me but is there better way to write ?
{
"query": {
"bool": {
"should": [
{
"match": {
"student.id": "ORT-823FT"
}
},
{
"match": {
"student.name": "George"
}
}
]
, "minimum_should_match": 2
}
}
}
Updated
The mapping for student is as below, I have added "type": "nested", as explained in the document.
{
"student": {
"type": "nested",
"properties": {
"studentResidence": {
"properties": {
"residenceAddress": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"phoneNumber": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"parentEmail": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"studentParentRelationShip": {
"properties": {
"relationshipType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"residenceAddress": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"comments": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"validFor": {
"properties": {
"endDateTime": {
"type": "date"
},
"startDateTime": {
"type": "date"
}
}
}
}
}
}
The corresponding query for the same is:
{
"query": {
"nested": {
"path": "student",
"query": {
"bool": {
"must": [
{
"match": {
"student.id": "ORT-823FT"
}},{
"match": {
"student.name": "George"
}
}
]
}
}
}
}
}
I am still getting an incorrect output. Not sure where I am going wrong.
Based on your sample data, Below mapping should work, note I created nested only for student property and rest properties are normal.
{
"mappings": {
"properties": {
"class": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"student": {
"type": "nested", --> note this
"properties": {
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"subject": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
After that you index the sample documents that you provided in your question+ below one more sample to test it properly.
{
"class": " Grade 2",
"subject": [
"Mathematics",
"German",
"Geometry",
"French",
"Arts",
"Physical Education"
],
"student": [
{
"name": "Gibbs",
"id": "ORT-abc"
},
{
"name": "Elizabeth",
"id": "ORT-973DG"
},
{
"name": "Michale",
"id": "ORT-983DG"
},
{
"name": "XYZ",
"id": "ORT-923DG"
}
]
}
Now same search query that your provided will return the proper results.
{
"query": {
"nested": {
"path": "student",
"query": {
"bool": {
"must": [
{
"match": {
"student.id": "ORT-823FT"
}
},
{
"match": {
"student.name": "George"
}
}
]
}
}
}
}
}
SR
"hits": [
{
"_index": "nested_mapping_student",
"_id": "1",
"_score": 4.0313807,
"_source": {
"class": " Grade 1",
"subject": [
"Mathematics",
"German",
"Geometry",
"Arts",
"Physical Education"
],
"student": [
{
"name": "George",
"id": "ORT-823FT"
},
{
"name": "Travis",
"id": "ORT-873FT"
},
{
"name": "Scott",
"id": "ORT-883FT"
}
]
}
}
]
what you did is not correct as you are not using the nested field to store your student information, hence relationship between id and name is lost, this is very well explained in this official example.
You can also try it by indexing below document
{
"class": " Grade 2",
"subject": [
"Mathematics",
"German",
"Geometry",
"French",
"Arts",
"Physical Education"
],
"student": [
{
"name": "Gibbs",
"id": "ORT-abc"
},
{
"name": "Elizabeth",
"id": "ORT-973DG"
},
{
"name": "Michale",
"id": "ORT-983DG"
},
{
"name": "XYZ",
"id": "ORT-923DG"
}
]
}
And see this is also coming in the query of Gibbs and id ORT-923DG which shouldn't be the case.
You need to use the nested query to get your expected results in all cases.

Unable to run elastic search nested aggregate query

I am trying to create a query that aggregates the sum of 3 different field and also matches three different conditions. I don't understand what the error message is saying.
The query below gives this specific error message:
{
"error": {
"root_cause": [
{
"type": "parsing_exception",
"reason": "Unknown key for a VALUE_NUMBER in [Type].",
"line": 1,
"col": 9
}
],
"type": "parsing_exception",
"reason": "Unknown key for a VALUE_NUMBER in [Type].",
"line": 1,
"col": 9
}
}
My query looks as follow:
{
"aggs": {
"filtered": {
"filter": {
"bool": {
"must": [
{
"term": {
"data.entity.productId": "45c29143b3bb4073a9fd325106784ce2"
}
},
{
"term": {
"data.entity.locationId": "c5f45ffc4fd94dcb926f96f1d5b9d835"
}
},
{
"term": {
"type.keyword": "StockLocationActivityAggregate"
}
}
]
}
}
},
"aggs": {
"directStock": {
"sum": { "field": "data.entity.inStock" },
"aggs": {
"directOutgoing": {
"sum": { "field": "data.entity.outgoing" },
"aggs": {
"directIncoming": { "sum": { "field": "data.entity.incoming" } }
}
}
}
}
}
},
"size": 0
}
Update
I am using the following index map
{
"mapping": {
"_doc": {
"properties": {
"active": {
"type": "boolean"
},
"data": {
"properties": {
"entity": {
"properties": {
"activityDate": {
"type": "date"
},
"creationDate": {
"type": "date"
},
"deleted": {
"type": "boolean"
},
"hash": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"inStock": {
"type": "float"
},
"incoming": {
"type": "float"
},
"locationId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"modifiedOn": {
"type": "date"
},
"modifier": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"orderId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"orderItemId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"outgoing": {
"type": "float"
},
"productId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reservationDate": {
"type": "date"
},
"version": {
"type": "long"
}
}
},
"hash": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"modifiedOn": {
"type": "date"
},
"modifier": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tenantIdentifier": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "long"
}
}
},
"deleted": {
"type": "boolean"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tenantId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "long"
}
}
}
}
}
I've also tried the example from the elastic search docs and the sample from Val below. They all give the same rror.
The sum aggregation is a metric aggregation that cannot have sub-aggregations... So you cannot do sum -> sum -> sum.
If you need the 3 different sums, you can do something like this:
{
...
"aggs": {
"directIncoming": {
"sum": {
"field": "data.entity.incoming"
}
},
"directStock": {
"sum": {
"field": "data.entity.inStock"
}
},
"directOutgoing": {
"sum": {
"field": "data.entity.outgoing"
}
}
}
}

Integer.text is no longer working in Kibana -7.9.2 for Query string query when the field is not an array

I have below document been added in ElasticSearch 7.9.2 version.
{
"EmployeeID": 222,
"EmpName": "Lisa jay",
"Age": 33,
"Address": [{
"AddressNo": 1290,
"Street": "Park flower road",
"ZIPCODE": 700
}]
}
I tried without adding .txt but it throws query_shard_exception, however after adding it, it doesn't show the correct result.
GET school/_search
{
"query": {
"query_string": {
"fields": ["EmployeeID.text","Address.ZIPCODE.text"],
"query": "222 700",
"default_operator":"OR"
}
}
}
Please see the image .
For array elements .text is working fine . Any answer or comment is appreciated and won't be down voted by me . Thank you .
Output of GET school/
{
"school": {
"aliases": {},
"mappings": {
"properties": {
"Address": {
"properties": {
"AddressNo": {
"type": "long"
},
"Street": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ZIPCODE": {
"type": "long"
}
}
},
"Age": {
"type": "long"
},
"EmpName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"EmployeeID": {
"type": "long"
},
"query": {
"properties": {
"match": {
"properties": {
"phrase": {
"properties": {
"ZIPCODE": {
"type": "long"
}
}
}
}
}
}
},
"t1": {
"properties": {
"properties": {
"properties": {
"address": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"age": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"id": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"name": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"partime": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
}
}
},
"settings": {
"index": {
"creation_date": "1602337657007",
"number_of_shards": "1",
"number_of_replicas": "1",
"uuid": "9RNUa1gOQ7WrpNoK_KifLA",
"version": {
"created": "7090299"
},
"provided_name": "school"
}
}
}
}

my phrase_prefix query does not work for numeric values

my query is pretty simple, it looks like this:
{
"query": {
"bool": {
"should": [
{
"multi_match": {
"query": "something_to_search",
"type": "phrase_prefix",
"fields": [
"name",
"id"
...
],
"lenient": true
}
}
],
"minimum_should_match": 1,
"boost": 1.0
}
}
}
name is text value and id is numeric value, if I search for "Jo" I will get people who's names starts with "Jo", but if I search for "123" I wont get people who's id's starts with "123", but if I search for the exact id I will get a result.
can someone please tell me how can I get also prefix queries on numeric?
my mappings:
{
"people_db": {
"mappings": {
"person": {
"properties": {
"address": {
"properties": {
"city": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"street": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"streetNumber": {
"type": "long"
},
"zipCode": {
"type": "long"
}
}
},
"country": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "long"
}
}
}
}
}
}

Why is my term query not working?

Here is my settings file -
{
"settings": {
"mappings": {
"default": {
"properties": {
"AIRPORT_CODE": {
"type": "text"
},
"PROVINCE_NAME": {
"type": "text"
},
"AIRPORT_NAME": {
"type": "text"
},
"CITY_NAME": {
"type": "text"
},
"TYPE": {
"type": "keyword"
},
"COUNTRY_NAME": {
"type": "text"
}
}
}
}
}
}
Here are 2 documents that I ingested into the index -
Document - 1
{
"AIRPORT_CODE": "SQA",
"PROVINCE_NAME": "California",
"AIRPORT_NAME": "Santa Ynez Airport",
"CITY_NAME": "SANTA YNEZ",
"TYPE": "AIRPORT"
}
Document - 2
{
"PROVINCE_NAME": "SANTIAGO",
"CITY_NAME": "SANTIAGO",
"COUNTRY_NAME": "DOMINICAN REPUBLIC",
"TYPE": "HOTEL"
}
Here is my search query -
{
"size": 4,
"timeout": "2m",
"query": {
"bool": {
"must": [
{
"multi_match": {
"query": "SQA"
}
}
],
"filter": [
{
"term": {
"TYPE": "AIRPORT"
}
}
]
}
},
"explain": false
}
The "TYPE" field has been defined as keyword. But the above query doesn't return anything. If change the "TYPE" value from "AIRPORT" to "airport" (lowercase), I get the results back. What am I doing wrong?
Note # My effort is to get this query working so I can take the advantage of filter cache (node query cache).
Thanks in advance!
Update - 1: adding out of GET index/_mapping
{
"rc-filter-cache": {
"mappings": {
"default": {
"properties": {
"AIRPORT_CODE": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"AIRPORT_NAME": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"CITY_NAME": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"COUNTRY_NAME": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"PROVINCE_NAME": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"TYPE": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"settings": {
"properties": {
"index": {
"properties": {
"analysis": {
"properties": {
"analyzer": {
"properties": {
"keylower": {
"properties": {
"tokenizer": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
}
}
},
"mappings": {
"properties": {
"default": {
"properties": {
"properties": {
"properties": {
"AIRPORT_CODE": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"AIRPORT_NAME": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"CITY_NAME": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"COUNTRY_NAME": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"PROVINCE_NAME": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"TYPE": {
"properties": {
"analyzer": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
Here is a way to solve that:
DELETE index
PUT index
{
"mappings": {
"default": {
"properties": {
"AIRPORT_CODE": {
"type": "text"
},
"PROVINCE_NAME": {
"type": "text"
},
"AIRPORT_NAME": {
"type": "text"
},
"CITY_NAME": {
"type": "text"
},
"TYPE": {
"type": "keyword"
},
"COUNTRY_NAME": {
"type": "text"
}
}
}
}
}
PUT index/default/1
{
"AIRPORT_CODE": "SQA",
"PROVINCE_NAME": "California",
"AIRPORT_NAME": "Santa Ynez Airport",
"CITY_NAME": "SANTA YNEZ",
"TYPE": "AIRPORT"
}
PUT index/default/2
{
"PROVINCE_NAME": "SANTIAGO",
"CITY_NAME": "SANTIAGO",
"COUNTRY_NAME": "DOMINICAN REPUBLIC",
"TYPE": "HOTEL"
}
GET index/_search
{
"query": {
"bool": {
"filter": [
{
"term": {
"TYPE": "AIRPORT"
}
}
]
}
}
}

Resources