How to set order in Elasticsearch sortion - elasticsearch

I want to apply order for sorting in Elasticsearch. Unfortunately, common standard sort can't sort in next way:
special chars
numbers
alphabetic(aA-zZ)
My mapping:
PUT /sorting_by_name
{
"mappings": {
"sorting_by_name": {
"properties": {
"name": {
"type": "keyword"
}
}
}
}
}
I am querying Elasticsearch as follows:
GET /sorting_by_name/_search
{
"query": {
"match_all": {}
},
"sort": [
{ "name": { "order": "asc" } }
]
}
And I would want to get the following result:
[
{
"_id": "AXe1Y7vv7OkJxgqzrpj9",
"_source": {
"name": "{{{{ A {{{ sdasd"
}
},
{
"_id": "AXe1Y7vv7OkJxgqzrpj9",
"_source": {
"name": "{{{{ A {{{ sdasd"
}
},
{
"_id": "AXe1QBZz7OkJxgqzrpj2",
"_source": {
"name": "11asdasd"
}
}
{
"_id": "AXe1Y6id7OkJxgqzrpj8",
"_source": {
"name": "A {{{ sdasd"
}
},
{
"_id": "AXe1QFwn7OkJxgqzrpj4",
"_source": {
"name": "asdasd"
}
},
{
"_id": "AXe1ZOGe7OkJxgqzrpkC",
"_source": {
"name": "Ooopopl"
}
},
{
"_id": "AXe1ZM4V7OkJxgqzrpkB",
"_source": {
"name": "ooopopl"
}
}
]
But now I get the following result:
[
{
"_id": "AXe1Y7vv7OkJxgqzrpj9",
"_source": {
"name": "{{{{ A {{{ sdasd"
}
},
{
"_id": "AXe1QBZz7OkJxgqzrpj2",
"_source": {
"name": "11asdasd"
}
},
{
"_id": "AXe1Y6id7OkJxgqzrpj8",
"_source": {
"name": "A {{{ sdasd"
}
},
{
"_id": "AXe1ZOGe7OkJxgqzrpkC",
"_source": {
"name": "Ooopopl"
}
},
{
"_id": "AXe1QFwn7OkJxgqzrpj4",
"_source": {
"name": "asdasd"
}
},
{
"_id": "AXe1ZM4V7OkJxgqzrpkB",
"_source": {
"name": "ooopopl"
}
},
{
"_id": "AXe1Y7vv7OkJxgqzrpj9",
"_source": {
"name": "{{{{ A {{{ sdasd"
}
}
]
How would I get the expected result?

Related

Elastic search Query - How to pass list of queries

I have created an Index with 10000+ documents. Here is the sample from that:
{
"_index": "index_1",
"_type": "_doc",
"_id": "48a454f9-71d2-41a0-9e62-08c149366f05",
"_score": 13.977877,
"_source": {
"customer_id":10,
"customer_name": Mike,
"customer_phone": 1111111111,
"customer_address": "XYZ"
}
},
{
"_index": "index_1",
"_type": "_doc",
"_id": "48a454f9-71d2-41a0-9e62-08c149366f71",
"_score": 12.977861,
"_source": {
"customer_id":20,
"customer_name": Angie,
"customer_phone": 2222222222,
"customer_address": "ABC"
}
},
{
"_index": "index_1",
"_type": "_doc",
"_id": "48a454f9-71d2-41a0-9e62-08c149366f62",
"_score": 10.978777,
"_source": {
"customer_id":30,
"customer_name": John,
"customer_phone": 3333333333,
"customer_address": "PQR"
}
},
{
"_index": "index_1",
"_type": "_doc",
"_id": "48a454f9-71d2-41a0-9e62-08c149366f54",
"_score": 11.817877,
"_source": {
"customer_id":40,
"customer_name": Andy,
"customer_phone": 4444444444,
"customer_address": "MNO"
}
},
{
"_index": "index_1",
"_type": "_doc",
"_id": "48a454f9-71d2-41a0-9e62-08c149366f32",
"_score": 14.457877,
"_source": {
"customer_id": 50,
"customer_name": Nick,
"customer_phone": 5555555555,
"customer_address": "CDE"
}
},
{
"_index": "index_1",
"_type": "_doc",
"_id": "48a454f9-71d2-41a0-9e62-08c149366f21",
"_score": 16.487877,
"_source": {
"customer_id":60,
"customer_name": Atlas,
"customer_phone": 6666666666,
"customer_address": "DFE"
}
}
I want to pass multiple queries at once as list in json body and get the result also in list format:
For example: -> I want to pass below 3 queries in the search condition at the same time:
1) customer_id = 10, customer_name = Mike, customer_phone = 1111111111
2) customer_id = 40, customer_name = Andy, customer_phone = 4444444444
3) customer_id = 50, customer_name = Nick, customer_phone = 5555555555
Although, I can combine these 3 queries using 'AND' and 'OR' like below:
{
"query": {
"query_string": {
"query": "(customer_id: 10 AND customer_name: Mike AND customer_phone: 1111111111) OR (customer_id: 40 AND customer_name: Andy AND customer_phone: 4444444444) OR (customer_id: 50 AND customer_name: Nick AND customer_phone: 5555555555)"
}
}
}
Other than combining the queries as above, is there any other better way to achieve the same (like passing the queries as list).
You can combinate should and must query.
{
"query": {
"bool": {
"should": [
{
"bool": {
"must": [
{
"term": {
"customer_id": {
"value": 10
}
}
},
{
"match": {
"custumer_name": "Mike"
}
},
{
"term": {
"customer_phone": {
"value": 1111111111
}
}
}
]
}
},
{
"bool": {
"must": [
{
"term": {
"customer_id": {
"value": 50
}
}
},
{
"match": {
"custumer_name": "Nick"
}
},
{
"term": {
"customer_phone": {
"value": 5555555555
}
}
}
]
}
},
{
"bool": {
"must": [
{
"term": {
"customer_id": {
"value": 40
}
}
},
{
"match": {
"custumer_name": "Andy"
}
},
{
"term": {
"customer_phone": {
"value": 4444444444
}
}
}
]
}
}
]
}
}
}

How to make flattened sub-field in the nested field in elastic search?

Here, I have a indexed document like:
doc = {
"id": 1,
"content": [
{
"txt": I,
"time": 0,
},
{
"txt": have,
"time": 1,
},
{
"txt": a book,
"time": 2,
},
{
"txt": do not match this block,
"time": 3,
},
]
}
And I want to match "I have a book", and return the matched time: 0,1,2. Is there anyone who knows how to build the index and the query for this situation?
I think the "content.txt" should be flattened but "content.time" should be nested?
want to match "I have a book", and return the matched time: 0,1,2.
Adding a working example with index mapping,search query, and search result
Index Mapping:
{
"mappings": {
"properties": {
"content": {
"type": "nested"
}
}
}
}
Search Query:
{
"query": {
"nested": {
"path": "content",
"query": {
"bool": {
"must": [
{
"match": {
"content.txt": "I have a book"
}
}
]
}
},
"inner_hits": {}
}
}
}
Search Result:
"inner_hits": {
"content": {
"hits": {
"total": {
"value": 3,
"relation": "eq"
},
"max_score": 2.5226097,
"hits": [
{
"_index": "64752029",
"_type": "_doc",
"_id": "1",
"_nested": {
"field": "content",
"offset": 2
},
"_score": 2.5226097,
"_source": {
"txt": "a book",
"time": 2
}
},
{
"_index": "64752029",
"_type": "_doc",
"_id": "1",
"_nested": {
"field": "content",
"offset": 0
},
"_score": 1.5580825,
"_source": {
"txt": "I",
"time": 0
}
},
{
"_index": "64752029",
"_type": "_doc",
"_id": "1",
"_nested": {
"field": "content",
"offset": 1
},
"_score": 1.5580825,
"_source": {
"txt": "have",
"time": 1
}
}
]
}
}
}
}

How to query IP range in Elastic search?

I want to query IP range from:172.16.0.0 to 172.31.0.0 in ELK
I try two query methods, but fail.
{
"query": {
"bool": {
"should": [
{
"regexp": {
"DstIP": "172.(3[0-1]|1[6-9]|2[0-9]).*"
}
}
],
"minimum_should_match": 1
}
}
}
{
"query": {
"range": {
"DstIP": {
"gte": "172.16.0.0",
"lte": "172.31.0.0"
}
}
}
}
How can query IP range in ELK?
For range queries to work correctly on IP values it is necessary to define the field data type as ip.
Below is the working example with mapping, sample docs, and search query.
Mapping:
{
"mappings": {
"properties": {
"dest": {
"type": "ip"
}
}
}
}
Index data:
Then I've taken a couple of sample documents like this:
{ "dest":"172.16.0.0"}
{ "dest":"172.31.0.0"}
{ "dest":"172.21.0.0"}
{ "dest":"172.1.0.0" }
{ "dest":"172.12.0.0"}
Search Query :
{
"query": {
"range": {
"dest": {
"gte": "172.16.0.0",
"lte": "172.31.0.0"
}
}
}
}
Search Result :
"hits": [
{
"_index": "foo4",
"_type": "_doc",
"_id": "1",
"_score": 1.0,
"_source": {
"dest": "172.16.0.0"
}
},
{
"_index": "foo4",
"_type": "_doc",
"_id": "2",
"_score": 1.0,
"_source": {
"dest": "172.31.0.0"
}
},
{
"_index": "foo4",
"_type": "_doc",
"_id": "3",
"_score": 1.0,
"_source": {
"dest": "172.21.0.0"
}
}
]

Elasticsearch search for Turkish characters

I have some documents that i am indexing with elasticsearch. But some of the documents are written with upper case and Tukish characters are changed. For example "kürşat" is written as "KURSAT".
I want to find this document by searching "kürşat". How can i do that?
Thanks
Take a look at the asciifolding token filter.
Here is a small example for you to try out in Sense:
Index:
DELETE test
PUT test
{
"settings": {
"analysis": {
"filter": {
"my_ascii_folding": {
"type": "asciifolding",
"preserve_original": true
}
},
"analyzer": {
"turkish_analyzer": {
"tokenizer": "standard",
"filter": [
"lowercase",
"my_ascii_folding"
]
}
}
}
},
"mappings": {
"test": {
"properties": {
"name": {
"type": "string",
"analyzer": "turkish_analyzer"
}
}
}
}
}
POST test/test/1
{
"name": "kürşat"
}
POST test/test/2
{
"name": "KURSAT"
}
Query:
GET test/_search
{
"query": {
"match": {
"name": "kursat"
}
}
}
Response:
"hits": {
"total": 2,
"max_score": 0.30685282,
"hits": [
{
"_index": "test",
"_type": "test",
"_id": "2",
"_score": 0.30685282,
"_source": {
"name": "KURSAT"
}
},
{
"_index": "test",
"_type": "test",
"_id": "1",
"_score": 0.30685282,
"_source": {
"name": "kürşat"
}
}
]
}
Query:
GET test/_search
{
"query": {
"match": {
"name": "kürşat"
}
}
}
Response:
"hits": {
"total": 2,
"max_score": 0.4339554,
"hits": [
{
"_index": "test",
"_type": "test",
"_id": "1",
"_score": 0.4339554,
"_source": {
"name": "kürşat"
}
},
{
"_index": "test",
"_type": "test",
"_id": "2",
"_score": 0.09001608,
"_source": {
"name": "KURSAT"
}
}
]
}
Now the 'preserve_original' flag will make sure that if a user types: 'kürşat', documents with that exact match will be ranked higher than documents that have 'kursat' (Notice the difference in scores for both query responses).
If you want the score to be equal, you can put the flag on false.
Hope I got your problem right!

Elastic search nested array query

I have the following schema elastic search:
I have abbreviated most of it for ease of reading
{
"took": 56,
"timed_out": false,
"hits": {
"hits": [
{
"_id": "2FREAL%2FShaarHanegev%2Faxis-ACCC8E43E0C6%2F20160314%2F16%2F20160314_164253_E23D_ACCC8E43E0C6%2F20160314_16%2F20160314_164253_8DA7_ACCC8E43E0C6.mkv%3A26.153_1508_1439_1763_1440",
"_source": {
"path": "2FREAL%2FShaar%2Faxis-ACCC8E43E0C6%2F20160314%2F16%2F20160314_164253_E23D_ACCC8E43E0C6%2F20160314_16%2F20160314_164253_8DA7_ACCC8E43E0C6.mkv%3A26.153_1508_1439_1763_1440",
"frameAttributes": {
"trackerId": "OB7E600",
"identities": [
{
"_id": "Abel_Nachos_034841460",
"_score": 0.451906
},
{
"_id": "judas_acorn_10000897",
"_score": 0.430024
},
{
"_id": "regenald_barbish_10000452",
"_score": 0.41979
},
{
"_id": "Matthew_Gordon_Douglas_0631B#1",
"_score": 0.412086
},
{
"_id": "NewYork_NYC_Gina_Lob_10000566",
"_score": 0.407909
}
]
},
"originalVideoFullPath": "REAL/Shaar/axis-ACCC8E43E0C6/20160314/16/20160314_164253_E23D_ACCC8E43E0C6/20160314_16/20160314_164253_8DA7_ACCC8E43E0C6.mkv"
}
},
{
"_id": "2FREAL%2FShaar%2Faxis-ACCC8E43E0C6%2F20160314%2F16%2F20160314_164253_E23D_ACCC8E43E0C6%2F20160314_16%2F20160314_164253_8DA7_ACCC8E43E0C6.mkv%3A26.393_1454_1457_1711_1454",
"_source": {
"path": "2FREAL%2FShaarHanegev%2Faxis-ACCC8E43E0C6%2F20160314%2F16%2F20160314_164253_E23D_ACCC8E43E0C6%2F20160314_16%2F20160314_164253_8DA7_ACCC8E43E0C6.mkv%3A26.393_1454_1457_1711_1454",
"frameAttributes": {
"trackerId": "OB7E600",
"identities": [
{
"_id": "Levon_Ayrapetyan_10000036",
"_score": 0.432837
},
{
"_id": "Patrick_sole_10001145",
"_score": 0.425161
},
{
"_id": "TAI_YUNG_LEE_LOUIE_10002450",
"_score": 0.404628
},
{
"_id": "Izak_Gold_10000159",
"_score": 0.400651
},
{
"_id": "Takahiro_Friend_10001663",
"_score": 0.392012
}
]
},
"clockwiseRotation": 90,
"numberOfFrames": 0,
"originalVideoFullPath": "REAL/Shaar/axis-ACCC8E43E0C6/20160314/16/20160314_164253_E23D_ACCC8E43E0C6/20160314_16/20160314_164253_8DA7_ACCC8E43E0C6.mkv"
}
}
]
}
}
Edit: I added the abbreviated mapping as requested
{
"index1": {
"mappings": {
"objects": {
"properties": {
"_results": {
"type": "object"
},
"cameraId": {
"type": "string"
},
"cameraType": {
"type": "string"
},
"date": {
"type": "date",
"format": "dateOptionalTime"
},
"frameAttributes": {
"properties": {
"identities": {
"properties": {
"_id": {
"type": "string"
},
"_score": {
"type": "double"
}
}
},
"testPassed": {
"type": "boolean"
}
}
},
"fraudType": {
"type": "string"
},
"index": {
"type": "long"
},
"path": {
"type": "string"
},
"siteName": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
}
I am trying to write a query that will find all the videos (id) where the Person {NewYork_NYC_Gina_Lob_10000566} is located.
I don't see mapping, so this is a query ignoring possible nested objects or parent-child relationships.
GET your_index/_search
{
"query": {
"bool": {
"must": [
{
"term": {
"frameAttributes.identities._id": "NewYork_NYC_Gina_Lob_10000566"
}
}
]
}
}
}

Resources