Elasticsearch + Kibana + Alerting (X-Pack) For Energy Monitoring System - elasticsearch

Can somebody help me with Alerting Via X-Pack for Energy monitoring system project? The main problem here is I can't collect the 'Value' data from the database, as I want to compare it later with the upper and the lower threshold.
So here is the index:
PUT /test-1
{
"mappings": {
"Test1": {
"properties": {
"Value": {
"type": "integer"
},
"date": {
"type": "date",
"format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ"
},
"UpperThreshold": {
"type": "integer"
},
"LowerThreshold": {
"type": "integer"
}
}
}
}
}
Here is the example of the input:
POST /test-1/Test1
{
"Value": "500",
"date": "2017-06-13T16:20:00.000Z",
"UpperThreshold":"450",
"LowerThreshold": "380"
}
This is my alerting code
{
"trigger": {
"schedule": {
"interval": "10s"
}
},
"input": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
"logs"
],
"types": [],
"body": {
"query": {
"match": {
"message": "error"
}
}
}
}
}
},
"condition": {
"compare": {
"ctx.payload.hits.total": {
"gt": 0
}
}
},
"actions": {
"send_email": {
"email": {
"profile": "standard",
"to": [
"<account#gmail.com>"
],
"subject": "Watcher Notification",
"body": {
"text": "{{ctx.payload.hits.total}} error logs found"
}
}
}
}
}
Here is the response I got from the alerting plugin
{
"watch_id": "Alerting-Test",
"state": "execution_not_needed",
"_status": {
"state": {
"active": true,
"timestamp": "2017-07-26T15:27:35.497Z"
},
"last_checked": "2017-07-26T15:27:38.625Z",
"actions": {
"logging": {
"ack": {
"timestamp": "2017-07-26T15:27:35.497Z",
"state": "awaits_successful_execution"
}
}
}
},
"trigger_event": {
"type": "schedule",
"triggered_time": "2017-07-26T15:27:38.625Z",
"schedule": {
"scheduled_time": "2017-07-26T15:27:38.175Z"
}
},
"input": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
"test-1"
],
"types": [
"Test1"
],
"body": {
"query": {
"match_all": {}
}
}
}
}
},
"condition": {
"compare": {
"ctx.payload.hits.hits.0.Value": {
"gt": 450
}
}
},
"metadata": {
"name": "Alerting-Test"
},
"result": {
"execution_time": "2017-07-26T15:27:38.625Z",
"execution_duration": 0,
"input": {
"type": "search",
"status": "success",
"payload": {
"_shards": {
"total": 5,
"failed": 0,
"successful": 5
},
"hits": {
"hits": [
{
"_index": "test-1",
"_type": "Test1",
"_source": {
"date": "2017-07-22T12:00:00.000Z",
"LowerThreshold": "380",
"Value": "350",
"UpperThreshold": "450"
},
"_id": "AV1-1P3lArbJ1tbnct4e",
"_score": 1
},
{
"_index": "test-1",
"_type": "Test1",
"_source": {
"date": "2017-07-22T18:00:00.000Z",
"LowerThreshold": "380",
"Value": "4100",
"UpperThreshold": "450"
},
"_id": "AV1-1Sq0ArbJ1tbnct4v",
"_score": 1
},
{
"_index": "test-1",
"_type": "Test1",
"_source": {
"date": "2017-07-24T18:00:00.000Z",
"LowerThreshold": "380",
"Value": "450",
"UpperThreshold": "450"
},
"_id": "AV1-1eLJArbJ1tbnct6G",
"_score": 1
},
{
"_index": "test-1",
"_type": "Test1",
"_source": {
"date": "2017-07-23T00:00:00.000Z",
"LowerThreshold": "380",
"Value": "400",
"UpperThreshold": "450"
},
"_id": "AV1-1VUzArbJ1tbnct5A",
"_score": 1
},
{
"_index": "test-1",
"_type": "Test1",
"_source": {
"date": "2017-07-23T12:00:00.000Z",
"LowerThreshold": "380",
"Value": "390",
"UpperThreshold": "450"
},
"_id": "AV1-1X4FArbJ1tbnct5R",
"_score": 1
},
{
"_index": "test-1",
"_type": "Test1",
"_source": {
"date": "2017-07-23T18:00:00.000Z",
"LowerThreshold": "380",
"Value": "390",
"UpperThreshold": "450"
},
"_id": "AV1-1YySArbJ1tbnct5T",
"_score": 1
},
{
"_index": "test-1",
"_type": "Test1",
"_source": {
"date": "2017-07-26T00:00:00.000Z",
"LowerThreshold": "380",
"Value": "4700",
"UpperThreshold": "450"
},
"_id": "AV1-1mflArbJ1tbnct67",
"_score": 1
},
{
"_index": "test-1",
"_type": "Test1",
"_source": {
"date": "2017-07-26T06:00:00.000Z",
"LowerThreshold": "380",
"Value": "390",
"UpperThreshold": "450"
},
"_id": "AV1-1oluArbJ1tbnct7M",
"_score": 1
},
{
"_index": "test-1",
"_type": "Test1",
"_source": {
"date": "2017-07-21T12:00:00.000Z",
"LowerThreshold": "380",
"Value": "400",
"UpperThreshold": "450"
},
"_id": "AV1-1IrZArbJ1tbnct3r",
"_score": 1
},
{
"_index": "test-1",
"_type": "Test1",
"_source": {
"date": "2017-07-21T18:00:00.000Z",
"LowerThreshold": "380",
"Value": "440",
"UpperThreshold": "450"
},
"_id": "AV1-1LwzArbJ1tbnct38",
"_score": 1
}
],
"total": 20,
"max_score": 1
},
"took": 1,
"timed_out": false
},
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
"test-1"
],
"types": [
"Test1"
],
"body": {
"query": {
"match_all": {}
}
}
}
}
},
"condition": {
"type": "compare",
"status": "success",
"met": false,
"compare": {
"resolved_values": {
**"ctx.payload.hits.hits.0.Value": null**
}
}
},
"actions": []
},
"messages": []
}
Really appreciate for your help!!

Related

How can I prioritize documents in Elasticsearch query

I have products in my index. Documents are basically structured like these:
{
"_id": "product",
"_source": {
...
"type": "product",
"id": 1,
"mainTaxon": {
"name": "T-SHIRT",
},
"attributes": [
{
"code": "name",
"name": "Name",
"value": [
"BANANA T-SHIRT"
],
"score": 50
},
]
}
},
{
"_id": "product",
"_source": {
...
"type": "product",
"id": 2,
"mainTaxon": {
"name": "JEANS",
},
"attributes": [
{
"code": "name",
"name": "Name",
"value": [
"BANANA JEANS"
],
"score": 50
},
]
}
}
}
When I search for 'BANANA' I would prioritize products with mainTaxon different from JEANS. So, every product with the mainTaxon name T_SHIRT or something else would be listed before products with mainTaxon JEANS.
You can use boosting query to prioritize documents
{
"query": {
"boosting": {
"positive": {
"match": {
"attributes.value": "banana"
}
},
"negative": {
"match": {
"mainTaxon.name": "JEANS"
}
},
"negative_boost": 0.5
}
}
}
Search Result will be
"hits": [
{
"_index": "67164768",
"_type": "_doc",
"_id": "1",
"_score": 0.5364054,
"_source": {
"type": "product",
"id": 1,
"mainTaxon": {
"name": "T-SHIRT"
},
"attributes": [
{
"code": "name",
"name": "Name",
"value": [
"BANANA T-SHIRT"
],
"score": 50
}
]
}
},
{
"_index": "67164768",
"_type": "_doc",
"_id": "2",
"_score": 0.32743764,
"_source": {
"type": "product",
"id": 2,
"mainTaxon": {
"name": "JEANS"
},
"attributes": [
{
"code": "name",
"name": "Name",
"value": [
"BANANA JEANS"
],
"score": 50
}
]
}
}
]

ElasticSearch 2.4 - inner_hits nested merge queries result

I'm using ElasticSearch 2.4
I need to get all Purchases that match all queries.
I'm actually using inner_hits function but it doesn´t works as expected because it only shows the match of the current nested query and the problem is the combination with main document query.
I have this mapping and bellow I created an example with my comments:
PUT /example_contact_purchases
{
"mappings": {
"contact": {
"dynamic": false,
"properties": {
"name": {
"type": "string"
},
"country": {
"type": "string"
},
"purchases": {
"type": "nested",
"properties": {
"uuid":{
"type":"string"
},
"brand":{
"type":"string"
}
}
}
}
}
}
}
POST example_contact_purchases/contact
{
"name" : "Fran",
"country": "ES",
"purchases" : [
{
"uuid" : "23",
"brand":"Sony"
},
{
"uuid":"23",
"brand":"Sony"
}
]
}
POST example_contact_purchases/contact
{
"name" : "Jhon",
"country": "UK",
"purchases" : [
{
"uuid" : "45",
"brand": "Lenovo"
},
{
"uuid":"23",
"brand":"Sony"
},
{
"uuid":"77",
"brand":"HP"
}
]
}
POST example_contact_purchases/contact
{
"name" : "Lucas",
"country": "ES",
"purchases" : [
{
"uuid" : "45",
"brand": "Lenovo"
},
{
"uuid":"23",
"brand":"Sony"
},
{
"uuid":"77",
"brand":"HP"
}
]
}
GET example_contact_purchases/contact/_search
{
"query": {
"bool": {
"should": [
{"bool": {
"must": [
{
"query_string": {
"query": "country:ES"
}
},
{
"nested": {
"path": "purchases",
"inner_hits":{
"name":"0"
},
"filter": {
"query": {
"query_string": {
"query": "(purchases.brand:Sony)"
}
}
}
}
}
]
}},
{"bool": {
"must": [
{
"query_string": {
"query": "country:UK"
}
},
{
"nested": {
"path": "purchases",
"inner_hits":{
"name":"1"
},
"filter": {
"query": {
"query_string": {
"query": "(purchases.uuid:45)"
}
}
}
}
}
]
}
}
]
}
}
}
I am using simple query like this:
"(country.raw:ES AND purchases.brand:Sony) OR (country:UK AND purchases.uuid:45)"
And the result of the search query is:
{
"took": 10,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 3,
"max_score": 0.5949223,
"hits": [
{
"_index": "example_contact_purchases",
"_type": "contact",
"_id": "AXFfJJdZXthyTIlmcERM",
"_score": 0.5949223,
"_source": {
"name": "Jhon",
"country": "UK",
"purchases": [
{
"uuid": "45",
"brand": "Lenovo"
},
{
"uuid": "23",
"brand": "Sony"
},
{
"uuid": "77",
"brand": "HP"
}
]
},
"inner_hits": {
"0": {
"hits": {
"total": 1,
"max_score": 1,
"hits": [
{
"_index": "example_contact_purchases",
"_type": "contact",
"_id": "AXFfJJdZXthyTIlmcERM",
"_nested": {
"field": "purchases",
"offset": 1
},
"_score": 1,
"_source": {
"uuid": "23",
"brand": "Sony"
}
}
]
}
},
"1": {
"hits": {
"total": 1,
"max_score": 1,
"hits": [
{
"_index": "example_contact_purchases",
"_type": "contact",
"_id": "AXFfJJdZXthyTIlmcERM",
"_nested": {
"field": "purchases",
"offset": 0
},
"_score": 1,
"_source": {
"uuid": "45",
"brand": "Lenovo"
}
}
]
}
}
}
},
{
"_index": "example_contact_purchases",
"_type": "contact",
"_id": "AXFfJKBHXthyTIlmcERN",
"_score": 0.5949223,
"_source": {
"name": "Lucas",
"country": "ES",
"purchases": [
{
"uuid": "45",
"brand": "Lenovo"
},
{
"uuid": "23",
"brand": "Sony"
},
{
"uuid": "77",
"brand": "HP"
}
]
},
"inner_hits": {
"0": {
"hits": {
"total": 1,
"max_score": 1,
"hits": [
{
"_index": "example_contact_purchases",
"_type": "contact",
"_id": "AXFfJKBHXthyTIlmcERN",
"_nested": {
"field": "purchases",
"offset": 1
},
"_score": 1,
"_source": {
"uuid": "23",
"brand": "Sony"
}
}
]
}
},
"1": {
"hits": {
"total": 1,
"max_score": 1,
"hits": [
{
"_index": "example_contact_purchases",
"_type": "contact",
"_id": "AXFfJKBHXthyTIlmcERN",
"_nested": {
"field": "purchases",
"offset": 0
},
"_score": 1,
"_source": {
"uuid": "45",
"brand": "Lenovo"
}
}
]
}
}
}
},
{
"_index": "example_contact_purchases",
"_type": "contact",
"_id": "AXFfJI1SXthyTIlmcERL",
"_score": 0.5139209,
"_source": {
"name": "Fran",
"country": "ES",
"purchases": [
{
"uuid": "23",
"brand": "Sony"
},
{
"uuid": "23",
"brand": "Sony"
}
]
},
"inner_hits": {
"0": {
"hits": {
"total": 2,
"max_score": 1,
"hits": [
{
"_index": "example_contact_purchases",
"_type": "contact",
"_id": "AXFfJI1SXthyTIlmcERL",
"_nested": {
"field": "purchases",
"offset": 1
},
"_score": 1,
"_source": {
"uuid": "23",
"brand": "Sony"
}
},
{
"_index": "example_contact_purchases",
"_type": "contact",
"_id": "AXFfJI1SXthyTIlmcERL",
"_nested": {
"field": "purchases",
"offset": 0
},
"_score": 1,
"_source": {
"uuid": "23",
"brand": "Sony"
}
}
]
}
},
"1": {
"hits": {
"total": 0,
"max_score": null,
"hits": []
}
}
}
}
]
}
}
Unfortunatly the first result is wrong:
"inner_hits": {
"0": {
"hits": {
"total": 1,
"max_score": 1,
"hits": [
{
"_index": "example_contact_purchases",
"_type": "contact",
"_id": "AXFfJJdZXthyTIlmcERM",
"_nested": {
"field": "purchases",
"offset": 1
},
"_score": 1,
"_source": {
"uuid": "23",
"brand": "Sony"
}
}
]
}
},
"1": {
"hits": {
"total": 1,
"max_score": 1,
"hits": [
{
"_index": "example_contact_purchases",
"_type": "contact",
"_id": "AXFfJJdZXthyTIlmcERM",
"_nested": {
"field": "purchases",
"offset": 0
},
"_score": 1,
"_source": {
"uuid": "45",
"brand": "Lenovo"
}
}
]
}
}
}
It should show the purchase for Jhon UK with parameters:
{"uuid": "45","brand":"Lenovo"} ( inner_hits with name "1")
Thanks

ES how to refactor a querying with a join like?

I've created 2 queries in elastic search:
GET poc-2020.01.09/_search
{
"size": 1000,
"query": {
"wildcard": {
"message": {
"value": "got*", <-------------
"boost": 1.0,
"rewrite": "constant_score"
}
}
}
}
returns:
{
"_index": "poc-2020.01.09",
"_type": "doc",
"_id": "YicNiG8BsW6Znkt6BLuc",
"_score": 1,
"_source": {
"offset": 618993630,
"node.tag": "taskmanager",
"logfile.name": "app-taskmanager-1-94hhg.log",
"logtype": "app",
"beat": {
"version": "6.3.2",
"hostname": "infra",
"name": "infra"
},
"#version": "1",
"type": "beats",
"tags": [
"beats_input_codec_plain_applied"
],
"#timestamp": "2020-01-09T02:05:51.251Z",
"source": "/nfsdata/ecs/log/app-taskmanager-1-94hhg.log",
"message": """2020-01-09 02:04:39,825 INFO Utils - got OOO: XYZ:912828YZ7 Metrics:[] """,
"app.tag": "flink-app-treasury-enriched-position",
"env": "DEV",
"host": {
"name": "infra-elkagent-29-xhx2x"
}
}
},
(it's returns got OOO messages in this example: with XYZ:912828YZ7)
And querying
GET gsp.datasphere.flink.poc-2020.01.09/_search
{
"size": 10,
"query": {
"wildcard": {
"message": {
"value": "outputing*",
"boost": 1.0,
"rewrite": "constant_score"
}
}
}
}
returns:
"hits": {
"total": 104605,
"max_score": 1,
"hits": [
{
"_index": "poc-2020.01.09",
"_type": "doc",
"_id": "3wfYh28BsW6Znkt67Sho",
"_score": 1,
"_source": {
"offset": 617979882,
"node.tag": "taskmanager",
"logfile.name": "app-taskmanager-1-94hhg.log",
"logtype": "app",
"beat": {
"version": "6.3.2",
"hostname": "infra-elkagent-29-xhx2x",
"name": "infra-elkagent-29-xhx2x"
},
"#version": "1",
"type": "beats",
"tags": [
"beats_input_codec_plain_applied"
],
"#timestamp": "2020-01-09T01:08:56.220Z",
"source": "/nfsdata/ecs/log/app-taskmanager-1-94hhg.log",
"message": "2020-01-09 01:07:34,011 INFO Function - Outputing gotoSchool: XYZ:912828YZ7",
"app.tag": "app-trx",
"env": "DEV",
"host": {
"name": "infra"
}
}
},
(it's returns got gotoSchool messages in this example: with :XYZ:912828YZ7)
please note that for each gotoSchool message there are many OOO messages.
what i really need is a query that can join between them. (by the value - XYZ:912828YZ7 - the number after the "XYZ:" is the join value.
(so for each parent to return itself with it's childs).
can you assist?
thanks.

Elastic search nested array query

I have the following schema elastic search:
I have abbreviated most of it for ease of reading
{
"took": 56,
"timed_out": false,
"hits": {
"hits": [
{
"_id": "2FREAL%2FShaarHanegev%2Faxis-ACCC8E43E0C6%2F20160314%2F16%2F20160314_164253_E23D_ACCC8E43E0C6%2F20160314_16%2F20160314_164253_8DA7_ACCC8E43E0C6.mkv%3A26.153_1508_1439_1763_1440",
"_source": {
"path": "2FREAL%2FShaar%2Faxis-ACCC8E43E0C6%2F20160314%2F16%2F20160314_164253_E23D_ACCC8E43E0C6%2F20160314_16%2F20160314_164253_8DA7_ACCC8E43E0C6.mkv%3A26.153_1508_1439_1763_1440",
"frameAttributes": {
"trackerId": "OB7E600",
"identities": [
{
"_id": "Abel_Nachos_034841460",
"_score": 0.451906
},
{
"_id": "judas_acorn_10000897",
"_score": 0.430024
},
{
"_id": "regenald_barbish_10000452",
"_score": 0.41979
},
{
"_id": "Matthew_Gordon_Douglas_0631B#1",
"_score": 0.412086
},
{
"_id": "NewYork_NYC_Gina_Lob_10000566",
"_score": 0.407909
}
]
},
"originalVideoFullPath": "REAL/Shaar/axis-ACCC8E43E0C6/20160314/16/20160314_164253_E23D_ACCC8E43E0C6/20160314_16/20160314_164253_8DA7_ACCC8E43E0C6.mkv"
}
},
{
"_id": "2FREAL%2FShaar%2Faxis-ACCC8E43E0C6%2F20160314%2F16%2F20160314_164253_E23D_ACCC8E43E0C6%2F20160314_16%2F20160314_164253_8DA7_ACCC8E43E0C6.mkv%3A26.393_1454_1457_1711_1454",
"_source": {
"path": "2FREAL%2FShaarHanegev%2Faxis-ACCC8E43E0C6%2F20160314%2F16%2F20160314_164253_E23D_ACCC8E43E0C6%2F20160314_16%2F20160314_164253_8DA7_ACCC8E43E0C6.mkv%3A26.393_1454_1457_1711_1454",
"frameAttributes": {
"trackerId": "OB7E600",
"identities": [
{
"_id": "Levon_Ayrapetyan_10000036",
"_score": 0.432837
},
{
"_id": "Patrick_sole_10001145",
"_score": 0.425161
},
{
"_id": "TAI_YUNG_LEE_LOUIE_10002450",
"_score": 0.404628
},
{
"_id": "Izak_Gold_10000159",
"_score": 0.400651
},
{
"_id": "Takahiro_Friend_10001663",
"_score": 0.392012
}
]
},
"clockwiseRotation": 90,
"numberOfFrames": 0,
"originalVideoFullPath": "REAL/Shaar/axis-ACCC8E43E0C6/20160314/16/20160314_164253_E23D_ACCC8E43E0C6/20160314_16/20160314_164253_8DA7_ACCC8E43E0C6.mkv"
}
}
]
}
}
Edit: I added the abbreviated mapping as requested
{
"index1": {
"mappings": {
"objects": {
"properties": {
"_results": {
"type": "object"
},
"cameraId": {
"type": "string"
},
"cameraType": {
"type": "string"
},
"date": {
"type": "date",
"format": "dateOptionalTime"
},
"frameAttributes": {
"properties": {
"identities": {
"properties": {
"_id": {
"type": "string"
},
"_score": {
"type": "double"
}
}
},
"testPassed": {
"type": "boolean"
}
}
},
"fraudType": {
"type": "string"
},
"index": {
"type": "long"
},
"path": {
"type": "string"
},
"siteName": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
}
I am trying to write a query that will find all the videos (id) where the Person {NewYork_NYC_Gina_Lob_10000566} is located.
I don't see mapping, so this is a query ignoring possible nested objects or parent-child relationships.
GET your_index/_search
{
"query": {
"bool": {
"must": [
{
"term": {
"frameAttributes.identities._id": "NewYork_NYC_Gina_Lob_10000566"
}
}
]
}
}
}

How to aggregate on nested objects in elasticsearch

I have the following mapping in ES:
"mappings": {
"products": {
"properties": {
"product": {
"type" : "nested",
"properties": {
"features": {
"type": "nested"
},
"sitedetails": {
"type": "nested"
}
}
}
}
}
}
and then 3 products like this:
"hits": [
{
"_index": "catalog",
"_type": "products",
"_id": "AVNE8F4mFYOWvB4rMqdO",
"_score": 1,
"_source": {
"product": {
"ean": "abc",
"features": {
"productType": "DVD player"
},
"color": "Black",
"manufacturer": "Sony",
"sitedetails": [
{
"name": "amazon.com",
"sku": "zzz",
"url": "http://www.amazon.com/dp/zzz"
}
],
"category": "Portable DVD Players"
}
}
},
{
"_index": "catalog",
"_type": "products",
"_id": "AVNE8XkXFYOWvB4rMqdQ",
"_score": 1,
"_source": {
"product": {
"ean": "def",
"features": {
"ProductType": "MP3 player"
},
"color": "Black",
"manufacturer": "LG",
"sitedetails": [
{
"name": "amazon.com",
"sku": "aaa",
"url": "http://www.amazon.com/dp/aaa"
}
],
"category": "MP3 Players"
}
}
},
{
"_index": "catalog",
"_type": "products",
"_id": "AVNIh-xVWwxj6Cz_r8AT",
"_score": 1,
"_source": {
"product": {
"ean": "abc",
"features": {
"productType": "DVD player"
},
"color": "White",
"manufacturer": "Sony",
"sitedetails": [
{
"name": "amazon.com",
"sku": "ggg",
"url": "http://www.amazon.com/dp/ggg"
}
],
"category": "Portable DVD Players"
}
}
}
]
I need to display on the UI side 2 filters, one for Manufacturer and one for website.
How can I aggregate on product.manufacturer and product.sitedetails.name?
tnx!
Figured it out:
GET /catalog/products/_search
{
"aggs": {
"byManufacturer": {
"nested": {
"path": "product"
},
"aggs": {
"byManufacturer": {
"terms": {
"field": "product.manufacturer"
}
}
}
},
"bySeller": {
"nested": {
"path": "product.sitedetails"
},
"aggs": {
"bySeller": {
"terms": {
"field": "product.sitedetails.name"
}
}
}
}
}
}

Resources