Elastic search (Kibana) - intersect between boolean results - elasticsearch

Question

I am facing a problem in Kibana how to correctly filter a data. Basically my aim is to filter PASSED or FAILED tests from the following data structure.
{
"_index":"qa-reporting-2020-04",
"_type":"qa-reporting",
"_id":"456.0",
"_version":1,
"_score":null,
"_source":{
"TestId":"CXXX01",
"TestStatus":0,
"Issues":[
],
"MetaData":{
"TestName":"Test1",
"LastException":null,
"DurationMs":1980.5899000000002,
"Platform":{
"BrowserName":"chrome",
"OS":"windows",
"OsVersion":"10"
},
"Categories":[
"Cat1",
"Cat2",
"CXXX01"
],
"Priority":null,
"TestStatusStr":"Passed",
"JobName":"My-Demo/dev/ServerJob1",
"Logs":"PASS - Passed - CXXX01",
"SuiteName":"Tests.ServerTests",
"LastMessage":"PASS - Passed - CXXX01: \n",
"Environment":"dev"
}
}
}
The problem is that during a time (day), the logs will have several entries (e.g. the test failed and later in the same day the test passed). I have an aggregation query that gives me both results which is not desired. I want to have a result returning the intersection so the report will contain either failed/passed tests.
Here is my query (I am begginer) which gives me aggregation for a specific test that failed and passed.
GET qa-reporting-*/_search
{"size": 0,
"query": {
"bool": {
"must": [
{
"match": {
"MetaData.Environment": "dev"
}
},
{
"match": {
"TestId": "CXXX01"
}
},
{
"range": {
"Created": {
"gte": "now-0d/d"
}
}
}
]
}
},
"aggs": {
"tests": {
"terms": {"field": "MetaData.TestStatusStr.keyword"}
}
}
}
It returns the following (excerpt from the entire object)
"aggregations": {
"tests": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "Passed",
"doc_count": 10
},
{
"key": "Failed",
"doc_count": 1
}
]
}
}
As above mentioned how to get the intersection between failed/passed tests during a specific date time? Is it possible to do it in Kibana or it needs to be solved on a script level outside of it?
Currently my report looks like this:
DateTime | TestId | Status
Apr 24,12:00 | CXXX01 | Failed
Apr 24,15:00 | CXXX01 | Passed
Wanted report will be only the intersection of the above status
Apr 24 | CXXX01 | Failed
or
Apr 24 | CXXX01 | Passed
So the latest result will have more weight I think.

Answer 1

I have taken data for two days. Two entries for each day. Last entry for day 1 is "Failed" and latest entry for day 2 is "Passed"
Data:
"hits" : [
{
"_index" : "index19",
"_type" : "_doc",
"_id" : "QKlosXEBuBOc-UQkKecO",
"_score" : 1.0,
"_source" : {
"TestId" : "CXXX01",
"TestStatus" : 0,
"Issues" : [ ],
"Created" : "2020-04-23T01:00:00",
"MetaData" : {
"TestName" : "Test1",
"LastException" : null,
"DurationMs" : 1980.5899000000002,
"Platform" : {
"BrowserName" : "chrome",
"OS" : "windows",
"OsVersion" : "10"
},
"Categories" : [
"Cat1",
"Cat2",
"CXXX01"
],
"Priority" : null,
"TestStatusStr" : "Passed",
"JobName" : "My-Demo/dev/ServerJob1",
"Logs" : "PASS - Passed - CXXX01",
"SuiteName" : "Tests.ServerTests",
"LastMessage" : "PASS - Passed - CXXX01: \n",
"Environment" : "dev"
}
}
},
{
"_index" : "index19",
"_type" : "_doc",
"_id" : "QalosXEBuBOc-UQkieeR",
"_score" : 1.0,
"_source" : {
"TestId" : "CXXX01",
"TestStatus" : 0,
"Issues" : [ ],
"Created" : "2020-04-23T10:00:00",
"MetaData" : {
"TestName" : "Test1",
"LastException" : null,
"DurationMs" : 1980.5899000000002,
"Platform" : {
"BrowserName" : "chrome",
"OS" : "windows",
"OsVersion" : "10"
},
"Categories" : [
"Cat1",
"Cat2",
"CXXX01"
],
"Priority" : null,
"TestStatusStr" : "Failed",
"JobName" : "My-Demo/dev/ServerJob1",
"Logs" : "PASS - Passed - CXXX01",
"SuiteName" : "Tests.ServerTests",
"LastMessage" : "PASS - Passed - CXXX01: \n",
"Environment" : "dev"
}
}
},
{
"_index" : "index19",
"_type" : "_doc",
"_id" : "QqlosXEBuBOc-UQkoue4",
"_score" : 1.0,
"_source" : {
"TestId" : "CXXX01",
"TestStatus" : 0,
"Issues" : [ ],
"Created" : "2020-04-24T10:00:00",
"MetaData" : {
"TestName" : "Test1",
"LastException" : null,
"DurationMs" : 1980.5899000000002,
"Platform" : {
"BrowserName" : "chrome",
"OS" : "windows",
"OsVersion" : "10"
},
"Categories" : [
"Cat1",
"Cat2",
"CXXX01"
],
"Priority" : null,
"TestStatusStr" : "Failed",
"JobName" : "My-Demo/dev/ServerJob1",
"Logs" : "PASS - Passed - CXXX01",
"SuiteName" : "Tests.ServerTests",
"LastMessage" : "PASS - Passed - CXXX01: \n",
"Environment" : "dev"
}
}
},
{
"_index" : "index19",
"_type" : "_doc",
"_id" : "Q6losXEBuBOc-UQkwecl",
"_score" : 1.0,
"_source" : {
"TestId" : "CXXX01",
"TestStatus" : 0,
"Issues" : [ ],
"Created" : "2020-04-24T11:00:00",
"MetaData" : {
"TestName" : "Test1",
"LastException" : null,
"DurationMs" : 1980.5899000000002,
"Platform" : {
"BrowserName" : "chrome",
"OS" : "windows",
"OsVersion" : "10"
},
"Categories" : [
"Cat1",
"Cat2",
"CXXX01"
],
"Priority" : null,
"TestStatusStr" : "Passed",
"JobName" : "My-Demo/dev/ServerJob1",
"Logs" : "PASS - Passed - CXXX01",
"SuiteName" : "Tests.ServerTests",
"LastMessage" : "PASS - Passed - CXXX01: \n",
"Environment" : "dev"
}
}
}
]
Query: I have used date_histogram to create bucket of each day and top_hits
aggregation to get last document of that day
{
"size": 0,
"query": {
"bool": {
"must": [
{
"match": {
"MetaData.Environment": "dev"
}
},
{
"match": {
"TestId": "CXXX01"
}
},
{
"range": {
"Created": {
"gte": "now-3d/d"
}
}
}
]
}
},
"aggs": {
"daily":{
"date_histogram": {
"field": "Created",
"interval": "day"
},
"aggs": {
"last_result": {
"top_hits": {
"size": 1,
"_source": ["MetaData.TestStatusStr"],
"sort": [
{"Created":"desc"}]
}
}
}
}
}
}
Result:
"aggregations" : {
"daily" : {
"buckets" : [
{
"key_as_string" : "2020-04-23T00:00:00.000Z",
"key" : 1587600000000,
"doc_count" : 2,
"last_result" : {
"hits" : {
"total" : {
"value" : 2,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "index19",
"_type" : "_doc",
"_id" : "QalosXEBuBOc-UQkieeR",
"_score" : null,
"_source" : {
"MetaData" : {
"TestStatusStr" : "Failed"
}
},
"sort" : [
1587636000000
]
}
]
}
}
},
{
"key_as_string" : "2020-04-24T00:00:00.000Z",
"key" : 1587686400000,
"doc_count" : 2,
"last_result" : {
"hits" : {
"total" : {
"value" : 2,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "index19",
"_type" : "_doc",
"_id" : "Q6losXEBuBOc-UQkwecl",
"_score" : null,
"_source" : {
"MetaData" : {
"TestStatusStr" : "Passed"
}
},
"sort" : [
1587726000000
]
}
]
}
}
}
]
}
}