Wrong terms aggregate doc_count in elasticsearch - elasticsearch

Question

I am trying to find duplicates in the index by aggregating users by [array] + field using script.
My question is why does terms aggregate only counts once document by a given key (smith#gmail.com_SMITH). And is it possible to change this behavior.
Data:
POST users/user
{
"name" :"SMITH",
"emails" : [
"smith#gmail.com"
]
}
POST users/user
{
"name" :"SMITH",
"emails" : [
"mrsmith#gmail.com",
"smith#gmail.com"
]
}
Distinct query:
POST users/_search
{
"size": 0,
"aggs": {
"duplicateCount": {
"terms": {
"script": {
"inline": "doc['emails.keyword'].value + '_' + doc['name.keyword'].value"
}
}
}
}
}
Result:
"aggregations": {
"duplicateCount": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "mrsmith#gmail.com_SMITH",
"doc_count": 1
},
{
"key": "smith#gmail.com_SMITH",
"doc_count": 1
}
]
}
}

Answer 1

You seem to be only to get the right terms aggregation count with "terms" + "field".
If you try out this query, you can see the difference of "terms" + "field" and "terms" + "script":
{
"from" : 0,
"size" : 0,
"_source" : true,
"query" : {
"bool" : {
"must" : [ {
"match" : {
"name" : {
"query" : "SMITH",
"operator" : "OR",
"fuzziness" : "AUTO",
"prefix_length" : 1,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 1
}
}
} ]
}
},
"aggs": {
"duplicateCount": {
"terms": {
"script": {
"inline": "doc['emails.keyword'].value + '_' + doc['name.keyword'].value"
}
}
},
"duplicateCount2": {
"terms": {
"field": "emails.keyword"
}
}
}
}
Here are the results. See duplicateCount2:
{
"took" : 53,
"timed_out" : false,
"_shards" : {
"total" : 3,
"successful" : 3,
"failed" : 0
},
"hits" : {
"total" : 2,
"max_score" : 0.0,
"hits" : [ ]
},
"aggregations" : {
"duplicateCount2" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [ {
"key" : "smith#gmail.com",
"doc_count" : 2
}, {
"key" : "mrsmith#gmail.com",
"doc_count" : 1
} ]
},
"duplicateCount" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [ {
"key" : "mrsmith#gmail.com_SMITH",
"doc_count" : 1
}, {
"key" : "smith#gmail.com_SMITH",
"doc_count" : 1
} ]
}
}
}

Answer 2

Ok. So I worked around it by iterating over terms array and manually creating desired keys:
def keys = [];
for (p in doc['emails.keyword'].values) {
keys.add(p + doc['name.keyword'].value);
}
return keys;
Here's the result:
"buckets": [
{
"key": "smith#gmail.com_SMITH",
"doc_count": 2
},
{
"key": "mrsmith#gmail.com_SMITH",
"doc_count": 1
}
]