How can I get distinct values of nested fields in elasticsearch? - elasticsearch

Question

I have below document structure in elasticsearch:
root
|-- userid: string (nullable = true)
|-- name: string (nullable = true)
|-- applications: array (nullable = true)
| |-- element: struct (containsNull = true)
| | |-- applicationid: string (nullable = true)
| | |-- createdat: string (nullable = true)
| | |-- source_name: string (nullable = true)
| | |-- accounts: array (nullable = true)
| | | |-- element: struct (containsNull = true)
| | | | |-- applicationcreditreportaccountid: string
(nullable = true)
| | | | |-- account_type: integer (nullable = true)
| | | | |-- account_department: string (nullable = true)
Below is the mapping of my index:
{
"bureau_data" : {
"mappings" : {
"dynamic_date_formats" : [
"yyyy-MM-dd"
],
"dynamic_templates" : [
{
"objects" : {
"match_mapping_type" : "object",
"mapping" : {
"type" : "nested"
}
}
}
],
"properties" : {
"raw_derived" : {
"type" : "nested",
"properties" : {
"applications" : {
"type" : "nested",
"properties" : {
"accounts" : {
"type" : "nested",
"properties" : {
"account_type_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"accounttypeid" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"applicationcreditreportaccountid" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"currentbalance" : {
"type" : "long"
},
"dayspastdue" : {
"type" : "long"
},
"institution_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"institutionid" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"applicationcreditreportid" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"applicationid" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"createdat" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"creditbureautypeid" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"dateofbirth" : {
"type" : "date",
"format" : "yyyy-MM-dd"
},
"firstname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"lastname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"score" : {
"type" : "long"
},
"source_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"status" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"updatedat" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"dob" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"firstname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"lastname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"middlename" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"mobilephone" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"source" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"userid" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"fielddata" : true
}
}
}
}
}
I want distinct values of account_type field which is a nested fields. I have tried query which is giving me only distinct count.
GET /my_index/_search?size=0
{
"aggs": {
"nested_path": {
"nested": {
"path": "raw_derived.applications.accounts"
},
"aggs": {
"distinct_values": {
"cardinality": {
"field": "raw_derived.applications.accounts.account_type.keyword"
}
}
}
}
}
}
I expected the output to have distinct values of account_type but the output is count only. Below is my output snippet:
"hits" : {
"total" : {
"value" : 50,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"aggregations" : {
"nested_path" : {
"doc_count" : 828,
"distinct_values" : {
"value" : 70
}
}
}
}
Below is the query I have tried and its working:
GET /bureau_data/_search?size=0
{
"_source": "{aggregations}",
"aggs": {
"unique": {
"nested": {
"path": "raw_derived.applications"
},
"aggs": {
"score_unq": {
"terms": {
"field": "raw_derived.applications.source_name.keyword"
}
}
}
}
}
}
Any suggestion would be helpful

Answer 1

From the official documentation -
Cardinality Aggregation :-
A single-value metrics aggregation that calculates an approximate count of distinct values. Values can be extracted either from specific fields in the document or generated by a script.
Instead of aggregating by "cardinality" , try a terms aggregation as below:
{
"size":0,
"aggregations": {
"distinct_values": {
"terms": {
"field": "raw_derived.applications.accounts.account_type.keyword",
"size": 1000,
"min_doc_count": 1,
"order": [
{
"_count": "desc"
},
{
"_key": "asc"
}
]
}
}
}