Elasticsearch separate aggregation based on values from first - elasticsearch

Question

I'm using a Elasticsearch 6.8.8 and trying to aggregate the number of entities and relationships over a given time period.
Here is the data structure and examples values of the index:
date entityOrRelationshipId startId endId type
=========================================================================
DATETIMESTAMP ENT1_ID null null ENTITY
DATETIMESTAMP ENT2_ID null null ENTITY
DATETIMESTAMP ENT3_ID null null ENTITY
DATETIMESTAMP REL1_ID ENT1_ID ENT2_ID RELATIONSHIP
DATETIMESTAMP REL2_ID ENT3_ID ENT1_ID RELATIONSHIP
etc.
For a given entity ID, I want to get the top 50 relationships. I have started with the following query.
{
"size": 0,
"query": {
"bool": {
"must": [
{
"range": {
"date": {
"gte": "2020-04-01T00:00:00.000+00:00",
"lt": "2020-04-28T00:00:00.000+00:00"
}
}
}
]
}
},
"aggs": {
"my_rels": {
"filter": {
"bool": {
"must": [
{
"term": {
"type": "RELATIONSHIP"
}
},
{
"bool": {
"should": [
{
"term": {"startId": "ENT1_ID"}
},
{
"term": {"endId": "ENT1_ID"}
}
]
}
}
]
}
},
"aggs": {
"my_rels2": {
"terms": {
"field": "entityOrRelationshipId",
"size": 50
},
"aggs": {
"my_rels3": {
"top_hits": {
"_source": {
"includes": ["startId","endId"]
},
"size": 1
}
}
}
}
}
}
}
}
This produces the following results:
{
"took": 54,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 93122,
"max_score": 0.0,
"hits": []
},
"aggregations": {
"my_rels": {
"doc_count": 332,
"my_rels2": {
"doc_count_error_upper_bound": 6,
"sum_other_doc_count": 259,
"buckets": [
{
"key": "REL1_ID",
"doc_count": 47,
"my_rels3": {
"hits": {
"total": 47,
"max_score": 1.0,
"hits": [
{
"_index": "trends",
"_type": "trend",
"_score": 1.0,
"_source": {
"endId": "ENT2_ID",
"startId": "ENT1_ID"
}
}
]
}
}
},
{
"key": "REL2_ID",
"doc_count": 26,
"my_rels3": {
"hits": {
"total": 26,
"max_score": 1.0,
"hits": [
{
"_index": "trends",
"_type": "trend",
"_score": 1.0,
"_source": {
"endId": "ENT1_ID",
"startId": "ENT3_ID"
}
}
]
}
}
}
]
}
}
}
}
This lists the top 50 relationships. For each relationship it lists the relationship ID, the count and the entity ids (startId, endId). What I would like to do now is produce another aggregation of entity counts for those distinct entities. Ideally this would not be a nested aggregation but a separate one using the rel ids identified in the first aggregation.
Is that possible to do in this query?

Answer 1

Unfortunately you cannot aggregate over the results of top_hits in Elasticsearch.
Here is the link to GitHub issue.
You can have other aggregation on a parallel level of top_hit but you cannot have any sub_aggregation below top_hit.
You can have a parallel level aggregation like:
"aggs": {
"top_hits_agg": {
"top_hits": {
"size": 10,
"_source": {
"includes": ["score"]
}
}
},
"avg_agg": {
"avg": {
"field": "score"
}
}
}