Elasticsearch sort on derivative aggregation to find trending topics - elasticsearch

Question

I am collecting twitter data and want to find the account that received the highest number of new followers during a period of 48 hours.
The index is populated regularly with account data and a count of followers, together with a datestamp.
I got so far as by getting a derivative value, but I can't figure out how to sort on the derivative so as to return the accounts with the highest derivatives (accounts which won or lost the highest amount of followers during the last 48 hours).
The mapping:
{
"twfollowers" : {
"mappings" : {
"twfollowers" : {
"properties" : {
"followers" : {
"type" : "long"
},
"logDate" : {
"type" : "date"
},
"screen_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"user_id" : {
"type" : "long"
}
}
}
}
}
}
And here is the query:
GET /twfollowers/twfollowers/_search
{
"query": {
"bool" : {
"must" : {
"range": {"logDate": {
"gte" : "now-2d/d",
"lt" : "now/d"
}}
}
}
},
"size": 0,
"aggs": {
"users": {
"terms": {
"field": "screen_name.keyword",
"size": 10
},
"aggs": {
"my_date_histo": {
"date_histogram": {
"field": "logDate",
"interval": "day"
},
"aggs": {
"the_sum": {
"max": {
"field": "followers"
}
},
"the_diff": {
"derivative": {
"buckets_path": "the_sum"
}
}
}
}
}
}
}
}
And finally the results:
{
"took": 12,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 129621,
"max_score": 0,
"hits": []
},
"aggregations": {
"users": {
"doc_count_error_upper_bound": 356,
"sum_other_doc_count": 122394,
"buckets": [
{
"key": "Sero83954560",
"doc_count": 968,
"my_date_histo": {
"buckets": [
{
"key_as_string": "2018-09-03T00:00:00.000Z",
"key": 1535932800000,
"doc_count": 268,
"the_sum": {
"value": 870
}
},
{
"key_as_string": "2018-09-04T00:00:00.000Z",
"key": 1536019200000,
"doc_count": 700,
"the_sum": {
"value": 873
},
"the_diff": {
"value": 3
}
}
]
}
},
{
"key": "Hajk",
"doc_count": 913,
"my_date_histo": {
"buckets": [
{
"key_as_string": "2018-09-03T00:00:00.000Z",
"key": 1535932800000,
"doc_count": 287,
"the_sum": {
"value": 1529
}
},
{
"key_as_string": "2018-09-04T00:00:00.000Z",
"key": 1536019200000,
"doc_count": 626,
"the_sum": {
"value": 1532
},
"the_diff": {
"value": 3
}
}
]
}
},
{
"key": "idagoraSE",
"doc_count": 831,
"my_date_histo": {
"buckets": [
{
"key_as_string": "2018-09-03T00:00:00.000Z",
"key": 1535932800000,
"doc_count": 173,
"the_sum": {
"value": 1165
}
},
{
"key_as_string": "2018-09-04T00:00:00.000Z",
"key": 1536019200000,
"doc_count": 658,
"the_sum": {
"value": 1166
},
"the_diff": {
"value": 1
}
}
]
}
},
{
"key": "Dodgeman2ever",
"doc_count": 779,
"my_date_histo": {
"buckets": [
{
"key_as_string": "2018-09-03T00:00:00.000Z",
"key": 1535932800000,
"doc_count": 235,
"the_sum": {
"value": 2017
}
},
{
"key_as_string": "2018-09-04T00:00:00.000Z",
"key": 1536019200000,
"doc_count": 544,
"the_sum": {
"value": 2031
},
"the_diff": {
"value": 14
}
}
]
}
},
{
"key": "KettilsMead",
"doc_count": 743,
"my_date_histo": {
"buckets": [
{
"key_as_string": "2018-09-03T00:00:00.000Z",
"key": 1535932800000,
"doc_count": 453,
"the_sum": {
"value": 2860
}
},
{
"key_as_string": "2018-09-04T00:00:00.000Z",
"key": 1536019200000,
"doc_count": 290,
"the_sum": {
"value": 2862
},
"the_diff": {
"value": 2
}
}
]
}
},
{
"key": "BWhalbergarvid",
"doc_count": 683,
"my_date_histo": {
"buckets": [
{
"key_as_string": "2018-09-03T00:00:00.000Z",
"key": 1535932800000,
"doc_count": 87,
"the_sum": {
"value": 200
}
},
{
"key_as_string": "2018-09-04T00:00:00.000Z",
"key": 1536019200000,
"doc_count": 596,
"the_sum": {
"value": 203
},
"the_diff": {
"value": 3
}
}
]
}
},
{
"key": "MorKarins",
"doc_count": 601,
"my_date_histo": {
"buckets": [
{
"key_as_string": "2018-09-03T00:00:00.000Z",
"key": 1535932800000,
"doc_count": 86,
"the_sum": {
"value": 3874
}
},
{
"key_as_string": "2018-09-04T00:00:00.000Z",
"key": 1536019200000,
"doc_count": 515,
"the_sum": {
"value": 3875
},
"the_diff": {
"value": 1
}
}
]
}
},
{
"key": "erlhel",
"doc_count": 593,
"my_date_histo": {
"buckets": [
{
"key_as_string": "2018-09-03T00:00:00.000Z",
"key": 1535932800000,
"doc_count": 218,
"the_sum": {
"value": 730
}
},
{
"key_as_string": "2018-09-04T00:00:00.000Z",
"key": 1536019200000,
"doc_count": 375,
"the_sum": {
"value": 730
},
"the_diff": {
"value": 0
}
}
]
}
},
{
"key": "SaveSweden2018",
"doc_count": 560,
"my_date_histo": {
"buckets": [
{
"key_as_string": "2018-09-03T00:00:00.000Z",
"key": 1535932800000,
"doc_count": 157,
"the_sum": {
"value": 711
}
},
{
"key_as_string": "2018-09-04T00:00:00.000Z",
"key": 1536019200000,
"doc_count": 403,
"the_sum": {
"value": 732
},
"the_diff": {
"value": 21
}
}
]
}
},
{
"key": "MXCartoons",
"doc_count": 556,
"my_date_histo": {
"buckets": [
{
"key_as_string": "2018-09-03T00:00:00.000Z",
"key": 1535932800000,
"doc_count": 40,
"the_sum": {
"value": 4313
}
},
{
"key_as_string": "2018-09-04T00:00:00.000Z",
"key": 1536019200000,
"doc_count": 516,
"the_sum": {
"value": 4315
},
"the_diff": {
"value": 2
}
}
]
}
}
]
}
}
}