>10sec aggregation performance in elasticsearch - elasticsearch

Question

I'm new to elasticsearch and have recently setup a flow of storing JMX data polled every 10seconds into an elasticearch index that looks like this:
{
"jmxmetrics": {
"mappings": {
"kafkajmx": {
"properties": {
"#timestamp": {
"type": "date"
},
"#version": {
"type": "keyword"
},
"attr_type": {
"type": "keyword"
},
"host": {
"type": "keyword"
},
"kafka_broker_id": {
"type": "keyword"
},
"kafka_clientId": {
"type": "keyword"
},
"kafka_fetcher_id": {
"type": "keyword"
},
"kafka_metric_group": {
"type": "keyword"
},
"kafka_metric_name": {
"type": "keyword"
},
"kafka_metric_type": {
"type": "keyword"
},
"kafka_topic": {
"type": "keyword"
},
"metric_path": {
"type": "text"
},
"metric_value_number": {
"type": "long"
},
"metric_value_string": {
"type": "keyword"
},
"path": {
"type": "text"
},
"topic_partition": {
"type": "keyword"
}
}
}
}
}
}
I'm then using a web portal built in ReactJs to hit the API endpoints in elastic search so that I can populate things like: hosts in a drop down, or topics in that host that was selected.
For example, if I run this query:
POST /jmxmetrics/kafkajmx/_search
{
"size": 0,
"aggs" : {
"topics": {
"filter": {
"range": {
"#timestamp": {
"from": "now-60s"
}
}
},
"aggs": {
"topics":{
"terms": {
"field": "kafka_topic",
"size": 500
}
}
}
}
},
"query": {
"match": {
"host": "host_A"
}
}
}
it takes about 25seconds to return:
{
"took": 24709,
"timed_out": false,
"_shards": {
"total": 3,
"successful": 3,
"failed": 0
},
"hits": {
"total": 1053380207,
"max_score": 0,
"hits": []
},
"aggregations": {
"topics": {
"doc_count": 54915,
"topics": {
"doc_count_error_upper_bound": 189,
"sum_other_doc_count": 45098,
"buckets": [
{
"key": "__consumer_offsets",
"doc_count": 503
},
{
"key": "topic 1",
"doc_count": 196
},
{
"key": "topic 2",
"doc_count": 193
},
{
"key": "topic 3",
"doc_count": 142
},
...
]
}
}
}
}
Any elasticsearch aggregation query seems to take 10+ seconds to run, and my cluster topology looks like this (running on AWS Ec2):
3 master/data nodes c4.2xlarge (8 cores, and 15Gb ram)
1 client node t2.large (2 cores, 8gb ram)
Any idea on what I could be improving here? I know this is a fairly broad question but any pointers will be helpful.
-------EDIT--------
Here are my settings:
{
"jmxmetrics": {
"settings": {
"index": {
"creation_date": "1489771989403",
"number_of_shards": "3",
"number_of_replicas": "2",
"uuid": "DeLP9MrQT6S_kILMmXorew",
"version": {
"created": "5020199"
},
"provided_name": "sdpjmx"
}
}
}
}
also, here are my stats:
{
"_shards": {
"total": 9,
"successful": 9,
"failed": 0
},
"_all": {
"primaries": {
"docs": {
"count": 1402484967,
"deleted": 0
},
"store": {
"size_in_bytes": 261239372859,
"throttle_time_in_millis": 0
},
"indexing": {
"index_total": 1402487151,
"index_time_in_millis": 108231827,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 1,
"time_in_millis": 13,
"exists_total": 0,
"exists_time_in_millis": 0,
"missing_total": 1,
"missing_time_in_millis": 13,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 692,
"query_time_in_millis": 1390846,
"query_current": 0,
"fetch_total": 76,
"fetch_time_in_millis": 15444,
"fetch_current": 0,
"scroll_total": 0,
"scroll_time_in_millis": 0,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 4,
"current_docs": 57546764,
"current_size_in_bytes": 10961768409,
"total": 359830,
"total_time_in_millis": 298632418,
"total_docs": 8639051262,
"total_size_in_bytes": 1733171768674,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 208268389,
"total_auto_throttle_in_bytes": 15728640
},
"refresh": {
"total": 2290338,
"total_time_in_millis": 27698495,
"listeners": 0
},
"flush": {
"total": 1398,
"total_time_in_millis": 512144
},
"warmer": {
"current": 0,
"total": 2291685,
"total_time_in_millis": 365893
},
"query_cache": {
"memory_size_in_bytes": 131229184,
"total_count": 13019,
"hit_count": 905,
"miss_count": 12114,
"cache_size": 118,
"cache_count": 139,
"evictions": 21
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 167,
"memory_in_bytes": 868415719,
"terms_memory_in_bytes": 731359089,
"stored_fields_memory_in_bytes": 125593776,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 32064,
"points_memory_in_bytes": 11138810,
"doc_values_memory_in_bytes": 291980,
"index_writer_memory_in_bytes": 9788964,
"version_map_memory_in_bytes": 290304,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {}
},
"translog": {
"operations": 529724,
"size_in_bytes": 275528684
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 84
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 0
}
},
"total": {
"docs": {
"count": 4207454779,
"deleted": 0
},
"store": {
"size_in_bytes": 776849456528,
"throttle_time_in_millis": 0
},
"indexing": {
"index_total": 4207461365,
"index_time_in_millis": 333522565,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 2,
"time_in_millis": 13,
"exists_total": 0,
"exists_time_in_millis": 0,
"missing_total": 2,
"missing_time_in_millis": 13,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 2046,
"query_time_in_millis": 4439867,
"query_current": 0,
"fetch_total": 204,
"fetch_time_in_millis": 61910,
"fetch_current": 0,
"scroll_total": 0,
"scroll_time_in_millis": 0,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 9,
"current_docs": 90922106,
"current_size_in_bytes": 17436129588,
"total": 993305,
"total_time_in_millis": 889392309,
"total_docs": 25596362951,
"total_size_in_bytes": 5109527514241,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 622031369,
"total_auto_throttle_in_bytes": 47185920
},
"refresh": {
"total": 6870703,
"total_time_in_millis": 80442220,
"listeners": 0
},
"flush": {
"total": 4194,
"total_time_in_millis": 1542414
},
"warmer": {
"current": 0,
"total": 6874724,
"total_time_in_millis": 1048456
},
"query_cache": {
"memory_size_in_bytes": 428618904,
"total_count": 38315,
"hit_count": 3196,
"miss_count": 35119,
"cache_size": 371,
"cache_count": 438,
"evictions": 67
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 492,
"memory_in_bytes": 2604314152,
"terms_memory_in_bytes": 2194198429,
"stored_fields_memory_in_bytes": 375756256,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 94464,
"points_memory_in_bytes": 33422427,
"doc_values_memory_in_bytes": 842576,
"index_writer_memory_in_bytes": 22793264,
"version_map_memory_in_bytes": 853888,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {}
},
"translog": {
"operations": 1591047,
"size_in_bytes": 827579505
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 345
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 0
}
}
},
"indices": {
"jmxmetrics": {
"primaries": {
"docs": {
"count": 1402484967,
"deleted": 0
},
"store": {
"size_in_bytes": 261239372859,
"throttle_time_in_millis": 0
},
"indexing": {
"index_total": 1402487151,
"index_time_in_millis": 108231827,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 1,
"time_in_millis": 13,
"exists_total": 0,
"exists_time_in_millis": 0,
"missing_total": 1,
"missing_time_in_millis": 13,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 692,
"query_time_in_millis": 1390846,
"query_current": 0,
"fetch_total": 76,
"fetch_time_in_millis": 15444,
"fetch_current": 0,
"scroll_total": 0,
"scroll_time_in_millis": 0,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 4,
"current_docs": 57546764,
"current_size_in_bytes": 10961768409,
"total": 359830,
"total_time_in_millis": 298632418,
"total_docs": 8639051262,
"total_size_in_bytes": 1733171768674,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 208268389,
"total_auto_throttle_in_bytes": 15728640
},
"refresh": {
"total": 2290338,
"total_time_in_millis": 27698495,
"listeners": 0
},
"flush": {
"total": 1398,
"total_time_in_millis": 512144
},
"warmer": {
"current": 0,
"total": 2291685,
"total_time_in_millis": 365893
},
"query_cache": {
"memory_size_in_bytes": 131229184,
"total_count": 13019,
"hit_count": 905,
"miss_count": 12114,
"cache_size": 118,
"cache_count": 139,
"evictions": 21
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 167,
"memory_in_bytes": 868415719,
"terms_memory_in_bytes": 731359089,
"stored_fields_memory_in_bytes": 125593776,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 32064,
"points_memory_in_bytes": 11138810,
"doc_values_memory_in_bytes": 291980,
"index_writer_memory_in_bytes": 9788964,
"version_map_memory_in_bytes": 290304,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {}
},
"translog": {
"operations": 529724,
"size_in_bytes": 275528684
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 84
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 0
}
},
"total": {
"docs": {
"count": 4207454779,
"deleted": 0
},
"store": {
"size_in_bytes": 776849456528,
"throttle_time_in_millis": 0
},
"indexing": {
"index_total": 4207461365,
"index_time_in_millis": 333522565,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 2,
"time_in_millis": 13,
"exists_total": 0,
"exists_time_in_millis": 0,
"missing_total": 2,
"missing_time_in_millis": 13,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 2046,
"query_time_in_millis": 4439867,
"query_current": 0,
"fetch_total": 204,
"fetch_time_in_millis": 61910,
"fetch_current": 0,
"scroll_total": 0,
"scroll_time_in_millis": 0,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 9,
"current_docs": 90922106,
"current_size_in_bytes": 17436129588,
"total": 993305,
"total_time_in_millis": 889392309,
"total_docs": 25596362951,
"total_size_in_bytes": 5109527514241,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 622031369,
"total_auto_throttle_in_bytes": 47185920
},
"refresh": {
"total": 6870703,
"total_time_in_millis": 80442220,
"listeners": 0
},
"flush": {
"total": 4194,
"total_time_in_millis": 1542414
},
"warmer": {
"current": 0,
"total": 6874724,
"total_time_in_millis": 1048456
},
"query_cache": {
"memory_size_in_bytes": 428618904,
"total_count": 38315,
"hit_count": 3196,
"miss_count": 35119,
"cache_size": 371,
"cache_count": 438,
"evictions": 67
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 492,
"memory_in_bytes": 2604314152,
"terms_memory_in_bytes": 2194198429,
"stored_fields_memory_in_bytes": 375756256,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 94464,
"points_memory_in_bytes": 33422427,
"doc_values_memory_in_bytes": 842576,
"index_writer_memory_in_bytes": 22793264,
"version_map_memory_in_bytes": 853888,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {}
},
"translog": {
"operations": 1591047,
"size_in_bytes": 827579505
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 345
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 0
}
}
}
}
}