Memory distribution in elasticsearch - elasticsearch

Question

I am using AWS elastic search service and having each node with 16GB RAM, 4 cores and 160GB EBS. While looking at the
GET _nodes/stats
I found that I have only 0.27GB of memory left in my node. I understand the fact that approximately the following amount of memory is dedicated for these processes:
Heap : 8GB
Swap: 2GB
Fielddate + Segments: 1GB
I don't understand what happens with the rest of the 4-5GB of memory, which process is consuming this memory?
The following is the stat of one node which I got from the api mentioned above:
{
"timestamp": 1536212516381,
"name": "",
"roles": [
"master",
"data",
"ingest"
],
"indices": {
"docs": {
"count": 40811869,
"deleted": 12313549
},
"store": {
"size_in_bytes": 21211684802
},
"indexing": {
"index_total": 17141254,
"index_time_in_millis": 23271050,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 11,
"time_in_millis": 4,
"exists_total": 10,
"exists_time_in_millis": 4,
"missing_total": 1,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 60495650,
"query_time_in_millis": 164996296,
"query_current": 0,
"fetch_total": 1338,
"fetch_time_in_millis": 3947,
"fetch_current": 0,
"scroll_total": 1541,
"scroll_time_in_millis": 147381782,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 752561,
"total_time_in_millis": 156448427,
"total_docs": 6278661063,
"total_size_in_bytes": 2061451815435,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 64499,
"total_auto_throttle_in_bytes": 231968018126
},
"refresh": {
"total": 9621793,
"total_time_in_millis": 126038098,
"listeners": 0
},
"flush": {
"total": 135925,
"total_time_in_millis": 2306674
},
"warmer": {
"current": 0,
"total": 8576200,
"total_time_in_millis": 3784313
},
"query_cache": {
"memory_size_in_bytes": 47531137,
"total_count": 315917952,
"hit_count": 69088483,
"miss_count": 246829469,
"cache_size": 16561,
"cache_count": 740755,
"evictions": 724194
},
"fielddata": {
"memory_size_in_bytes": 19006048,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 2256,
"memory_in_bytes": 62556325,
"terms_memory_in_bytes": 50154620,
"stored_fields_memory_in_bytes": 6587256,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 557504,
"points_memory_in_bytes": 2812593,
"doc_values_memory_in_bytes": 2444352,
"index_writer_memory_in_bytes": 0,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 6743824,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {}
},
"translog": {
"operations": 322600,
"size_in_bytes": 511751430,
"uncommitted_operations": 22528,
"uncommitted_size_in_bytes": 35107521
},
"request_cache": {
"memory_size_in_bytes": 84792534,
"evictions": 13922143,
"hit_count": 7667542,
"miss_count": 35693456
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 952628
}
},
"os": {
"timestamp": 1536212516872,
"cpu": {
"percent": 23,
"load_average": {
"1m": 1.29,
"5m": 1.19,
"15m": 1.16
}
},
"mem": {
"total_in_bytes": 16823488512,
"free_in_bytes": 297766912,
"used_in_bytes": 16525721600,
"free_percent": 2,
"used_percent": 98
},
"swap": {
"total_in_bytes": 2147479552,
"free_in_bytes": 2143141888,
"used_in_bytes": 4337664
}
},
"process": {
"timestamp": 1536212516872,
"open_file_descriptors": 2573,
"max_file_descriptors": 128000,
"cpu": {
"percent": 19,
"total_in_millis": 4177826450
},
"mem": {
"total_virtual_in_bytes": 34029547520
}
},
"jvm": {
"timestamp": 1536212516874,
"uptime_in_millis": 6132906547,
"mem": {
"heap_used_in_bytes": 5277036968,
"heap_used_percent": 61,
"heap_committed_in_bytes": 8555069440,
"heap_max_in_bytes": 8555069440,
"non_heap_used_in_bytes": 278172640,
"non_heap_committed_in_bytes": 304259072,
"pools": {
"young": {
"used_in_bytes": 246875928,
"max_in_bytes": 279183360,
"peak_used_in_bytes": 279183360,
"peak_max_in_bytes": 279183360
},
"survivor": {
"used_in_bytes": 6813256,
"max_in_bytes": 34865152,
"peak_used_in_bytes": 34865152,
"peak_max_in_bytes": 34865152
},
"old": {
"used_in_bytes": 5023347784,
"max_in_bytes": 8241020928,
"peak_used_in_bytes": 6966292552,
"peak_max_in_bytes": 8241020928
}
}
},
"threads": {
"count": 234,
"peak_count": 250
},
"gc": {
"collectors": {
"young": {
"collection_count": 4474572,
"collection_time_in_millis": 127468649
},
"old": {
"collection_count": 920,
"collection_time_in_millis": 325448
}
}
},
"buffer_pools": {
"direct": {
"count": 463,
"used_in_bytes": 154976141,
"total_capacity_in_bytes": 154976139
},
"mapped": {
"count": 4914,
"used_in_bytes": 21027708516,
"total_capacity_in_bytes": 21027708516
}
},
"classes": {
"current_loaded_count": 26154,
"total_loaded_count": 26712,
"total_unloaded_count": 558
}
},
"thread_pool": {
"bulk": {
"threads": 4,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 4,
"completed": 17141525
},
"fetch_shard_started": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"fetch_shard_store": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"flush": {
"threads": 2,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 2,
"completed": 271881
},
"force_merge": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"generic": {
"threads": 32,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 128,
"completed": 43205055
},
"get": {
"threads": 4,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 4,
"completed": 8
},
"index": {
"threads": 3,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 3,
"completed": 3
},
"listener": {
"threads": 2,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 2,
"completed": 4
},
"management": {
"threads": 5,
"queue": 0,
"active": 1,
"rejected": 0,
"largest": 5,
"completed": 28115022
},
"refresh": {
"threads": 2,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 2,
"completed": 1023804661
},
"search": {
"threads": 7,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 7,
"completed": 161702240
},
"snapshot": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 2,
"completed": 21133
},
"warmer": {
"threads": 2,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 2,
"completed": 95146934
}
},
"fs": {
"timestamp": 1536212516874,
"total": {
"total_in_bytes": 168968957952,
"free_in_bytes": 147126472704,
"available_in_bytes": 138519760896
},
"data": [
{
"type": "ext4",
"total_in_bytes": 168968957952,
"free_in_bytes": 147126472704,
"available_in_bytes": 138519760896
}
],
"io_stats": {
"devices": [
{
"device_name": "xvdf",
"operations": 133367878,
"read_operations": 1456763,
"write_operations": 131911115,
"read_kilobytes": 104740824,
"write_kilobytes": 3178855500
}
],
"total": {
"operations": 133367878,
"read_operations": 1456763,
"write_operations": 131911115,
"read_kilobytes": 104740824,
"write_kilobytes": 3178855500
}
}
},
"breakers": {
"request": {
"limit_size_in_bytes": 5133041664,
"limit_size": "4.7gb",
"estimated_size_in_bytes": 0,
"estimated_size": "0b",
"overhead": 1,
"tripped": 0
},
"fielddata": {
"limit_size_in_bytes": 5133041664,
"limit_size": "4.7gb",
"estimated_size_in_bytes": 19006048,
"estimated_size": "18.1mb",
"overhead": 1.03,
"tripped": 0
},
"in_flight_requests": {
"limit_size_in_bytes": 8555069440,
"limit_size": "7.9gb",
"estimated_size_in_bytes": 1657,
"estimated_size": "1.6kb",
"overhead": 1,
"tripped": 0
},
"accounting": {
"limit_size_in_bytes": 8555069440,
"limit_size": "7.9gb",
"estimated_size_in_bytes": 62556325,
"estimated_size": "59.6mb",
"overhead": 1,
"tripped": 0
},
"parent": {
"limit_size_in_bytes": 5988548608,
"limit_size": "5.5gb",
"estimated_size_in_bytes": 81564030,
"estimated_size": "77.7mb",
"overhead": 1,
"tripped": 0
}
},
"script": {
"compilations": 5,
"cache_evictions": 0
},
"discovery": {
"cluster_state_queue": {
"total": 0,
"pending": 0,
"committed": 0
},
"published_cluster_states": {
"full_states": 1,
"incompatible_diffs": 0,
"compatible_diffs": 70303
}
},
"ingest": {
"total": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
},
"pipelines": {}
},
"adaptive_selection": {
"qhqf-YADRF2gS1Vu6EV_8Q": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 6520537,
"avg_response_time_ns": 15403676,
"rank": "15.4"
},
"I21NiLksQqCNV3-vP3uCNA": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 3079216,
"avg_response_time_ns": 1115616,
"rank": "1.1"
},
"T2_cLfzrQBKd7WTR0p6jWQ": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 12493125,
"avg_response_time_ns": 1141514,
"rank": "1.1"
},
"ZZYgl3WARsKE-80HXhRbVw": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 3919880,
"avg_response_time_ns": 6063593,
"rank": "6.1"
}
}
}
Can anyone please help explain the memory distribution in this node? Is it okay to have only 0.27GB free memory space or what should I do to make more free memory available in the node?