ensure minimum functionality under heavy search load elk - elasticsearch

Question

I have a 3x node elk cluster with fillowing node characteristics:
96GB RAM
25GB heap
10x CPU
6TB total indexes size for past 30 days distributed in 1.5TB per node
the filebeat index 50GB per index split across 3x shards
~ total 100GB/day , including the 1xreplica ~ 200GB/day
The thing is that when viewing my log metrics for past 1-7 days everything is fine , but if I open dashboard to see details for past 30 days I cannot do anything else , and if I open kibana from other browser tab is throw error 408 , all CPU's are at 100%
Please, advice if there is any option to ensure single request will not take the 100% from all CPUs in all nodes and I would be able to see from second tab at least the cluster monitoring?
Thanks,
R2D2
Output from GET _cluster/stats?pretty&human
{
"_nodes" : {
"total" : 6,
"successful" : 6,
"failed" : 0
},
"cluster_name" : "elk",
"cluster_uuid" : "CztzsocFTgmKHWI_GnqDvQ",
"timestamp" : 1634106014829,
"status" : "green",
"indices" : {
"count" : 116,
"shards" : {
"total" : 304,
"primaries" : 152,
"replication" : 1.0,
"index" : {
"shards" : {
"min" : 2,
"max" : 6,
"avg" : 2.6206896551724137
},
"primaries" : {
"min" : 1,
"max" : 3,
"avg" : 1.3103448275862069
},
"replication" : {
"min" : 1.0,
"max" : 1.0,
"avg" : 1.0
}
}
},
"docs" : {
"count" : 4672727260,
"deleted" : 1262101
},
"store" : {
"size" : "4tb",
"size_in_bytes" : 4490413194045,
"reserved" : "0b",
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size" : "58.3mb",
"memory_size_in_bytes" : 61151040,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "4.1gb",
"memory_size_in_bytes" : 4406840440,
"total_count" : 2390354,
"hit_count" : 14385,
"miss_count" : 2375969,
"cache_size" : 4760,
"cache_count" : 5724,
"evictions" : 964
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 3404,
"memory" : "60.1mb",
"memory_in_bytes" : 63090152,
"terms_memory" : "29.9mb",
"terms_memory_in_bytes" : 31446928,
"stored_fields_memory" : "3.9mb",
"stored_fields_memory_in_bytes" : 4122896,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "13.8kb",
"norms_memory_in_bytes" : 14208,
"points_memory" : "0b",
"points_memory_in_bytes" : 0,
"doc_values_memory" : "26.2mb",
"doc_values_memory_in_bytes" : 27506120,
"index_writer_memory" : "59.6mb",
"index_writer_memory_in_bytes" : 62543712,
"version_map_memory" : "830b",
"version_map_memory_in_bytes" : 830,
"fixed_bit_set" : "1gb",
"fixed_bit_set_memory_in_bytes" : 1168657528,
"max_unsafe_auto_id_timestamp" : 1634083207166,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "alias",
"count" : 2686,
"index_count" : 79
},
{
"name" : "binary",
"count" : 14,
"index_count" : 3
},
{
"name" : "boolean",
"count" : 8714,
"index_count" : 103
},
{
"name" : "byte",
"count" : 1,
"index_count" : 1
},
{
"name" : "date",
"count" : 11809,
"index_count" : 115
},
{
"name" : "double",
"count" : 2607,
"index_count" : 79
},
{
"name" : "flattened",
"count" : 641,
"index_count" : 80
},
{
"name" : "float",
"count" : 2611,
"index_count" : 88
},
{
"name" : "geo_point",
"count" : 639,
"index_count" : 80
},
{
"name" : "half_float",
"count" : 56,
"index_count" : 14
},
{
"name" : "integer",
"count" : 186,
"index_count" : 11
},
{
"name" : "ip",
"count" : 9888,
"index_count" : 80
},
{
"name" : "keyword",
"count" : 285169,
"index_count" : 113
},
{
"name" : "long",
"count" : 79464,
"index_count" : 109
},
{
"name" : "nested",
"count" : 272,
"index_count" : 95
},
{
"name" : "object",
"count" : 57866,
"index_count" : 114
},
{
"name" : "short",
"count" : 7981,
"index_count" : 80
},
{
"name" : "text",
"count" : 7837,
"index_count" : 104
}
]
},
"analysis" : {
"char_filter_types" : [ ],
"tokenizer_types" : [ ],
"filter_types" : [
{
"name" : "pattern_capture",
"count" : 1,
"index_count" : 1
}
],
"analyzer_types" : [
{
"name" : "custom",
"count" : 1,
"index_count" : 1
}
],
"built_in_char_filters" : [ ],
"built_in_tokenizers" : [
{
"name" : "uax_url_email",
"count" : 1,
"index_count" : 1
}
],
"built_in_filters" : [
{
"name" : "lowercase",
"count" : 1,
"index_count" : 1
},
{
"name" : "unique",
"count" : 1,
"index_count" : 1
}
],
"built_in_analyzers" : [ ]
}
},
"nodes" : {
"count" : {
"total" : 6,
"coordinating_only" : 0,
"data" : 3,
"data_cold" : 3,
"data_content" : 3,
"data_hot" : 3,
"data_warm" : 3,
"ingest" : 3,
"master" : 3,
"ml" : 6,
"remote_cluster_client" : 3,
"transform" : 3,
"voting_only" : 0
},
"versions" : [
"7.10.2"
],
"os" : {
"available_processors" : 36,
"allocated_processors" : 36,
"names" : [
{
"name" : "Linux",
"count" : 6
}
],
"pretty_names" : [
{
"pretty_name" : "CentOS Linux 8",
"count" : 6
}
],
"mem" : {
"total" : "294gb",
"total_in_bytes" : 315680096256,
"free" : "36.5gb",
"free_in_bytes" : 39295123456,
"used" : "257.4gb",
"used_in_bytes" : 276384972800,
"free_percent" : 12,
"used_percent" : 88
}
},
"process" : {
"cpu" : {
"percent" : 3
},
"open_file_descriptors" : {
"min" : 389,
"max" : 1318,
"avg" : 849
}
},
"jvm" : {
"max_uptime" : "4.6d",
"max_uptime_in_millis" : 399502750,
"versions" : [
{
"version" : "15.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "15.0.1+9",
"vm_vendor" : "AdoptOpenJDK",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 6
}
],
"mem" : {
"heap_used" : "30.9gb",
"heap_used_in_bytes" : 33225823136,
"heap_max" : "78gb",
"heap_max_in_bytes" : 83751862272
},
"threads" : 428
},
"fs" : {
"total" : "5.9tb",
"total_in_bytes" : 6521808826368,
"free" : "1.8tb",
"free_in_bytes" : 2028853403648,
"available" : "1.5tb",
"available_in_bytes" : 1697288646656
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"security4" : 6
},
"http_types" : {
"security4" : 6
}
},
"discovery_types" : {
"zen" : 6
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "docker",
"count" : 6
}
],
"ingest" : {
"number_of_pipelines" : 4,
"processor_stats" : {
"conditional" : {
"count" : 307645168,
"failed" : 0,
"current" : 0,
"time" : "18.5s",
"time_in_millis" : 18569
},
"geoip" : {
"count" : 307645168,
"failed" : 0,
"current" : 0,
"time" : "11.3s",
"time_in_millis" : 11315
},
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"rename" : {
"count" : 615290336,
"failed" : 0,
"current" : 0,
"time" : "7.1s",
"time_in_millis" : 7100
},
"script" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
}
}
}
}
}
Output from : GET _cat/nodes?v&h=v,n,m,r,rm,du,dt,cpu,ram.percent,ram.current,heap.percent,search.query_current,heap.current,heap.total,segments.count&s=n:asc
v n m r rm du dt cpu ram.percent ram.current heap.percent search.query_current heap.current segments.count
7.10.2 elk-ingest-data-0 - cdhilrstw 96gb 1.3tb 1.9tb 100 78 74.5gb 46 0 11.7gb 1162
7.10.2 elk-ingest-data-1 - cdhilrstw 96gb 1.5tb 1.9tb 71 77 73.7gb 34 0 8.6gb 1155
7.10.2 elk-ingest-data-2 - cdhilrstw 96gb 1.5tb 1.9tb 68 100 95.9gb 52 0 13gb 1091
7.10.2 elk-master-0 * lm 2gb 568.3mb 9.7gb 53 100 1.9gb 37 0 381.9mb 0
7.10.2 elk-master-1 - lm 2gb 568.3mb 9.7gb 2 68 1.3gb 61 0 628.9mb 0
7.10.2 elk-master-2 - lm 2gb 568.3mb 9.7gb 3 68 1.3gb 55 0 572.6mb 0