Cytoscape3 Desktop: Dual Labels (from two columns) per Node? - graph-visualization

I have a cytoscape graph for which I would like to present two labels simultaneously per node in the desktop application. Is there a way to do this ? Here is an example (image and export to cyjs). I would like to in addition to the current label (name), have the classID or the synonyms (note that this label is a list).
Here is the cyjs (Cytoscape JSON) for this network:
{
"format_version" : "1.0",
"generated_by" : "cytoscape-3.9.1",
"target_cytoscapejs_version" : "~2.1",
"data" : {
"shared_name" : "Alzheimer's Disease Ontology",
"name" : "Alzheimer's Disease Ontology",
"SUID" : 124,
"shared_name" : "Alzheimer's Disease Ontology",
"selected" : true
},
"elements" : {
"nodes" : [ {
"data" : {
"id" : "191",
"shared_name" : "NEURODEGENERATIVE DISEASE",
"classID" : "DOID_1289",
"synonyms" : [ "NEURODEGENERATIVE DISEASE", "DEGENERATIVE DISEASE" ],
"name" : "NEURODEGENERATIVE DISEASE",
"SUID" : 191,
"id_original" : "155",
"_foretell_labels" : [ "Disease" ],
"selected" : false
},
"position" : {
"x" : 18.285714285714292,
"y" : 67.0
},
"selected" : false
}, {
"data" : {
"id" : "188",
"shared_name" : "CENTRAL NERVOUS SYSTEM DISEASE",
"classID" : "DOID_331",
"name" : "CENTRAL NERVOUS SYSTEM DISEASE",
"SUID" : 188,
"id_original" : "182",
"_foretell_labels" : [ "Disease" ],
"selected" : false
},
"position" : {
"x" : 18.285714285714292,
"y" : 0.0
},
"selected" : false
}, {
"data" : {
"id" : "185",
"shared_name" : "NERVOUS SYSTEM DISEASE",
"classID" : "DOID_863",
"name" : "NERVOUS SYSTEM DISEASE",
"SUID" : 185,
"id_original" : "185",
"_foretell_labels" : [ "Disease" ],
"selected" : false
},
"position" : {
"x" : 18.079472132200525,
"y" : -73.37937578464862
},
"selected" : false
}, {
"data" : {
"id" : "182",
"shared_name" : "DISEASE",
"classID" : "DOID_4",
"name" : "DISEASE",
"SUID" : 182,
"id_original" : "188",
"_foretell_labels" : [ "Disease" ],
"selected" : false
},
"position" : {
"x" : 18.285714285714292,
"y" : -201.0
},
"selected" : false
}, {
"data" : {
"id" : "179",
"shared_name" : "DISEASE OF ANATOMICAL ENTITY",
"classID" : "DOID_7",
"name" : "DISEASE OF ANATOMICAL ENTITY",
"SUID" : 179,
"id_original" : "191",
"_foretell_labels" : [ "Disease" ],
"selected" : false
},
"position" : {
"x" : -103.76660535458818,
"y" : -137.18968789232434
},
"selected" : false
}, {
"data" : {
"id" : "176",
"shared_name" : "TAUOPATHY",
"classID" : "DOID_680",
"name" : "TAUOPATHY",
"SUID" : 176,
"id_original" : "194",
"_foretell_labels" : [ "Disease" ],
"selected" : false
},
"position" : {
"x" : 18.285714285714292,
"y" : 134.0
},
"selected" : false
}, {
"data" : {
"id" : "155",
"shared_name" : "ALZHEIMER'S DISEASE",
"classID" : "DOID_10652",
"synonyms" : [ "ALZHEIMERS DEMENTIA", "ALZHEIMER DISEASE" ],
"name" : "ALZHEIMER'S DISEASE",
"SUID" : 155,
"id_original" : "197",
"_foretell_labels" : [ "Disease" ],
"selected" : false
},
"position" : {
"x" : 18.285714285714292,
"y" : 201.0
},
"selected" : false
} ],
"edges" : [ {
"data" : {
"id" : "242",
"source" : "191",
"target" : "188",
"id_original" : "235",
"shared_interaction" : "IS_CHILD",
"source_original" : "155",
"shared_name" : "IS_CHILD",
"source_original" : "223",
"target_original" : "182",
"shared_name" : "IS_CHILD",
"shared_interaction" : "IS_CHILD",
"name" : "IS_CHILD",
"interaction" : "IS_CHILD",
"SUID" : 242,
"id_original" : "200",
"refid" : 903.0,
"selected" : false,
"target_original" : "220"
},
"selected" : false
}, {
"data" : {
"id" : "239",
"source" : "188",
"target" : "182",
"id_original" : "244",
"shared_interaction" : "IS_CHILD",
"source_original" : "182",
"shared_name" : "IS_CHILD",
"source_original" : "220",
"target_original" : "188",
"shared_name" : "IS_CHILD",
"shared_interaction" : "IS_CHILD",
"name" : "IS_CHILD",
"interaction" : "IS_CHILD",
"SUID" : 239,
"id_original" : "221",
"refid" : 6410.0,
"selected" : false,
"target_original" : "214"
},
"selected" : false
}, {
"data" : {
"id" : "236",
"source" : "188",
"target" : "185",
"id_original" : "238",
"shared_interaction" : "IS_CHILD",
"source_original" : "182",
"shared_name" : "IS_CHILD",
"source_original" : "220",
"target_original" : "185",
"shared_name" : "IS_CHILD",
"shared_interaction" : "IS_CHILD",
"name" : "IS_CHILD",
"interaction" : "IS_CHILD",
"SUID" : 236,
"id_original" : "224",
"refid" : 6409.0,
"selected" : false,
"target_original" : "217"
},
"selected" : false
}, {
"data" : {
"id" : "233",
"source" : "185",
"target" : "182",
"id_original" : "232",
"shared_interaction" : "IS_CHILD",
"source_original" : "185",
"shared_name" : "IS_CHILD",
"source_original" : "217",
"target_original" : "188",
"shared_name" : "IS_CHILD",
"shared_interaction" : "IS_CHILD",
"name" : "IS_CHILD",
"interaction" : "IS_CHILD",
"SUID" : 233,
"id_original" : "227",
"refid" : 6951.0,
"selected" : false,
"target_original" : "214"
},
"selected" : false
}, {
"data" : {
"id" : "230",
"source" : "185",
"target" : "179",
"id_original" : "229",
"shared_interaction" : "IS_CHILD",
"source_original" : "185",
"shared_name" : "IS_CHILD",
"source_original" : "217",
"target_original" : "191",
"shared_name" : "IS_CHILD",
"shared_interaction" : "IS_CHILD",
"name" : "IS_CHILD",
"interaction" : "IS_CHILD",
"SUID" : 230,
"id_original" : "230",
"refid" : 6950.0,
"selected" : false,
"target_original" : "211"
},
"selected" : false
}, {
"data" : {
"id" : "227",
"source" : "179",
"target" : "182",
"id_original" : "226",
"shared_interaction" : "IS_CHILD",
"source_original" : "191",
"shared_name" : "IS_CHILD",
"source_original" : "211",
"target_original" : "188",
"shared_name" : "IS_CHILD",
"shared_interaction" : "IS_CHILD",
"name" : "IS_CHILD",
"interaction" : "IS_CHILD",
"SUID" : 227,
"id_original" : "233",
"refid" : 854.0,
"selected" : false,
"target_original" : "214"
},
"selected" : false
}, {
"data" : {
"id" : "224",
"source" : "176",
"target" : "191",
"id_original" : "241",
"shared_interaction" : "IS_CHILD",
"source_original" : "194",
"shared_name" : "IS_CHILD",
"source_original" : "208",
"target_original" : "155",
"shared_name" : "IS_CHILD",
"shared_interaction" : "IS_CHILD",
"name" : "IS_CHILD",
"interaction" : "IS_CHILD",
"SUID" : 224,
"id_original" : "236",
"refid" : 7129.0,
"selected" : false,
"target_original" : "223"
},
"selected" : false
}, {
"data" : {
"id" : "194",
"source" : "155",
"target" : "176",
"id_original" : "247",
"shared_interaction" : "IS_CHILD",
"source_original" : "197",
"shared_name" : "IS_CHILD",
"source_original" : "184",
"target_original" : "194",
"shared_name" : "IS_CHILD",
"shared_interaction" : "IS_CHILD",
"name" : "IS_CHILD",
"interaction" : "IS_CHILD",
"SUID" : 194,
"id_original" : "239",
"refid" : 14236.0,
"selected" : false,
"target_original" : "208"
},
"selected" : false
} ]
}
}

Glad you found at least a partial answer. I would suggest that you post a bug against the Cytoscape desktop so that we can add something to 3.10 to deal with empty columns. I agree that we definitely need to provide some support for empty columns.
-- scooter

A solution to having two labels simultaneously for a graph in Cytoscape 3 Desktop is to concatenate two columns (containing the desired labels) either by using the CONCATENATE function or the & operator. In the example, the following was done to achieve this:
In the node Table panel, Add a new single column of the type String (see image below)
Using the function builder (see image below), add the function to concatenate the two columns, note that in this case the line break character was used "\n". Either CONCATENATE($name,"\n",$classID) or $name & "\n" & $classID and insert formula for the entire column.
On the Style pannel, under Label select the newly created column, in this case nameANDclassID.
Formula builder (concatenate the two columns):
Cytoscape Node Table with column (formula has been already inserted in entire column):
Alzheimer's Disease Ontology showing Ontological names and classID:

Related

ensure minimum functionality under heavy search load elk

I have a 3x node elk cluster with fillowing node characteristics:
96GB RAM
25GB heap
10x CPU
6TB total indexes size for past 30 days distributed in 1.5TB per node
the filebeat index 50GB per index split across 3x shards
~ total 100GB/day , including the 1xreplica ~ 200GB/day
The thing is that when viewing my log metrics for past 1-7 days everything is fine , but if I open dashboard to see details for past 30 days I cannot do anything else , and if I open kibana from other browser tab is throw error 408 , all CPU's are at 100%
Please, advice if there is any option to ensure single request will not take the 100% from all CPUs in all nodes and I would be able to see from second tab at least the cluster monitoring?
Thanks,
R2D2
Output from GET _cluster/stats?pretty&human
{
"_nodes" : {
"total" : 6,
"successful" : 6,
"failed" : 0
},
"cluster_name" : "elk",
"cluster_uuid" : "CztzsocFTgmKHWI_GnqDvQ",
"timestamp" : 1634106014829,
"status" : "green",
"indices" : {
"count" : 116,
"shards" : {
"total" : 304,
"primaries" : 152,
"replication" : 1.0,
"index" : {
"shards" : {
"min" : 2,
"max" : 6,
"avg" : 2.6206896551724137
},
"primaries" : {
"min" : 1,
"max" : 3,
"avg" : 1.3103448275862069
},
"replication" : {
"min" : 1.0,
"max" : 1.0,
"avg" : 1.0
}
}
},
"docs" : {
"count" : 4672727260,
"deleted" : 1262101
},
"store" : {
"size" : "4tb",
"size_in_bytes" : 4490413194045,
"reserved" : "0b",
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size" : "58.3mb",
"memory_size_in_bytes" : 61151040,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "4.1gb",
"memory_size_in_bytes" : 4406840440,
"total_count" : 2390354,
"hit_count" : 14385,
"miss_count" : 2375969,
"cache_size" : 4760,
"cache_count" : 5724,
"evictions" : 964
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 3404,
"memory" : "60.1mb",
"memory_in_bytes" : 63090152,
"terms_memory" : "29.9mb",
"terms_memory_in_bytes" : 31446928,
"stored_fields_memory" : "3.9mb",
"stored_fields_memory_in_bytes" : 4122896,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "13.8kb",
"norms_memory_in_bytes" : 14208,
"points_memory" : "0b",
"points_memory_in_bytes" : 0,
"doc_values_memory" : "26.2mb",
"doc_values_memory_in_bytes" : 27506120,
"index_writer_memory" : "59.6mb",
"index_writer_memory_in_bytes" : 62543712,
"version_map_memory" : "830b",
"version_map_memory_in_bytes" : 830,
"fixed_bit_set" : "1gb",
"fixed_bit_set_memory_in_bytes" : 1168657528,
"max_unsafe_auto_id_timestamp" : 1634083207166,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "alias",
"count" : 2686,
"index_count" : 79
},
{
"name" : "binary",
"count" : 14,
"index_count" : 3
},
{
"name" : "boolean",
"count" : 8714,
"index_count" : 103
},
{
"name" : "byte",
"count" : 1,
"index_count" : 1
},
{
"name" : "date",
"count" : 11809,
"index_count" : 115
},
{
"name" : "double",
"count" : 2607,
"index_count" : 79
},
{
"name" : "flattened",
"count" : 641,
"index_count" : 80
},
{
"name" : "float",
"count" : 2611,
"index_count" : 88
},
{
"name" : "geo_point",
"count" : 639,
"index_count" : 80
},
{
"name" : "half_float",
"count" : 56,
"index_count" : 14
},
{
"name" : "integer",
"count" : 186,
"index_count" : 11
},
{
"name" : "ip",
"count" : 9888,
"index_count" : 80
},
{
"name" : "keyword",
"count" : 285169,
"index_count" : 113
},
{
"name" : "long",
"count" : 79464,
"index_count" : 109
},
{
"name" : "nested",
"count" : 272,
"index_count" : 95
},
{
"name" : "object",
"count" : 57866,
"index_count" : 114
},
{
"name" : "short",
"count" : 7981,
"index_count" : 80
},
{
"name" : "text",
"count" : 7837,
"index_count" : 104
}
]
},
"analysis" : {
"char_filter_types" : [ ],
"tokenizer_types" : [ ],
"filter_types" : [
{
"name" : "pattern_capture",
"count" : 1,
"index_count" : 1
}
],
"analyzer_types" : [
{
"name" : "custom",
"count" : 1,
"index_count" : 1
}
],
"built_in_char_filters" : [ ],
"built_in_tokenizers" : [
{
"name" : "uax_url_email",
"count" : 1,
"index_count" : 1
}
],
"built_in_filters" : [
{
"name" : "lowercase",
"count" : 1,
"index_count" : 1
},
{
"name" : "unique",
"count" : 1,
"index_count" : 1
}
],
"built_in_analyzers" : [ ]
}
},
"nodes" : {
"count" : {
"total" : 6,
"coordinating_only" : 0,
"data" : 3,
"data_cold" : 3,
"data_content" : 3,
"data_hot" : 3,
"data_warm" : 3,
"ingest" : 3,
"master" : 3,
"ml" : 6,
"remote_cluster_client" : 3,
"transform" : 3,
"voting_only" : 0
},
"versions" : [
"7.10.2"
],
"os" : {
"available_processors" : 36,
"allocated_processors" : 36,
"names" : [
{
"name" : "Linux",
"count" : 6
}
],
"pretty_names" : [
{
"pretty_name" : "CentOS Linux 8",
"count" : 6
}
],
"mem" : {
"total" : "294gb",
"total_in_bytes" : 315680096256,
"free" : "36.5gb",
"free_in_bytes" : 39295123456,
"used" : "257.4gb",
"used_in_bytes" : 276384972800,
"free_percent" : 12,
"used_percent" : 88
}
},
"process" : {
"cpu" : {
"percent" : 3
},
"open_file_descriptors" : {
"min" : 389,
"max" : 1318,
"avg" : 849
}
},
"jvm" : {
"max_uptime" : "4.6d",
"max_uptime_in_millis" : 399502750,
"versions" : [
{
"version" : "15.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "15.0.1+9",
"vm_vendor" : "AdoptOpenJDK",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 6
}
],
"mem" : {
"heap_used" : "30.9gb",
"heap_used_in_bytes" : 33225823136,
"heap_max" : "78gb",
"heap_max_in_bytes" : 83751862272
},
"threads" : 428
},
"fs" : {
"total" : "5.9tb",
"total_in_bytes" : 6521808826368,
"free" : "1.8tb",
"free_in_bytes" : 2028853403648,
"available" : "1.5tb",
"available_in_bytes" : 1697288646656
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"security4" : 6
},
"http_types" : {
"security4" : 6
}
},
"discovery_types" : {
"zen" : 6
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "docker",
"count" : 6
}
],
"ingest" : {
"number_of_pipelines" : 4,
"processor_stats" : {
"conditional" : {
"count" : 307645168,
"failed" : 0,
"current" : 0,
"time" : "18.5s",
"time_in_millis" : 18569
},
"geoip" : {
"count" : 307645168,
"failed" : 0,
"current" : 0,
"time" : "11.3s",
"time_in_millis" : 11315
},
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"rename" : {
"count" : 615290336,
"failed" : 0,
"current" : 0,
"time" : "7.1s",
"time_in_millis" : 7100
},
"script" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
}
}
}
}
}
Output from : GET _cat/nodes?v&h=v,n,m,r,rm,du,dt,cpu,ram.percent,ram.current,heap.percent,search.query_current,heap.current,heap.total,segments.count&s=n:asc
v n m r rm du dt cpu ram.percent ram.current heap.percent search.query_current heap.current segments.count
7.10.2 elk-ingest-data-0 - cdhilrstw 96gb 1.3tb 1.9tb 100 78 74.5gb 46 0 11.7gb 1162
7.10.2 elk-ingest-data-1 - cdhilrstw 96gb 1.5tb 1.9tb 71 77 73.7gb 34 0 8.6gb 1155
7.10.2 elk-ingest-data-2 - cdhilrstw 96gb 1.5tb 1.9tb 68 100 95.9gb 52 0 13gb 1091
7.10.2 elk-master-0 * lm 2gb 568.3mb 9.7gb 53 100 1.9gb 37 0 381.9mb 0
7.10.2 elk-master-1 - lm 2gb 568.3mb 9.7gb 2 68 1.3gb 61 0 628.9mb 0
7.10.2 elk-master-2 - lm 2gb 568.3mb 9.7gb 3 68 1.3gb 55 0 572.6mb 0

Group logs by latest timestamp, by day

Please bear with me as I am very new to Elastic Search. Below is the schema of the data piping into Elastic Search. The "updated_at" field at the very bottom is the "timestamp." A new record of this data is piped into ES every two hours, with an "updated_at" timestamp as well. I would like to write a DSL query that only pulls each record by the maximum (latest) timestamp of each day, but I'm not sure how or where to start. I tried using aggs but didn't get too far.
{
"_index" : "analyticspeoplecountbycompany",
"_type" : "analytics_PeopleCountByCompany",
"_id" : "2hmEZW4Bxxxxx",
"_score" : 1.0,
"_source" : {
"data" : {
"result" : {
"result" : [
{
"EntityName" : "",
"Type" : "analytics_PeopleCountByCompany",
"Value" : null,
"Template" : {
"Company" : "XXX",
"Claimed" : "2",
"Not Claimed" : "49",
"Type" : "analytics_PeopleCountByCompany"
}
},
{
"EntityName" : "",
"Type" : "analytics_PeopleCountByCompany",
"Value" : null,
"Template" : {
"Company" : "YYY",
"Claimed" : "75",
"Not Claimed" : "108",
"Type" : "analytics_PeopleCountByCompany"
}
},
{
"EntityName" : "",
"Type" : "analytics_PeopleCountByCompany",
"Value" : null,
"Template" : {
"Company" : "ZZZ",
"Claimed" : "34",
"Not Claimed" : "92",
"Type" : "analytics_PeopleCountByCompany"
}
},
{
"EntityName" : "",
"Type" : "analytics_PeopleCountByCompany",
"Value" : null,
"Template" : {
"Company" : "AAA",
"Claimed" : "97",
"Not Claimed" : "260",
"Type" : "analytics_PeopleCountByCompany"
}
{
"EntityName" : "",
"Type" : "analytics_PeopleCountByCompany",
"Value" : null,
"Template" : {
"Company" : "BBB",
"Claimed" : "92",
"Not Claimed" : "269",
"Type" : "analytics_PeopleCountByCompany"
}
}
]
},
"type" : "analytics_PeopleCountByCompany",
"description" : "Count of People by Company",
"updated_at" : "2019-11-13T16:06:47.704Z"

where does ES store data

Similar questions were asked before suggestion to hit:
http://127.0.0.1:9200/_nodes/stats/fs?pretty
This shows:
{
"_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"cluster_name" : "elasticsearch",
"nodes" : {
"-MS6L0okT7SwuQQzp64F0Q" : {
"timestamp" : 1568228571281,
"name" : "Bladiblah",
"transport_address" : "127.0.0.1:9300",
"host" : "127.0.0.1",
"ip" : "127.0.0.1:9300",
"roles" : [
"ingest",
"master",
"data"
],
"attributes" : {
"ml.machine_memory" : "274841600000",
"xpack.installed" : "true",
"ml.max_open_jobs" : "20"
},
"fs" : {
"timestamp" : 1568228571281,
"total" : {
"total_in_bytes" : 400052711424,
"free_in_bytes" : 317666811904,
"available_in_bytes" : 317666811904
},
"least_usage_estimate" : {
"path" : "C:\\elasticsearch-7.3.1\\data\\nodes\\0",
"total_in_bytes" : 400052711424,
"available_in_bytes" : 317666963456,
"used_disk_percent" : 20.593723180814195
},
"most_usage_estimate" : {
"path" : "C:\\elasticsearch-7.3.1\\data\\nodes\\0",
"total_in_bytes" : 400052711424,
"available_in_bytes" : 317666963456,
"used_disk_percent" : 20.593723180814195
},
"data" : [
{
"path" : "C:\\elasticsearch-7.3.1\\data\\nodes\\0",
"mount" : "(C:)",
"type" : "NTFS",
"total_in_bytes" : 400052711424,
"free_in_bytes" : 317666811904,
"available_in_bytes" : 317666811904
}
]
}
}
}
}
This shows that quite a bit of memory is used. However, when I go to:
C:\elasticsearch-7.3.1\data\nodes\0
and go to properties the used memory is only a few kilobytes. Am I missing something?

Spring Mongo Template doesn't return data even if exists

I have a below structure. When I run query db.OrganizationAttributeMetaData.find({ "orgid" : 14}).pretty(); using command prompt it returns below data. But when try to fetch it using Spring MongoTemplate using below code it is returning blank data object. Debug log
2017-04-17 13:26:31,661 [http-nio-8080-exec-4] DEBUG
o.s.data.mongodb.core.MongoTemplate - find using query: { "orgid" :
14} fields: { "attrUniqueCode" : 1 , "attributeName" : 1 , "_id" : 1 ,
"metadataVersion" : 1} for class: class
com.debopam.amsapp.model.OrganizationAttributeMetaData in collection:
OrganizationAttributeMetaData
Method
#Override
public Page<OrganizationAttributeMetaData> searchOrgMetadata(int orgidentifier,String orgHierachyIdentifier,String serachtext,Pageable page) {
Query query = Query.query(Criteria.where("orgid").is(orgidentifier));
if(StringUtils.hasText(orgHierachyIdentifier)){
List<String> orgHierarchyList = com.debopam.amsapp.util.StringUtils.delimitedStringToConcatenatedStringListReversed(orgidentifier+"", AppConstants.ORG_DELIMITER);
query.addCriteria(Criteria.where(orgHierachyIdentifier).in(orgHierarchyList));
}
if(StringUtils.hasText(serachtext)){
query.addCriteria(Criteria.where("attributeName").regex(Pattern.compile(serachtext, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE))
.orOperator(Criteria.where("attrUniqueCode").regex(Pattern.compile(serachtext, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE))));
}
query.fields().include("attributeName").include("attrUniqueCode").include("id").include("metadataVersion");
//query.with(new Sort(Sort.Direction.ASC, "attrUniqueCode").and(new Sort(Sort.Direction.DESC, "metadataVersion")));
query.with(page);
List<OrganizationAttributeMetaData> orgmetadata = mongoTemplate.find(query, OrganizationAttributeMetaData.class);
long total = mongoTemplate.count(query, OrganizationAttributeMetaData.class);
Page<OrganizationAttributeMetaData> orgMetadataPage = new PageImpl<OrganizationAttributeMetaData>(orgmetadata, page, /*(null == answers)?0:answers.size()*/total);
return orgMetadataPage;
}
Data
{
"_id" : ObjectId("58efd07fb9eb52cb113f21ac"),
"_class" : "com.debopam.amsapp.model.OrganizationAttributeMetaData",
"attributeName" : "ISU",
"attrUniqueCode" : "ISU",
"metadataVersion" : 1,
"htmlAttributes" : {
"0" : {
"isuname" : {
"class" : "com.debopam.amsapp.model.html.HTMLInputTag",
"htmlinputclasstype" : "htmlinputtag",
"serialVersionUID" : NumberLong(1),
"label" : "Name",
"id" : "isuname",
"tagName" : "input",
"inputtype" : "text",
"attributes" : [
{
"class" : "com.debopam.amsapp.model.html.HTMLAttribute",
"serialVersionUID" : NumberLong(1),
"name" : "pattern",
"value" : "",
"valueRef" : null
},
{
"class" : "com.debopam.amsapp.model.html.HTMLAttribute",
"serialVersionUID" : NumberLong(1),
"name" : "readonly",
"value" : "false",
"valueRef" : null
},
{
"class" : "com.debopam.amsapp.model.html.HTMLAttribute",
"serialVersionUID" : NumberLong(1),
"name" : "id",
"value" : "isuname",
"valueRef" : null
},
{
"class" : "com.debopam.amsapp.model.html.HTMLAttribute",
"serialVersionUID" : NumberLong(1),
"name" : "placeholder",
"value" : "ISU Name",
"valueRef" : null
},
{
"class" : "com.debopam.amsapp.model.html.HTMLAttribute",
"serialVersionUID" : NumberLong(1),
"name" : "required",
"value" : "true",
"valueRef" : null
},
{
"class" : "com.debopam.amsapp.model.html.HTMLAttribute",
"serialVersionUID" : NumberLong(1),
"name" : "name",
"value" : "isuname",
"valueRef" : null
}
]
}
},
"1" : {
"lob" : {
"class" : "com.debopam.amsapp.model.html.HTMLInputTag",
"htmlinputclasstype" : "htmlinputtag",
"serialVersionUID" : NumberLong(1),
"label" : "LOB",
"id" : "lob",
"tagName" : "select",
"inputtype" : "select",
"attributes" : [
{
"class" : "com.debopam.amsapp.model.html.HTMLAttribute",
"serialVersionUID" : NumberLong(1),
"name" : "name",
"value" : "lob",
"valueRef" : null
},
{
"class" : "com.debopam.amsapp.model.html.HTMLAttribute",
"serialVersionUID" : NumberLong(1),
"name" : "id",
"value" : "lob",
"valueRef" : null
}
],
"nestedTags" : [
{
"class" : "com.debopam.amsapp.model.html.HTMLOptionTag",
"htmlinputclasstype" : "htmloptiontag",
"serialVersionUID" : NumberLong(1),
"attributes" : [
{
"class" : "com.debopam.amsapp.model.html.HTMLAttribute",
"serialVersionUID" : NumberLong(1),
"name" : "Telecom",
"value" : "Telecom",
"valueRef" : null
},
{
"class" : "com.debopam.amsapp.model.html.HTMLAttribute",
"serialVersionUID" : NumberLong(1),
"name" : "Banking",
"value" : "Banking",
"valueRef" : null
},
{
"class" : "com.debopam.amsapp.model.html.HTMLAttribute",
"serialVersionUID" : NumberLong(1),
"name" : "Media & Entertainment",
"value" : "Media & Entertainment",
"valueRef" : null
},
{
"class" : "com.debopam.amsapp.model.html.HTMLAttribute",
"serialVersionUID" : NumberLong(1),
"name" : "Govt.",
"value" : "Govt.",
"valueRef" : null
},
{
"class" : "com.debopam.amsapp.model.html.HTMLAttribute",
"serialVersionUID" : NumberLong(1),
"name" : "Insurance",
"value" : "Insurance",
"valueRef" : null
}
]
}
]
}
}
},
"orgHierachyIdentifier" : "14",
"orgid" : 14,
"createDate" : ISODate("2017-04-13T19:24:23.535Z"),
"lastModifiedDate" : ISODate("2017-04-13T19:24:23.535Z"),
"createdBy" : "appadmin",
"lastModifiedBy" : "appadmin"
}

ElasticSearch Freezes and Crashes

I have setup a cluster with 3 elasticsearch instances and they are being feed with documents provided by logstash (~165K docs per minute). I have those 3 machines with 16Gb RAM and each instance is started up with 8Gb.
The indexing works quite well, and I'm able to perform all required search operations I was expecting. The thing now is that I want to make it general available, but unfortunately depending on the queries that are executed (i.e. range facets over all indexes) it freezes the entire cluster and ends up in a split-brain state.
I already limited some things like:
indices.memory.index_buffer_size: 30%
indices.memory.min_shard_index_buffer_size: 12mb
indices.memory.min_index_buffer_size: 96mb
indices.fielddata.cache.size: 15%
indices.fielddata.cache.expire: 6h
indices.cache.filter.size: 15%
indices.cache.filter.expire: 6h
My entire config file looks like:
index.number_of_shards: 10
index.number_of_replicas: 0
bootstrap.mlockall: true
# Indices settings
indices.memory.index_buffer_size: 30%
indices.memory.min_shard_index_buffer_size: 12mb
indices.memory.min_index_buffer_size: 96mb
# Cache Sizes
indices.fielddata.cache.size: 15%
indices.fielddata.cache.expire: 6h
indices.cache.filter.size: 15%
indices.cache.filter.expire: 6h
# Indexing Settings for Writes
index.refresh_interval: 30s
index.translog.flush_threshold_ops: 50000
Is there something else I could improve here to avoid such freeze and split brain state?
Output of my nodes info:
{
"cluster_name" : "elasticsearch",
"nodes" : {
"7i5sZj_jT_qe6HNESfzO3A" : {
"name" : "Captain Fate",
"transport_address" : "inet[/192.168.0.83:9300]",
"host" : "esserver02",
"ip" : "192.168.0.83",
"version" : "1.1.1",
"build" : "f1585f0",
"http_address" : "inet[/192.168.0.83:9200]",
"settings" : {
"index" : {
"number_of_replicas" : "0",
"translog" : {
"flush_threshold_ops" : "50000"
},
"number_of_shards" : "40",
"refresh_interval" : "30s"
},
"bootstrap" : {
"mlockall" : "true"
},
"transport" : {
"tcp" : {
"port" : "9300"
}
},
"http" : {
"port" : "9200"
},
"name" : "Captain Fate",
"path" : {
"logs" : "/opt/as/es/logs",
"home" : "/opt/as/es"
},
"cluster" : {
"name" : "elasticsearch"
},
"indices" : {
"memory" : {
"index_buffer_size" : "50%",
"min_shard_index_buffer_size" : "12mb",
"min_index_buffer_size" : "96mb"
}
},
"discovery" : {
"zen" : {
"minimum_master_nodes" : "1",
"ping" : {
"unicast" : {
"hosts" : [ "esserver02", "esserver03", "esserver04" ]
},
"multicast" : {
"enabled" : "false"
},
"timeout" : "30s"
}
}
}
},
"os" : {
"refresh_interval" : 1000,
"available_processors" : 16
},
"process" : {
"refresh_interval" : 1000,
"id" : 8482,
"max_file_descriptors" : 128000,
"mlockall" : false
},
"jvm" : {
"pid" : 8482,
"version" : "1.7.0_45",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "24.45-b08",
"vm_vendor" : "Oracle Corporation",
"start_time" : 1411976625093,
"mem" : {
"heap_init_in_bytes" : 2147483648,
"heap_max_in_bytes" : 12771524608,
"non_heap_init_in_bytes" : 24313856,
"non_heap_max_in_bytes" : 136314880,
"direct_max_in_bytes" : 12771524608
},
"gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
"memory_pools" : [ "Code Cache", "Par Eden Space", "Par Survivor Space", "CMS Old Gen", "CMS Perm Gen" ]
},
"thread_pool" : {
"generic" : {
"type" : "cached",
"keep_alive" : "30s"
},
"index" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "200"
},
"get" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"snapshot" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"merge" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"suggest" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"bulk" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "50"
},
"optimize" : {
"type" : "fixed",
"min" : 1,
"max" : 1
},
"warmer" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"flush" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"search" : {
"type" : "fixed",
"min" : 48,
"max" : 48,
"queue_size" : "1k"
},
"percolate" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"management" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"refresh" : {
"type" : "scaling",
"min" : 1,
"max" : 8,
"keep_alive" : "5m"
}
},
"network" : {
"refresh_interval" : 5000
},
"transport" : {
"bound_address" : "inet[/0:0:0:0:0:0:0:0:9300]",
"publish_address" : "inet[/192.168.0.83:9300]"
},
"http" : {
"bound_address" : "inet[/0:0:0:0:0:0:0:0:9200]",
"publish_address" : "inet[/192.168.0.83:9200]",
"max_content_length_in_bytes" : 104857600
},
"plugins" : [ {
"name" : "head",
"version" : "NA",
"description" : "No description found.",
"url" : "/_plugin/head/",
"jvm" : false,
"site" : true
} ]
},
"0OaMqY6IR1SYeL6rd6P61Q" : {
"name" : "Blonde Phantom",
"transport_address" : "inet[/192.168.0.100:9300]",
"host" : "esserver03",
"ip" : "192.168.0.100",
"version" : "1.1.1",
"build" : "f1585f0",
"http_address" : "inet[/192.168.0.100:9200]",
"settings" : {
"index" : {
"number_of_replicas" : "0",
"translog" : {
"flush_threshold_ops" : "50000"
},
"number_of_shards" : "40",
"refresh_interval" : "30s"
},
"bootstrap" : {
"mlockall" : "true"
},
"transport" : {
"tcp" : {
"port" : "9300"
}
},
"http" : {
"port" : "9200"
},
"name" : "Blonde Phantom",
"path" : {
"logs" : "/opt/as/es/logs",
"home" : "/opt/as/es"
},
"cluster" : {
"name" : "elasticsearch"
},
"indices" : {
"memory" : {
"index_buffer_size" : "50%",
"min_shard_index_buffer_size" : "12mb",
"min_index_buffer_size" : "96mb"
}
},
"discovery" : {
"zen" : {
"minimum_master_nodes" : "1",
"ping" : {
"unicast" : {
"hosts" : [ "esserver02", "esserver03", "esserver04" ]
},
"multicast" : {
"enabled" : "false"
},
"timeout" : "30s"
}
}
}
},
"os" : {
"refresh_interval" : 1000,
"available_processors" : 16
},
"process" : {
"refresh_interval" : 1000,
"id" : 98772,
"max_file_descriptors" : 128000,
"mlockall" : false
},
"jvm" : {
"pid" : 98772,
"version" : "1.7.0_45",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "24.45-b08",
"vm_vendor" : "Oracle Corporation",
"start_time" : 1414657551806,
"mem" : {
"heap_init_in_bytes" : 2147483648,
"heap_max_in_bytes" : 12771524608,
"non_heap_init_in_bytes" : 24313856,
"non_heap_max_in_bytes" : 136314880,
"direct_max_in_bytes" : 12771524608
},
"gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
"memory_pools" : [ "Code Cache", "Par Eden Space", "Par Survivor Space", "CMS Old Gen", "CMS Perm Gen" ]
},
"thread_pool" : {
"generic" : {
"type" : "cached",
"keep_alive" : "30s"
},
"index" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "200"
},
"get" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"snapshot" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"merge" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"suggest" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"bulk" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "50"
},
"optimize" : {
"type" : "fixed",
"min" : 1,
"max" : 1
},
"warmer" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"flush" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"search" : {
"type" : "fixed",
"min" : 48,
"max" : 48,
"queue_size" : "1k"
},
"percolate" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"management" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"refresh" : {
"type" : "scaling",
"min" : 1,
"max" : 8,
"keep_alive" : "5m"
}
},
"network" : {
"refresh_interval" : 5000,
"primary_interface" : {
"address" : "",
"name" : "",
"mac_address" : ""
}
},
"transport" : {
"bound_address" : "inet[/0:0:0:0:0:0:0:0%0:9300]",
"publish_address" : "inet[/192.168.0.100:9300]"
},
"http" : {
"bound_address" : "inet[/0:0:0:0:0:0:0:0%0:9200]",
"publish_address" : "inet[/192.168.0.100:9200]",
"max_content_length_in_bytes" : 104857600
},
"plugins" : [ {
"name" : "head",
"version" : "NA",
"description" : "No description found.",
"url" : "/_plugin/head/",
"jvm" : false,
"site" : true
} ]
},
"H2h01oNGSuCL0uu8J3SF6w" : {
"name" : "Dakimh the Enchanter",
"transport_address" : "inet[/192.168.0.101:9300]",
"host" : "esserver04",
"ip" : "192.168.0.101",
"version" : "1.1.1",
"build" : "f1585f0",
"http_address" : "inet[/192.168.0.101:9200]",
"settings" : {
"index" : {
"number_of_replicas" : "0",
"translog" : {
"flush_threshold_ops" : "50000"
},
"number_of_shards" : "40",
"refresh_interval" : "30s"
},
"bootstrap" : {
"mlockall" : "true"
},
"transport" : {
"tcp" : {
"port" : "9300"
}
},
"http" : {
"port" : "9200"
},
"name" : "Dakimh the Enchanter",
"path" : {
"logs" : "/opt/as/es/logs",
"home" : "/opt/as/es"
},
"cluster" : {
"name" : "elasticsearch"
},
"indices" : {
"memory" : {
"index_buffer_size" : "50%",
"min_shard_index_buffer_size" : "12mb",
"min_index_buffer_size" : "96mb"
}
},
"discovery" : {
"zen" : {
"minimum_master_nodes" : "1",
"ping" : {
"unicast" : {
"hosts" : [ "esserver02", "esserver03", "esserver04" ]
},
"multicast" : {
"enabled" : "false"
},
"timeout" : "30s"
}
}
}
},
"os" : {
"refresh_interval" : 1000,
"available_processors" : 16
},
"process" : {
"refresh_interval" : 1000,
"id" : 88019,
"max_file_descriptors" : 128000,
"mlockall" : false
},
"jvm" : {
"pid" : 88019,
"version" : "1.7.0_45",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "24.45-b08",
"vm_vendor" : "Oracle Corporation",
"start_time" : 1414657560829,
"mem" : {
"heap_init_in_bytes" : 2147483648,
"heap_max_in_bytes" : 12771524608,
"non_heap_init_in_bytes" : 24313856,
"non_heap_max_in_bytes" : 136314880,
"direct_max_in_bytes" : 12771524608
},
"gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
"memory_pools" : [ "Code Cache", "Par Eden Space", "Par Survivor Space", "CMS Old Gen", "CMS Perm Gen" ]
},
"thread_pool" : {
"generic" : {
"type" : "cached",
"keep_alive" : "30s"
},
"index" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "200"
},
"get" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"snapshot" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"merge" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"suggest" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"bulk" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "50"
},
"optimize" : {
"type" : "fixed",
"min" : 1,
"max" : 1
},
"warmer" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"flush" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"search" : {
"type" : "fixed",
"min" : 48,
"max" : 48,
"queue_size" : "1k"
},
"percolate" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : "1k"
},
"management" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m"
},
"refresh" : {
"type" : "scaling",
"min" : 1,
"max" : 8,
"keep_alive" : "5m"
}
},
"network" : {
"refresh_interval" : 5000,
"primary_interface" : {
"address" : "",
"name" : "",
"mac_address" : ""
}
},
"transport" : {
"bound_address" : "inet[/0:0:0:0:0:0:0:0%0:9300]",
"publish_address" : "inet[/192.168.0.101:9300]"
},
"http" : {
"bound_address" : "inet[/0:0:0:0:0:0:0:0%0:9200]",
"publish_address" : "inet[/192.168.0.101:9200]",
"max_content_length_in_bytes" : 104857600
},
"plugins" : [ {
"name" : "head",
"version" : "NA",
"description" : "No description found.",
"url" : "/_plugin/head/",
"jvm" : false,
"site" : true
} ]
}
}
}
My old configuration:
index.number_of_shards: 40
index.number_of_replicas: 0
bootstrap.mlockall: true
## Threadpool Settings ##
# Indices settings
indices.memory.index_buffer_size: 50%
indices.memory.min_shard_index_buffer_size: 12mb
indices.memory.min_index_buffer_size: 96mb
# Indexing Settings for Writes
index.refresh_interval: 30s
index.translog.flush_threshold_ops: 50000
Your indices.fielddata.cache.size is set to 15% only. Why?
These data are used for aggregations/facets so it could be related. You should remove indices.fielddata.cache.expire ASAP : this setting is really not recommended at all, as evictions are really costful and it schedules evictions for fielddata values, even if they are used. Could you give us the result of a node stats API ?
Update 1 :
I see minimum_master_nodes is set to 1, but you say you have 3 nodes. This should be set to 2 according to the equation generally used (number of nodes/2 + 1, documentation)
Update 2 :
Do you still experience split-brains with the updated configuration?
Depending on your cluster ES version (> 1.0) , you might want to tune the fielddata circuit-breaker to prevent the most costly requests to be done : take a look here.

Resources