Count and Stats api showing different doc's count in Elasticsearch? - elasticsearch

Question

1)I have an index "data" which has 479427 documents in it. To get the count i had used GET /data/_search i got response like this
"hits": {
"total": 479427,
"max_score": 1,
2)I checked using count api GET /data/_count and response i got is like this
{
"count": 479427,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
}
}
FROM 1 and 2 the count of articles is same .
But when i tried GET /data/_stats command i am getting different counts in get and total fields.
"data": {
"primaries": {
"docs": {
"count": 2407154,
"deleted": 357392
},
"indexing": {
"index_total": 4926760,
"index_time_in_millis": 31181542,
"index_current": 2744,
"index_failed": 260136,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 175,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 4773829,
"time_in_millis": 857412,
"exists_total": 4773829,
"exists_time_in_millis": 857412,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 82217,
"total_time_in_millis": 19552894,
"total_docs": 87321796,
"total_size_in_bytes": 281542406990,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 488983,
"total_auto_throttle_in_bytes": 35543080
},
"refresh": {
"total": 956338,
"total_time_in_millis": 29144660
},
"flush": {
"total": 697,
"total_time_in_millis": 469603
},
"segments": {
"count": 114,
"memory_in_bytes": 7145544,
"terms_memory_in_bytes": 4862940,
"stored_fields_memory_in_bytes": 1037064,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 116928,
"doc_values_memory_in_bytes": 1128612,
"index_writer_memory_in_bytes": 13753207,
"index_writer_max_memory_in_bytes": 548831232,
"version_map_memory_in_bytes": 2182,
"fixed_bit_set_memory_in_bytes": 350648
},
"translog": {
"operations": 19488,
"size_in_bytes": 799972235
},
"suggest": {
"total": 0,
"time_in_millis": 0,
"current": 0
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 0
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 13306
}
},
**"total": {
"docs": { >Here i am getting total docs count 4814308
"count": 4814308,
"deleted": 715540
},**
"store": {
"size_in_bytes": 11910376476,
"throttle_time_in_millis": 0
},
"indexing": {
"index_total": 9590499,
"index_time_in_millis": 61324893,
"index_current": 2744,
"index_failed": 310323,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 175,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 4773829,
"time_in_millis": 857412,
"exists_total": 4773829,
"exists_time_in_millis": 857412,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 21901088,
"query_time_in_millis": 11241895,
"query_current": 0,
"fetch_total": 4578094,
"fetch_time_in_millis": 1774794,
"fetch_current": 0,
"scroll_total": 0,
"scroll_time_in_millis": 0,
"scroll_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 153172,
"total_time_in_millis": 37586865,
"total_docs": 170014671,
"total_size_in_bytes": 542992816504,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 920242,
"total_auto_throttle_in_bytes": 71693630
},
"refresh": {
"total": 1841635,
"total_time_in_millis": 56292736
},
"flush": {
"total": 1343,
"total_time_in_millis": 946306
},
"warmer": {
"current": 0,
"total": 3822250,
"total_time_in_millis": 1098530
},
"query_cache": {
"memory_size_in_bytes": 2706088,
"total_count": 20222398,
"hit_count": 4846746,
"miss_count": 15375652,
"cache_size": 271,
"cache_count": 2267,
"evictions": 1996
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0
},
"percolate": {
"total": 0,
"time_in_millis": 0,
"current": 0,
"memory_size_in_bytes": -1,
"memory_size": "-1b",
"queries": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 229,
"memory_in_bytes": 14245875,
"terms_memory_in_bytes": 9804839,
"stored_fields_memory_in_bytes": 2068360,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 234752,
"doc_values_memory_in_bytes": 2137924,
"index_writer_memory_in_bytes": 25849179,
"index_writer_max_memory_in_bytes": 1097662464,
"version_map_memory_in_bytes": 4364,
"fixed_bit_set_memory_in_bytes": 701360
},
"translog": {
"operations": 39262,
"size_in_bytes": 1616348491
},
"suggest": {
"total": 0,
"time_in_millis": 0,
"current": 0
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 0
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 32764
}
}
}
I even checked the same in curator by giving curator_cli show_indices --verbose which given response like this:
data open 11.1GB 4814308 5 1 2017-05-31T13:00:37Z
Why there is difference ?
Update ----------------------
1)You mean stats API considers each field in nested as a document right?
Note:BTW I am talking about the total.docs.count field in stats api.
2)Where as count API will give the count of documents that are present in the index based on unique id.
So My question is if i want to know how many documents are present inside the index which api gives the right count whether count,get,stats or curator_cli show_indices --verbose(using curator) ?
Thanks

Answer 1

the stats API includes so called nested documents, as fields, which are marked as nested in the mapping are counted as own lucene documents, where as the count API is just counting the top level objects.