Find duplicates by id in two indices in Elasticsearch - elasticsearch

Question

I have several indices for storing my data in week-related index, with template myindex-2022-weekOfYear
How to find all duplicates by id across these indices?
I've tried to used aggregations (based from another questions here)
GET myindex-*/_search
{
"stored_fields": [
"myKey"
],
"size": 100,
"aggs": {
"duplicateNames": {
"terms": {
"field": "myKey",
"min_doc_count": 2
},
"aggs": {
"duplicateDocuments": {
"top_hits": {}
}
}
}
}
}
But it looks like that query is not working properly, as searching a single document by id (from query result) returns only one index, so I assume that min_doc_count is not working as I am expecting.
EDIT:
I see in response:
"genres" : {
"doc_count_error_upper_bound" : 530,
"sum_other_doc_count" : 357290963,
"buckets" : [ ]
}
so probably shard_size is too low (and I cant really increase it, due to es resources)

Answer 1

Tldr;
I could not find why this is not working, but I made a POC which show it is working correctly.
(For a rather smalls dimensions)
Poc
POST _bulk
{"index": {"_index": "74473038-0", "_id": "1"}}
{"data": "some dummy data", "id": 1}
{"index": {"_index": "74473038-1", "_id": "1"}}
{"data": "some dummy data", "id": 1}
{"index": {"_index": "74473038-2", "_id": "1"}}
{"data": "some dummy data", "id": 1}
{"index": {"_index": "74473038-3", "_id": "1"}}
{"data": "some dummy data", "id": 1}
{"index": {"_index": "74473038-0", "_id": "2"}}
{"data": "some dummy data", "id": 2}
{"index": {"_index": "74473038-2", "_id": "2"}}
{"data": "some dummy data", "id": 2}
{"index": {"_index": "74473038-0", "_id": "3"}}
{"data": "some dummy data", "id": 3}
{"index": {"_index": "74473038-1", "_id": "3"}}
{"data": "some dummy data", "id": 3}
{"index": {"_index": "74473038-3", "_id": "3"}}
{"data": "some dummy data", "id": 3}
{"index": {"_index": "74473038-0", "_id": "4"}}
{"data": "some dummy data", "id": 4}
{"index": {"_index": "74473038-2", "_id": "4"}}
{"data": "some dummy data", "id": 4}
{"index": {"_index": "74473038-3", "_id": "4"}}
{"data": "some dummy data", "id": 4}
{"index": {"_index": "74473038-0", "_id": "5"}}
{"data": "some dummy data", "id": 5}
GET 74473038-*/_search
{
"size": 0,
"aggs": {
"duplicateNames": {
"terms": {
"field": "id",
"min_doc_count": 2
},
"aggs": {
"duplicateDocuments": {
"top_hits": {}
}
}
}
}
}
I am getting as expected, document with id : 1, 2, 3, 4.
Omitting 5.
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 4,
"successful": 4,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 13,
"relation": "eq"
},
"max_score": null,
"hits": []
},
"aggregations": {
"duplicateNames": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": 1,
"doc_count": 4,
"duplicateDocuments": {
"hits": {
"total": {
"value": 4,
"relation": "eq"
},
"max_score": 1,
"hits": [
{
"_index": "74473038-0",
"_id": "1",
"_score": 1,
"_source": {
"data": "some dummy data",
"id": 1
}
},
{
"_index": "74473038-1",
"_id": "1",
"_score": 1,
"_source": {
"data": "some dummy data",
"id": 1
}
},
{
"_index": "74473038-3",
"_id": "1",
"_score": 1,
"_source": {
"data": "some dummy data",
"id": 1
}
}
]
}
}
},
{
"key": 3,
"doc_count": 3,
"duplicateDocuments": {
"hits": {
"total": {
"value": 3,
"relation": "eq"
},
"max_score": 1,
"hits": [
{
"_index": "74473038-0",
"_id": "3",
"_score": 1,
"_source": {
"data": "some dummy data",
"id": 3
}
},
{
"_index": "74473038-1",
"_id": "3",
"_score": 1,
"_source": {
"data": "some dummy data",
"id": 3
}
},
{
"_index": "74473038-3",
"_id": "3",
"_score": 1,
"_source": {
"data": "some dummy data",
"id": 3
}
}
]
}
}
},
{
"key": 4,
"doc_count": 3,
"duplicateDocuments": {
"hits": {
"total": {
"value": 3,
"relation": "eq"
},
"max_score": 1,
"hits": [
{
"_index": "74473038-3",
"_id": "4",
"_score": 1,
"_source": {
"data": "some dummy data",
"id": 4
}
},
{
"_index": "74473038-2",
"_id": "4",
"_score": 1,
"_source": {
"data": "some dummy data",
"id": 4
}
},
{
"_index": "74473038-0",
"_id": "4",
"_score": 1,
"_source": {
"data": "some dummy data",
"id": 4
}
}
]
}
}
},
{
"key": 2,
"doc_count": 2,
"duplicateDocuments": {
"hits": {
"total": {
"value": 2,
"relation": "eq"
},
"max_score": 1,
"hits": [
{
"_index": "74473038-2",
"_id": "2",
"_score": 1,
"_source": {
"data": "some dummy data",
"id": 2
}
},
{
"_index": "74473038-0",
"_id": "2",
"_score": 1,
"_source": {
"data": "some dummy data",
"id": 2
}
}
]
}
}
}
]
}
}
}