Remove results with same id from Elasticsearch search result - elasticsearch

Question

Let's assume we have a search result with 3 documents. Two of them share a key attribute (product-ID or similar).
Is it possible to remove duplicates from the search result by using Elasticsearch, so that only 2 documents would be returned in that case? I don't want to implement this in application logic as I would still like to use pagination, aggregation, etc. It does not matter which of the two documents with the same id is removed.
Thanks,
Philipp
Edit:
This would be the example in Elasticsearch:
PUT /tmp_pd_articles
{
"mappings": {
"properties": {
"name": { "type": "text" },
"articleNumber": { "type": "keyword" }
}
}
}
PUT /tmp_pd_articles/_doc/1
{
"name": "My Book 1",
"articleNumber": "A9781"
}
PUT /tmp_pd_articles/_doc/2
{
"name": "My Book 1 (with some other title)",
"articleNumber": "A9781"
}
PUT /tmp_pd_articles/_doc/3
{
"name": "My Book 2",
"articleNumber": "A9782"
}
GET /tmp_pd_articles/_search
{
"query": { "match_all": {} }
}
The goal is to write a query that returns only two articles instead of all three:
#1 ("A9781", "My Book 1") OR #2 ("A9781", "My Book 1 (with some other title)") AND
#3 ("A9782", "My Book 2")
This reduction should be applied because #1 and #2 share the same productNumber "A9781". I wonder whether there is a Elasticsearch query to accomplish this goal.

Answer 1

Yes, its possible using top-hits aggregation, please use below query to filter the data., note tested it on your mapping and sample data, and it provides your expected data.
{
"size": 0, --> returns only aggregate data, if you want to include all 3 documents remove this size param.
"aggs": {
"dedup": {
"terms": {
"field": "articleNumber"
},
"aggs": {
"dedup_docs": {
"top_hits": {
"size": 1
}
}
}
}
}
}
And Search result
"aggregations": {
"dedup": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "A9781",
"doc_count": 2,
"dedup_docs": {
"hits": {
"total": {
"value": 2,
"relation": "eq"
},
"max_score": 1.0,
"hits": [
{
"_index": "tmp_pd_articles",
"_type": "_doc",
"_id": "1",
"_score": 1.0,
"_source": {
"name": "My Book 1",
"articleNumber": "A9781"
}
}
]
}
}
},
{
"key": "A9782",
"doc_count": 1,
"dedup_docs": {
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 1.0,
"hits": [
{
"_index": "tmp_pd_articles",
"_type": "_doc",
"_id": "3",
"_score": 1.0,
"_source": {
"name": "My Book 2",
"articleNumber": "A9782"
}
}
]
}
}
}
]
}