Rescoring and Sorting of documents - sorting

Question

My goal is to write a query which would rescore documents based on value of a field in the document. To achieve this I was using a rescore query and then sorting the results. However, an explain on the query shows me that the sorting of the documents is done based on the previously computed score and not the new one.
I saw the following which explains that I couldn't use rescore and sort together.
"Sometimes we want to show results, where the ordering of the first documents on the page is affected by the additional rules. Unfortunately this cannot be achieved by the rescore functionality. The first idea points to window_size parameter, but this parameter in fact is not connected with the first documents on the result list but with number of results returned on every shard. In addition window_size cannot be less than page size. (If it is less, ElasticSearch silently use page size). Also, one very important thing – rescoring cannot be combined with sorting, because sorting is done after changes introduced by rescoring."
http://elasticsearchserverbook.com/elasticsearch-0-90-using-rescore/
My query is:
{
"query": {
"filtered": {
"query": {
"bool": {
"should": [
{
"constant_score": {
"query": {
"match": {
"question": {
"query": "diabetes"
}
}
},
"boost": 1
}
},
{
"dis_max": {
"queries": [
{
"constant_score": {
"query": {
"match": {
"question": {
"query": "diabetes"
}
}
},
"boost": 0.01
}
},
{
"constant_score": {
"query": {
"match": {
"answer_text": {
"query": "diabetes"
}
}
},
"boost": 0.0001
}
}
]
}
},
{
"dis_max": {
"queries": [
{
"constant_score": {
"query": {
"match_phrase": {
"question_phrase": {
"query": "what is diabetes",
"slop": 0
}
}
},
"boost": 100
}
},
{
"constant_score": {
"query": {
"match_phrase": {
"question_phrase": {
"query": "what is diabetes",
"slop": 1
}
}
},
"boost": 50
}
},
{
"constant_score": {
"query": {
"match_phrase": {
"question_phrase": {
"query": "what is diabetes",
"slop": 2
}
}
},
"boost": 33
}
},
{
"constant_score": {
"query": {
"match_phrase": {
"question_phrase": {
"query": "what is diabetes",
"slop": 3
}
}
},
"boost": 25
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "question_group_four",
"query": "what__is__diabetes"
}
},
"boost": 0.1
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "question_group_five",
"query": "what__is__diabetes"
}
},
"boost": 0.15
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_no_synonyms_20",
"query": "what__is__diabetes"
}
},
"boost": 35
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_no_synonyms_15",
"query": "what__is__diabetes"
}
},
"boost": 25
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_no_synonyms_10",
"query": "what__is__diabetes"
}
},
"boost": 15
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_20",
"query": "what__is__diabetes"
}
},
"boost": 28
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_15",
"query": "what__is__diabetes"
}
},
"boost": 16
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_10",
"query": "what__is__diabetes"
}
},
"boost": 13
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_05",
"query": "what__is__diabetes"
}
},
"boost": 4
}
}
]
}
},
{
"dis_max": {
"queries": [
{
"constant_score": {
"query": {
"query_string": {
"default_field": "question_group_four",
"query": "diabetes"
}
},
"boost": 0.1
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "question_group_five",
"query": "diabetes"
}
},
"boost": 0.15
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_no_synonyms_20",
"query": "diabetes"
}
},
"boost": 35
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_no_synonyms_15",
"query": "diabetes"
}
},
"boost": 25
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_no_synonyms_10",
"query": "diabetes"
}
},
"boost": 15
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_20",
"query": "diabetes"
}
},
"boost": 28
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_15",
"query": "diabetes"
}
},
"boost": 16
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_10",
"query": "diabetes"
}
},
"boost": 13
}
},
{
"constant_score": {
"query": {
"query_string": {
"default_field": "concept_words_05",
"query": "diabetes"
}
},
"boost": 4
}
}
]
}
}
],
"disable_coord": true
}
},
"filter": {
"and": [
{
"term": {
"posted_by_expert": false
}
},
{
"term": {
"tip_question": false
}
},
{
"term": {
"show_in_work_queue": true
}
},
{
"range": {
"verified_answers_count": {
"gt": 0
}
}
}
]
}
}
},
"rescore": {
"window_size": 100,
"query": {
"rescore_query": {
"function_score": {
"functions": [
{
"script_score": {
"script": "_score * _source.concierge_boost"
}
}
]
}
}
}
},
"sort": [
"_score",
{
"count_words_with_high_concepts": {
"order": "asc"
}
},
{
"popularity": {
"order": "desc"
}
},
{
"length": {
"order": "asc"
}
}
],
"fields": [],
"size": 10,
"from": 0
}
Any help highly appreciated !

Answer 1

This is not possible, indeed. But this has been discussed and decided is not worth implementing it at the moment. The discussion on github, though, reveals the difficulty about this - documents need to be sorted, top 100 (in your case) chosen, then a rescore is applied and then they are, again, sorted. I suggest reading the comments in that github issue, especially the ones from simonw. The issue is still open but it doesn't seem it will be implemented soon, if it will at all.
Regarding your sorting after another level of scoring, I understand the need to rescore only few documents, but it seems is not possible. What if you wrap your query in another function_score where you define a script_score function to compute the final score? Something like this:
{
"query": {
"function_score": {
"query": {
.......
},
"functions": [
{
"script_score": {
"script": "doc['concierge_boost'].value"
}
}
]
}
},
"sort": [
"_score",
{
"count_words_with_high_concepts": {
"order": "asc"
}
},
{
"popularity": {
"order": "desc"
}
},
{
"length": {
"order": "asc"
}
}
],
"fields": [],
"size": 10,
"from": 0
}