script_score not works on ElasticSearch 7 - elasticsearch

I upgraded my ES from 2.1 to 7.10, when I tried to use the same query way to search, it always show
{
"error": {
"root_cause": [
{
"type": "query_shard_exception",
"reason": "script_score: the script could not be loaded",
"index": "designs",
"index_uuid": "jqvLIrY2TIyJU2bdAsGgkQ"
}
],
"type": "search_phase_execution_exception",
"reason": "all shards failed",
"phase": "query",
"grouped": true,
"failed_shards": [
{
"shard": 0,
"index": "designs",
"node": "jkt6cdUBTKOJ_HTA83wgWQ",
"reason": {
"type": "query_shard_exception",
"reason": "script_score: the script could not be loaded",
"index": "designs",
"index_uuid": "jqvLIrY2TIyJU2bdAsGgkQ",
"caused_by": {
"type": "illegal_argument_exception",
"reason": "script_lang not supported [native]"
}
}
}
]
},
"status": 400
}
Old query string:
{
"from": 0,
"size": 20,
"explain": false,
"_source": {
"include": [
"designId",
"memberId",
"isPersonalizable",
"sellerTags",
"products.productId",
"products.productTypeId",
"products.imageOneId",
"products.imageTwoId",
"products.hasPersonalizableSvg",
"products.storeId"
]
},
"sort": [
{
"_score": {}
},
{
"designId": {
"order": "desc"
}
}
],
"query": {
"function_score": {
"functions": [
{
"script_score": {
"script": {
"inline": "combined-conversion-scorer",
"lang": "native",
"params": {
"query": "peanuts",
"productTypeIds": [
128,
134,
755,
96,
113,
1230,
1231
]
}
}
}
}
],
"query": {
"bool": {
"must": [
{
"multi_match": {
"type": "most_fields",
"query": "peanuts",
"minimum_should_match": "3<75%",
"fields": [
"sellerTags",
"sellerTags.shingles",
"sellerTags.stemmed",
"editorialTags",
"editorialTags.shingles",
"editorialTags.stemmed"
]
}
}
],
"filter": [
{
"bool": {
"must": [
{
"nested": {
"query": {
"terms": {
"products.productTypeId": [
128,
134,
755,
96,
113,
1230,
1231
]
}
},
"path": "products"
}
},
{
"nested": {
"query": {
"term": {
"products.locations": {
"value": "US-1"
}
}
},
"path": "products"
}
}
],
"must_not": [
{
"multi_match": {
"query": "some query",
"fields": [
"sellerTags",
"editorialTags"
]
}
}
]
}
}
]
}
},
"boost_mode": "multiply"
}
}
}
New query string:
{
"explain": true,
"from": 0,
"query": {
"function_score": {
"boost_mode": "multiply",
"functions": [
{
"script_score": {
"script": {
"source": "combined-conversion-scorer",
"lang": "native",
"params": {
"query": "peanuts"
}
}
}
}
],
"query": {
"bool": {
"filter": [
{
"bool": {
"must": [
{
"nested": {
"path": "products",
"query": {
"term": {
"products.locations": {
"value": "US-1"
}
}
}
}
}
],
"must_not": [
{
"multi_match": {
"fields": [
"sellerTags",
"editorialTags"
],
"query": "another query"
}
}
]
}
}
],
"must": [
{
"multi_match": {
"fields": [
"sellerTags",
"sellerTags.shingles",
"sellerTags.stemmed",
"editorialTags",
"editorialTags.shingles",
"editorialTags.stemmed"
],
"minimum_should_match": "3<75%",
"query": "peanuts",
"type": "most_fields"
}
}
]
}
}
}
},
"size": 200,
"sort": [
{
"_score": {}
},
{
"designId": {
"order": "desc"
}
}
],
"_source": {
"includes": [
"designId",
"memberId",
"isPersonalizable",
"sellerTags",
"products.productId",
"products.productTypeId",
"products.imageOneId",
"products.imageTwoId",
"products.hasPersonalizableSvg",
"products.storeId"
]
}
}
Actually I'm not sure if combined-conversion-scorer is custom script or not, I can't find it on old version by GET _scripts/combined-conversion-scorer.
How can I update my script to let it works? Thank you

native scripts are actually compiled Java classes bundled as plugins.
So you need to look for a JAR in your plugins folder. If you have the source files then you're good otherwise you might have to decompile it. In the end, you'll need to recompile and rebuild the plugin for your new ES version... or simply reimplement it in Painless.

Related

ElasticSearch DSL: query on custom tags too

I'm new to ES, but already have a basic query that I need to extend.
The query is currently doing a search based on keywords and also on geo-distance.
Now, I have added a custom tag into my index, and I wish to take it into account too.
I wish to filter and score my results based on the imageTags (tag + score)!
PS: I have seen other similar posts, but I can't figure out how to adapt my query (none of my attempts work).
Here is my query:
GET /posts_index/_search
{
"track_total_hits": true,
"from":0,
"size":10,
"_source": [
"id",
"tags",
"taxonomies",
"storesNames",
"geoLocations",
"storesIds",
"imageUri"
],
"query": {
"function_score": {
"query": {
"bool": {
"should": [
{
"distance_feature": {
"field": "geoLocations",
"pivot": "10km",
"origin": [
-71.3,
41.15
]
}
},
{
"distance_feature": {
"field": "creationTime",
"pivot": "14d",
"origin": "now"
}
},
{
"query_string": {
"fields": [
"tags^3",
"taxonomies^5"
],
"query": "",
"fuzziness": "auto"
}
},
]
}
},
"functions": [
{
"script_score": {
"script": {
"source": "Math.sqrt(doc['commentsCount'].value)"
}
}
},
{
"script_score": {
"script": {
"source": "Math.log(2 + doc['likesCount'].value)"
}
}
},
{
"script_score": {
"script": {
"source": "Math.log(2 + doc['viewsCount'].value)"
}
}
}
],
"score_mode": "avg"
}
}
}
Here is one of my attempts to modify it:
{
"query": {
"nested": {
"path": "imageTags",
"score_mode": "sum",
"query": {
"function_score": {
"query": {
"match": {
"imageTags.tag.keyword": "tripod"
}
},
"field_value_factor": {
"field": "imageTags.score",
"factor": 1,
"missing": 0
}
}
}
}
}
}
By example, here is and example of the index:
{
"id": "4a9afd93-62bc-e8b2-29b4-39f5b073336d",
"tags": [
"fashion",
"mode",
"summer"
],
"imageTags": [
{
"score": 0.95150965,
"tag": "four-poster"
},
{
"score": 0.014835004,
"tag": "window"
},
{
"score": 0.014835004,
"tag": "shade"
},
{
"score": 0.009375425,
"tag": "sliding"
},
{
"score": 0.009375425,
"tag": "door"
}
],
"taxonomies": [],
"categories": [],
"qualityScore": 0.0,
"geoLocations": [
{
"lat": 50.4651156,
"lon": 4.865208
}
],
"storesIds": [
"ba9b3f59-50aa-8774-11a7-39f5ad58ae1a"
],
"storesNames": [
"Zara Namur"
],
"creationTime": "2020-06-10T12:48:30.5710000Z",
"updateTime": "2020-06-10T12:48:30.5710000Z",
"imageUri": "https://localhost:44359/cdn/e_76372856-f7a0-49cc-d3d9-39f5ad58ad6d/653d147084637b2af68b39f5b0733359.jpg",
"description": "",
"likesCount": 0,
"viewsCount": 0,
"commentsCount": 0,
"ImageTags": [
{
"score": 0.95150965,
"tag": "four-poster"
},
{
"score": 0.014835004,
"tag": "window"
},
{
"score": 0.014835004,
"tag": "shade"
},
{
"score": 0.009375425,
"tag": "sliding"
},
{
"score": 0.009375425,
"tag": "door"
}
]
}

function_score for query spanning multiple indices / filter not working as expected

I implemented function_score successfully for queries running against one index, but have trouble to apply function_score to a query spanning multiple indices with different fields in ElasticSearch 6.2.4.
This query:
POST /web_document,web_part/_search
{
"from": 0,
"size": 100,
"query": {
"function_score": {
"query": {
"bool": {
"should": [
{
"bool": {
"must": [
{
"multi_match": {
"type": "best_fields",
"query": "sampling probe",
"fields": [
"titleD.autocomplete^5",
"titleE.autocomplete^5"
]
}
}
],
"filter": [
{
"term": {
"_index": {
"value": "web_document"
}
}
}
]
}
},
{
"bool": {
"must": [
{
"multi_match": {
"type": "best_fields",
"query": "sampling probe",
"fields": [
"textD.autocomplete^5",
"textE.autocomplete^5"
]
}
}
],
"filter": [
{
"term": {
"_index": {
"value": "web_part"
}
}
}
]
}
}
]
}
},
"functions": [
{
"filter": {
"bool": {
"must_not": [
{ "exists": { "field": "revDestDate" } },
{ "exists": { "field": "mutationDate" } }
]
}
},
"script_score": {
"script": "100"
}
},
{
"filter": {
"term": {
"_index": "web_part"
}
},
"gauss": {
"mutationDate": {
"origin": "now",
"scale": "90d",
"decay": 0.5
}
},
"weight": 20
},
{
"filter": {
"term": {
"_index": "web_document"
}
},
"gauss": {
"revDestDate": {
"origin": "now",
"scale": "90d",
"decay": 0.5
}
},
"weight": 20
}
],
"score_mode" : "first",
"boost_mode": "sum"
}
}
}
Returns the following error:
{
"error": {
…
"failed_shards": [
{
"index": "web_document",
…
"reason": {
"type": "parsing_exception",
"reason": "unknown field [mutationDate]",
…
}
},
{
"index": "web_part",
…
"reason": {
"type": "parsing_exception",
"reason": "unknown field [revDestDate]",
…
}
}
]
},
"status": 400
}
I also tried to incorporate the function_score in the induvidual query like this,
which returns the same error:
POST /web_document,web_part/_search
{
"from": 0,
"size": 100,
"query": {
"bool": {
"should": [
{
"function_score": {
"query": {
"bool": {
"must": [
{
"multi_match": {
"type": "best_fields",
"query": "sampling probe",
"fields": [
"titleD.autocomplete^5",
"titleE.autocomplete^5"
]
}
}
],
"filter": [
{
"term": {
"_index": {
"value": "web_document"
}
}
}
]
}
},
"functions": [
{
"gauss": {
"revDestDate": {
"origin": "now",
"scale": "90d",
"decay": 0.5
}
}
}
]
}
},
{
"function_score": {
"query": {
"bool": {
"must": [
{
"multi_match": {
"type": "best_fields",
"query": "sampling probe",
"fields": [
"textD.autocomplete^5",
"textE.autocomplete^5"
]
}
}
],
"filter": [
{
"term": {
"_index": {
"value": "web_part"
}
}
}
]
}
},
"functions": [
{
"gauss": {
"mutationDate": {
"origin": "now",
"scale": "90d",
"decay": 0.5
}
}
}
]
}
}
]
}
}
}
According to the comment from Darshan Pratil to this answer to the same problem this has worked in past versions.
How can this be done in elastic search 6.2.4?

Elasticsearch Exclude documents which does not match a criteria

user_profile index has branches mapping with a propery country_iso
{
"_index": "user_profile",
"_type": "user",
"_id": "188",
"_score": 1.0042034,
"_source": {
"user_id": 188,
"phone": "971550000322",
"profile_set_date": "Apr 6, 2017",
"phone_country": "AE",
"branches": [
{
"id": 27,
"brand_id": 20,
"country_iso": "KW",
"city_iso": "KW-02-145162",
"area_id": 33
},
{
"id": 45,
"brand_id": 56,
"country_iso": "AE",
"city_iso": "AE-01-206944",
"area_id": 18
},
{
"id": 46,
"brand_id": 56,
"country_iso": "AE",
"city_iso": "AE-01-206944",
"area_id": 18
}
],
"prog_id": 13,
"email": "971550000322#email.com"
}
}
I need to find the users who has branches mapping with country_iso=AE and not with anything else
must and must_not combination did not work for me
{
"query": {
"bool": {
"must": [
{
"match": {
"prog_id": 13
}
}
],
"should": [
{
"nested": {
"path": [
"branches"
],
"query": {
"query_string": {
"fields": [
"branches.country_iso"
],
"query": "AE"
}
}
}
}
],
"minimum_number_should_match": 1,
"must_not": [
{
"bool": {
"must_not": [
{
"nested": {
"path": [
"branches"
],
"query": {
"query_string": {
"fields": [
"branches.country_iso"
],
"query": "AE"
}
}
}
}
]
}
}
]
}
}
}
You were close. This one worked for me (notice the negated text search in the must_not statement):
{
"query": {
"bool": {
"must": [
{
"match": {
"prog_id": 13
}
},
{
"nested": {
"path": [
"branches"
],
"query": {
"query_string": {
"fields": [
"branches.country_iso"
],
"query": "AE"
}
}
}
}
],
"must_not": [
{
"nested": {
"path": [
"branches"
],
"query": {
"query_string": {
"fields": [
"branches.country_iso"
],
"query": "-AE"
}
}
}
}
]
}
}
}

Can I tell Elasticsearch to give particular field more boost?

I'm using a simple search in Elasticsearch but I would like to give a particular url a boost so it would come up first in the search result. I'm not sure if it's possible?
Here's my mapping.
"hal": {
"properties": {
"label": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
},
"url": {
"type": "string",
"index": "not_analyzed"
},
And here's my query
{
"fields": [ "url","brand"],
"query": {
"bool": {
"must": [{
"terms": {
"brand": ["brand"]
}
},{
"terms": {
"hal.label.raw": ["donald trump"]
}
}]
}
}
}
Now when I search I would get at least 500 results back. However, there's a particular pattern of url that I would like to give it a boost which is
http://www.anything.com/people/* So any url with /people would come up first in the search result. Is this even at all possible in Elasticsearch? Otherwise I would have to get everything and filter in the code instead.
You can add a should clause that will automatically boost any matching results (make sure url is set to type: string and index: not_analyzed):
{
"fields": [
"url",
"brand"
],
"query": {
"bool": {
"must": [
{
"terms": {
"brand": [
"brand"
]
}
},
{
"terms": {
"hal.label.raw": [
"donald trump"
]
}
}
],
"should": [
{
"wildcard": {
"url": "http://www.anything.com/people/*"
}
}
]
}
}
}
You can also specify a specific boost value:
{
"fields": [
"url",
"brand"
],
"query": {
"bool": {
"must": [
{
"terms": {
"brand": [
"brand"
]
}
},
{
"terms": {
"hal.label.raw": [
"donald trump"
]
}
}
],
"should": [
{
"wildcard": {
"url": {
"value": "http://www.anything.com/people/*",
"boost": 1
}
}
}
]
}
}
}
You can have regex query in should clause with high boost. Try the following query
{
"fields": [
"url",
"brand"
],
"query": {
"bool": {
"must": [
{
"terms": {
"brand": [
"brand"
]
}
},
{
"terms": {
"hal.label.raw": [
"donald trump"
]
}
}
],
"should": [
{
"regexp": {
"url": "http://www.anything.com/people/.*",
"boost" : 50
}
}
]
}
}
}

Elasticsearch error - No query registered for [query]

I hope someone can help, I'm getting the following error when I try to query Elasticsearch
No query registered for [query]
The query I'm using is:
{
"query": {
"bool": {
"must": {
"terms": {
"facet_1": [
"1",
"2"
]
},
"function_score": {
"query": {
"multi_match": {
"query": "fat dog food",
"fields": [
"family_name^20",
"parent_categories^2",
"description^0.2",
"product_suffixes^8",
"facet_values^10"
],
"operator": "and",
"type": "best_fields",
"tie_breaker": 0.3
}
},
"functions": [
{
"script_score": {
"script": "_score + ((_score * 0.3) + (log(1 + doc[\"popularity_score\"].value) * 2))"
}
}
],
"score_mode": "sum"
}
},
"must_not": {
"terms": {
"facet_1": [
"8"
]
}
}
}
},
"fields": [
"family_id",
"family_name",
"parent_categories",
"description",
"image",
"url",
"price_from",
"price_to",
"price_from_id",
"price_to_id",
"products_ids",
"popularity_score"
],
"from": 0,
"size": 48,
"sort": {
"_score": "desc"
}
}
I've tried loads of variations on this but cannot quite seem to get there. I'd really appreciate some help.
I've figured it out, to add extras to a must, must_not or should, they must be in an outer array like:
"query": {
"bool": {
"must": [
{
"terms": {
"facet_1": [
"1",
"2"
]
}
},
{
"function_score": {
"query": {
"multi_match": {
"query": "fat food",
"fields": [
"family_name^20",
"parent_categories^2",
"description^0.2",
"product_suffixes^8",
"facet_values^10"
],
"operator": "and",
"type": "best_fields",
"tie_breaker": 0.3
}
},
"functions": [
{
"script_score": {
"script": "_score + ((_score * 0.3) + (log(1 + doc[\"popularity_score\"].value) * 2))"
}
}
],
"score_mode": "sum"
}
}
]
}
},
"fields": [
"family_id",
"family_name",
"parent_categories",
"description",
"image",
"url",
"price_from",
"price_to",
"price_from_id",
"price_to_id",
"products_ids",
"popularity_score"
],
"from": 0,
"size": 48,
"sort": {
"_score": "desc"
}
}
Thanks for look anyway.

Resources