Elasticsearch - Missing Field Value For Nested Field - Function Score Query | v7.10.2 - elasticsearch

Question

I have already posted this on the ES group but I got no response and so posted it on SO. Link https://discuss.elastic.co/t/missing-field-value-for-nested-field-function-score-query-v7-10-2/291365
I have been trying for a long time now but the nested field value is always saying missing field value while calculating the score.
Mapping:
{
"doctor_idx" : {
"mappings" : {
"properties" : {
"_class" : {
"type" : "keyword",
"index" : false,
"doc_values" : false
},
"service" : {
"type" : "nested",
"properties" : {
"_class" : {
"type" : "keyword",
"index" : false,
"doc_values" : false
},
"serviceTypeEarliestAvailability" : {
"type" : "nested",
"properties" : {
"_class" : {
"type" : "keyword",
"index" : false,
"doc_values" : false
},
"earliestAvailableDateTimeByType" : {
"type" : "date",
"format" : "date_hour_minute_second"
},
"serviceType" : {
"type" : "text"
},
"servicesMedium" : {
"type" : "keyword"
}
}
}
}
}
}
}
}
}
For simplicity, I have kept just one record. A glimpse of the record
"serviceTypeEarliestAvailability" : [
{
"serviceType" : "Service Type",
"earliestAvailableDateTimeByType" : "2021-12-09T19:39:16",
"servicesMedium" : [
"MED1",
"MED2",
"MED3",
"MED4"
]
}
],
The following query gives: "A document doesn't have a value for a field! Use doc[].size()==0 to check if a document is missing a field!"
I tried using field_value_factor instead of script_score but it's the same problem and it complains about the missing field value.
Query 1: with script_score
GET /doctor_idx/_search
{
"explain": true,
"query": {
"nested": {
"path": "service",
"query": {
"nested": {
"score_mode": "max",
"path": "service.serviceTypeEarliestAvailability",
"query": {
"function_score": {
"query": {
"match_all": {
"boost": 1
}
},
"functions": [
{
"filter": {
"match": {
"service.serviceTypeEarliestAvailability.serviceType": "type" // no complaints about this
}
},
"weight": 10
},
{
"script_score": {
"script": {
"source": "(doc['service.serviceTypeEarliestAvailability.earliestAvailableDateTimeByType'].value.getMillis())"
}
}
}
]
}
}
}
}
}
}
}
Query 2 : With field_value_factor
GET /doctor_idx/_search
{
"explain": true,
"query": {
"nested": {
"path": "service",
"query": {
"nested": {
"score_mode": "max",
"path": "service.serviceTypeEarliestAvailability",
"query": {
"function_score": {
"query": {
"match_all": {
"boost": 1
}
},
"functions": [
{
"filter": {
"match": {
"service.serviceTypeEarliestAvailability.serviceType": "type"
}
},
"weight": 10
},
{
"field_value_factor": {
"field": "service.serviceTypeEarliestAvailability.earliestAvailableDateTimeByType"
}
}
]
}
}
}
}
}
}
}
Error From ES for Query 1:
{
"error" : {
"root_cause" : [
{
"type" : "script_exception",
"reason" : "runtime error",
"script_stack" : [
"org.elasticsearch.index.fielddata.ScriptDocValues$Dates.get(ScriptDocValues.java:160)",
"org.elasticsearch.index.fielddata.ScriptDocValues$Dates.getValue(ScriptDocValues.java:154)",
"(doc['service.serviceTypeEarliestAvailability.earliestAvailableDateTimeByType'].value.getMillis())",
" ^---- HERE"
],
"script" : "(doc['service.serviceTypeEarliestAvailability.earliestAvailableDateTimeByType'].value.getMillis())",
"lang" : "painless",
"position" : {
"offset" : 79,
"start" : 0,
"end" : 98
}
}
],
"type" : "search_phase_execution_exception",
"reason" : "all shards failed",
"phase" : "query",
"grouped" : true,
"failed_shards" : [
{
"shard" : 0,
"index" : "doctor_idx",
"node" : "mvh5k24dQPqM-d7JVeNomQ",
"reason" : {
"type" : "script_exception",
"reason" : "runtime error",
"script_stack" : [
"org.elasticsearch.index.fielddata.ScriptDocValues$Dates.get(ScriptDocValues.java:160)",
"org.elasticsearch.index.fielddata.ScriptDocValues$Dates.getValue(ScriptDocValues.java:154)",
"(doc['service.serviceTypeEarliestAvailability.earliestAvailableDateTimeByType'].value.getMillis())",
" ^---- HERE"
],
"script" : "(doc['service.serviceTypeEarliestAvailability.earliestAvailableDateTimeByType'].value.getMillis())",
"lang" : "painless",
"position" : {
"offset" : 79,
"start" : 0,
"end" : 98
},
"caused_by" : {
"type" : "illegal_state_exception",
"reason" : "A document doesn't have a value for a field! Use doc[<field>].size()==0 to check if a document is missing a field!"
}
}
}
]
},
"status" : 400
}
Error From ES for Query 2:
{
"error" : {
"root_cause" : [
{
"type" : "exception",
"reason" : "Missing value for field [service.serviceTypeEarliestAvailability.earliestAvailableDateTimeByType]"
}
],
"type" : "search_phase_execution_exception",
"reason" : "all shards failed",
"phase" : "query",
"grouped" : true,
"failed_shards" : [
{
"shard" : 0,
"index" : "doctor_idx",
"node" : "mvh5k24dQPqM-d7JVeNomQ",
"reason" : {
"type" : "exception",
"reason" : "Missing value for field [service.serviceTypeEarliestAvailability.earliestAvailableDateTimeByType]"
}
}
]
},
"status" : 500
}
The only record I have in the ES
{
"took" : 32,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "doctor_idx",
"_type" : "_doc",
"_id" : "xM20oH0Bmp1zsT0w8pQe",
"_score" : 1.0,
"_source" : {
"_class" : "com.insig.clinic_apps.services_server.booking.dao.DoctorSchema",
"id" : "xM20oH0Bmp1zsT0w8pQe",
"metadata" : {
"doctorId" : "xhnz2lGvXxelWyVekz82c2v6Srb2",
"fullName" : "Dave Insig Email Test",
"credentials" : [
"MD"
],
"languagesSpoken" : [
"EN",
"NOT_DEFINED"
],
"specialitiesServed" : [ ],
"city" : "VAUGHAN",
"provincesPermitted" : [ ],
"province" : "ON",
"country" : "CANADA"
},
"service" : {
"servicesMedium" : [
"IN_CLINIC",
"VIDEO",
"MESSAGING",
"PHONE"
],
"servicesTypeDuration" : [
{
"serviceType" : "Acne Symptoms",
"duration" : 5,
"servicesMedium" : [
"IN_CLINIC",
"MESSAGING",
"PHONE",
"VIDEO"
]
}
],
"serviceTypeEarliestAvailability" : [
{
"serviceType" : "Acne Symptoms",
"earliestAvailableDateTimeByType" : "2021-12-09T19:39:16",
"servicesMedium" : [
"IN_CLINIC",
"MESSAGING",
"PHONE",
"VIDEO"
]
}
],
"bufferTimeForNextAvailability" : 0
},
"earliestAvailableDateTime" : "2021-12-09T19:39:16",
"patientRating" : 4.384481,
"onTimeRating" : 3.171053
}
}
]
}
}

Answer 1

/!\ Partial solution /!\
Query 1
When I was trying to execute your query I got this error:
{
"caused_by" : {
"type" : "illegal_argument_exception",
"reason" : "dynamic method [org.elasticsearch.script.JodaCompatibleZonedDateTime, toEpochMilli/0] not found"
}
}
So I did a slight change to your query .toInstant().toEpochMilli()
GET /so_custom_score/_search
{
"explain": true,
"query": {
"nested": {
"path": "service",
"query": {
"nested": {
"score_mode": "max",
"path": "service.serviceTypeEarliestAvailability",
"query": {
"function_score": {
"query": {
"match_all": {
"boost": 1
}
},
"functions": [
{
"filter": {
"match": {
"service.serviceTypeEarliestAvailability.serviceType": "type"
}
},
"weight": 10
},
{
"script_score": {
"script": {
"source": """
def availability = doc['service.serviceTypeEarliestAvailability.earliestAvailableDateTimeByType'].value;
return availability.toInstant().toEpochMilli();
"""
}
}
}
]
}
}
}
}
}
}
}

Answer 2

Well, wasted a lot of time here and figured that there was no problem with the queries. Elasticsearch v7.10.2 has an issue when I try to get the explanation of the query.
It works perfectly fine without the explain parameter.
See the at org.elasticsearch.common.lucene.search.function.FieldValueFactorFunction$1.explainScore(FieldValueFactorFunction.java:103) ~[elasticsearch-7.10.2.jar:7.10.2]
PS: One of the comments in the OG question mentioned that the latest Elasticsearch v7.15 works fine even with explain parameter.