What is this undocumented state for ElasticSearch recovery? - elasticsearch

Question

I have an ElasticSearch index that I was using to store a whole bunch of data spread out across 2 data nodes. One of the data nodes accidentally filled up on hard drive space and caused ES to crash. When I cleared the drive (of extra stuff, I didn't touch the ES data folder) and restarted ES node, the index shards didn't recover. It knows there should be shards on this node, but the recovery logic seems to be stalled. Here is the full return from _recovery?human&detailed=true
{
"events_v4": {
"shards": [
{
"id": 3,
"type": "EMPTY_STORE",
"stage": "DONE",
"primary": true,
"start_time": "2018-09-26T17:35:35.275Z",
"start_time_in_millis": 1537983335275,
"stop_time": "2018-09-26T17:35:35.764Z",
"stop_time_in_millis": 1537983335764,
"total_time": "489ms",
"total_time_in_millis": 489,
"source": {},
"target": {
"id": "wu_A6DNPSiqrg6JfzAW28Q",
"host": "10.8.81.41",
"transport_address": "10.8.81.41:9300",
"ip": "10.8.81.41",
"name": "wu_A6DN"
},
"index": {
"size": {
"total": "0b",
"total_in_bytes": 0,
"reused": "0b",
"reused_in_bytes": 0,
"recovered": "0b",
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time": "0s",
"total_time_in_millis": 0,
"source_throttle_time": "-1",
"source_throttle_time_in_millis": 0,
"target_throttle_time": "-1",
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": 0,
"percent": "100.0%",
"total_on_start": 0,
"total_time": "471ms",
"total_time_in_millis": 471
},
"verify_index": {
"check_index_time": "0s",
"check_index_time_in_millis": 0,
"total_time": "0s",
"total_time_in_millis": 0
}
},
{
"id": 7,
"type": "EMPTY_STORE",
"stage": "DONE",
"primary": true,
"start_time": "2018-09-26T17:35:35.242Z",
"start_time_in_millis": 1537983335242,
"stop_time": "2018-09-26T17:35:35.481Z",
"stop_time_in_millis": 1537983335481,
"total_time": "238ms",
"total_time_in_millis": 238,
"source": {},
"target": {
"id": "wu_A6DNPSiqrg6JfzAW28Q",
"host": "10.8.81.41",
"transport_address": "10.8.81.41:9300",
"ip": "10.8.81.41",
"name": "wu_A6DN"
},
"index": {
"size": {
"total": "0b",
"total_in_bytes": 0,
"reused": "0b",
"reused_in_bytes": 0,
"recovered": "0b",
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time": "0s",
"total_time_in_millis": 0,
"source_throttle_time": "-1",
"source_throttle_time_in_millis": 0,
"target_throttle_time": "-1",
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": 0,
"percent": "100.0%",
"total_on_start": 0,
"total_time": "204ms",
"total_time_in_millis": 204
},
"verify_index": {
"check_index_time": "0s",
"check_index_time_in_millis": 0,
"total_time": "0s",
"total_time_in_millis": 0
}
},
{
"id": 1,
"type": "EMPTY_STORE",
"stage": "DONE",
"primary": true,
"start_time": "2018-09-26T17:35:35.261Z",
"start_time_in_millis": 1537983335261,
"stop_time": "2018-09-26T17:35:35.760Z",
"stop_time_in_millis": 1537983335760,
"total_time": "498ms",
"total_time_in_millis": 498,
"source": {},
"target": {
"id": "wu_A6DNPSiqrg6JfzAW28Q",
"host": "10.8.81.41",
"transport_address": "10.8.81.41:9300",
"ip": "10.8.81.41",
"name": "wu_A6DN"
},
"index": {
"size": {
"total": "0b",
"total_in_bytes": 0,
"reused": "0b",
"reused_in_bytes": 0,
"recovered": "0b",
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time": "0s",
"total_time_in_millis": 0,
"source_throttle_time": "-1",
"source_throttle_time_in_millis": 0,
"target_throttle_time": "-1",
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": 0,
"percent": "100.0%",
"total_on_start": 0,
"total_time": "460ms",
"total_time_in_millis": 460
},
"verify_index": {
"check_index_time": "0s",
"check_index_time_in_millis": 0,
"total_time": "0s",
"total_time_in_millis": 0
}
},
{
"id": 5,
"type": "EMPTY_STORE",
"stage": "DONE",
"primary": true,
"start_time": "2018-09-26T17:35:35.290Z",
"start_time_in_millis": 1537983335290,
"stop_time": "2018-09-26T17:35:35.784Z",
"stop_time_in_millis": 1537983335784,
"total_time": "493ms",
"total_time_in_millis": 493,
"source": {},
"target": {
"id": "wu_A6DNPSiqrg6JfzAW28Q",
"host": "10.8.81.41",
"transport_address": "10.8.81.41:9300",
"ip": "10.8.81.41",
"name": "wu_A6DN"
},
"index": {
"size": {
"total": "0b",
"total_in_bytes": 0,
"reused": "0b",
"reused_in_bytes": 0,
"recovered": "0b",
"recovered_in_bytes": 0,
"percent": "0.0%"
},
"files": {
"total": 0,
"reused": 0,
"recovered": 0,
"percent": "0.0%"
},
"total_time": "0s",
"total_time_in_millis": 0,
"source_throttle_time": "-1",
"source_throttle_time_in_millis": 0,
"target_throttle_time": "-1",
"target_throttle_time_in_millis": 0
},
"translog": {
"recovered": 0,
"total": 0,
"percent": "100.0%",
"total_on_start": 0,
"total_time": "465ms",
"total_time_in_millis": 465
},
"verify_index": {
"check_index_time": "0s",
"check_index_time_in_millis": 0,
"total_time": "0s",
"total_time_in_millis": 0
}
}
]
}
}
The recovery type EMPTY_STORE is undocumented and I can't find anything but a single unanswered question in the ES forums about it. Does anyone know if the index is hosed or if I just need to wait a bit? Is there anything I can do to trigger the recovery to advance?
This is ES version 5.4.3