AWS OpenSearch. Replica reappearing after migrating from hot to warm - elasticsearch

Question

I was having a problem dropping replicas when transitioning to warm nodes and I found out that the replicas need to be dropped while hot because the ultrawarm nodes have read-only S3 backed storage. So, I then added another hot stage called "hot_no_replicas" and I was seeing that in fact the replicas were being dropped as planned, However, after the warm migration, the replicas are reappearing. You can see it clearly here in the screenshot. 000022 still has a replica and is on hot. 000021, being older, has had the replica dropped and is still on hot. But 000020 being older than 000021 and having migrated to warm, now has a replica again. I will paste the logging policy and attach screenshot. Does anyone know why this is happening?
ps, I added another , probably superfluous "number_of_replicas" : 0" to the warm state to try one more time to get them removed before the warm transition.
{
"_id" : "shortterm_logging_policy",
"_version" : 37,
"_seq_no" : 1640996,
"_primary_term" : 1,
"policy" : {
"policy_id" : "shortterm_logging_policy",
"description" : "hot-warm-delete",
"last_updated_time" : 1672416827543,
"schema_version" : 14,
"error_notification" : null,
"default_state" : "hot",
"states" : [
{
"name" : "hot",
"actions" : [
{
"retry" : {
"count" : 3,
"backoff" : "exponential",
"delay" : "1m"
},
"rollover" : {
"min_size" : "1gb",
"min_doc_count" : 6000000,
"min_index_age" : "1d"
}
}
],
"transitions" : [
{
"state_name" : "hot_no_replicas",
"conditions" : {
"min_index_age" : "2d"
}
}
]
},
{
"name" : "hot_no_replicas",
"actions" : [
{
"retry" : {
"count" : 3,
"backoff" : "exponential",
"delay" : "1m"
},
"replica_count" : {
"number_of_replicas" : 0
}
}
],
"transitions" : [
{
"state_name" : "warm",
"conditions" : {
"min_index_age" : "3d"
}
}
]
},
{
"name" : "warm",
"actions" : [
{
"retry" : {
"count" : 3,
"backoff" : "exponential",
"delay" : "1m"
},
"replica_count" : {
"number_of_replicas" : 0
}
},
{
"retry" : {
"count" : 3,
"backoff" : "exponential",
"delay" : "1m"
},
"warm_migration" : { }
}
],
"transitions" : [
{
"state_name" : "delete",
"conditions" : {
"min_index_age" : "7d"
}
}
]
},
{
"name" : "delete",
"actions" : [
{
"retry" : {
"count" : 3,
"backoff" : "exponential",
"delay" : "1m"
},
"delete" : { }
}
],
"transitions" : [ ]
}
],
"ism_template" : [
{
"index_patterns" : [
"filebeat-syslog-*"
],
"priority" : 1,
"last_updated_time" : 1652815954078
},
{
"index_patterns" : [
"filebeat-mapi_logs-*"
],
"priority" : 2,
"last_updated_time" : 1666124989125
},
{
"index_patterns" : [
"filebeat-proftpd-*"
],
"priority" : 3,
"last_updated_time" : 1666126038959
}
]
}
}
Here I have run explain on 000017 which is in the same state as 000020 was.
{
"filebeat-syslog-000017" : {
"aliases" : {
"all_logs" : { },
"filebeat-syslog-write" : {
"is_write_index" : false
}
},
"mappings" : {
"properties" : {
"#timestamp" : {
"type" : "date"
},
"#version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"agent" : {
"properties" : {
"ephemeral_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"hostname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"apenv" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"app" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"beat" : {
"properties" : {
"hostname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"close_inactive" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ecs" : {
"properties" : {
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"host" : {
"properties" : {
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"input" : {
"properties" : {
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"log" : {
"properties" : {
"file" : {
"properties" : {
"path" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"offset" : {
"type" : "long"
}
}
},
"log_category" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"offset" : {
"type" : "long"
},
"prospector" : {
"properties" : {
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"received_at" : {
"type" : "date"
},
"received_from" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"site" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"source" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"syslog_hostname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"syslog_message" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"syslog_pid" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"syslog_program" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"syslog_timestamp" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"tags" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"timezone" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"settings" : {
"index" : {
"mapping" : {
"total_fields" : {
"limit" : "3000"
},
"ignore_malformed" : "true"
},
"refresh_interval" : "30s",
"translog" : {
"flush_threshold_size" : "1024mb"
},
"plugins" : {
"index_state_management" : {
"rollover_alias" : "filebeat-syslog-write"
}
},
"provided_name" : "filebeat-syslog-000017",
"query" : {
"default_field" : "message"
},
"creation_date" : "1672592177394",
"number_of_replicas" : "1",
"uuid" : "o6UaHr1oQaaDMQHDEe0bkA",
"version" : {
"created" : "135248027"
},
"routing" : {
"allocation" : {
"include" : {
"_tier_preference" : "data_hot"
}
}
},
"number_of_shards" : "3",
"merge" : {
"scheduler" : {
"max_thread_count" : "1"
}
}
}
}
}
}