Elasticsearch AccessDeniedException[/usr/share/elasticsearch/data/nodes] on ECS Fargate - elasticsearch

Question

I am deploying elasticserch 7.16.1 on ECS Fargate using bind mount. The container terminates with following error.
org.elasticsearch.bootstrap.StartupException: ElasticsearchException[failed to bind service]; nested: AccessDeniedException[/usr/share/elasticsearch/data/nodes];
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:170) ~[elasticsearch-7.16.1.jar:7.16.1]
at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:157) ~[elasticsearch-7.16.1.jar:7.16.1]
at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:77) ~[elasticsearch-7.16.1.jar:7.16.1]
at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:112) ~[elasticsearch-cli-7.16.1.jar:7.16.1]
at org.elasticsearch.cli.Command.main(Command.java:77) ~[elasticsearch-cli-7.16.1.jar:7.16.1]
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:122) ~[elasticsearch-7.16.1.jar:7.16.1]
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:80) ~[elasticsearch-7.16.1.jar:7.16.1]
Caused by: org.elasticsearch.ElasticsearchException: failed to bind service
at org.elasticsearch.node.Node.<init>(Node.java:1090) ~[elasticsearch-7.16.1.jar:7.16.1]
at org.elasticsearch.node.Node.<init>(Node.java:309) ~[elasticsearch-7.16.1.jar:7.16.1]
at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:234) ~[elasticsearch-7.16.1.jar:7.16.1]
at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:234) ~[elasticsearch-7.16.1.jar:7.16.1]
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:434) ~[elasticsearch-7.16.1.jar:7.16.1]
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:166) ~[elasticsearch-7.16.1.jar:7.16.1]
Looks like the container does not have write permission on the volume, but the volume is not read-only. The task-definition is as below.
{
"ipcMode": null,
"executionRoleArn": "arn:aws:iam::xxxxxxxxxxxx:role/ecsTaskExecutionRole",
"containerDefinitions": [
{
"dnsSearchDomains": null,
"environmentFiles": null,
"logConfiguration": {
"logDriver": "awslogs",
"secretOptions": null,
"options": {
"awslogs-group": "/ecs/elasticsearch-dev",
"awslogs-region": "us-east-1",
"awslogs-stream-prefix": "ecs"
}
},
"entryPoint": null,
"portMappings": [
{
"hostPort": 9200,
"protocol": "tcp",
"containerPort": 9200
}
],
"command": null,
"linuxParameters": null,
"cpu": 0,
"environment": [
{
"name": "discovery.type",
"value": "single-node"
},
{
"name": "ES_JAVA_OPTS",
"value": "-Xms512m -Xmx512m"
}
],
"resourceRequirements": null,
"ulimits": [
{
"name": "nofile",
"softLimit": 65536,
"hardLimit": 65536
}
],
"dnsServers": null,
"mountPoints": [
{
"readOnly": null,
"containerPath": "/usr/share/elasticsearch/data",
"sourceVolume": "host-data"
}
],
"workingDirectory": null,
"secrets": null,
"dockerSecurityOptions": null,
"memory": null,
"memoryReservation": null,
"volumesFrom": [],
"stopTimeout": null,
"image": "docker.elastic.co/elasticsearch/elasticsearch:7.16.1",
"startTimeout": null,
"firelensConfiguration": null,
"dependsOn": null,
"disableNetworking": null,
"interactive": null,
"healthCheck": null,
"essential": true,
"links": null,
"hostname": null,
"extraHosts": null,
"pseudoTerminal": null,
"user": null,
"readonlyRootFilesystem": null,
"dockerLabels": null,
"systemControls": null,
"privileged": null,
"name": "elasticsearch-dev"
}
],
"placementConstraints": [],
"memory": "8192",
"taskRoleArn": "arn:aws:iam::xxxxxxxxxxxx:role/ecsTaskExecutionRole",
"compatibilities": [
"EC2",
"FARGATE"
],
"taskDefinitionArn": "arn:aws:ecs:us-east-1:xxxxxxxxxxxx:task-definition/elasticsearch-dev:7",
"family": "elasticsearch-dev",
"requiresAttributes": [
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.ecr-auth"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.task-iam-role"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-ecr-pull"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.task-eni"
}
],
"pidMode": null,
"requiresCompatibilities": [
"FARGATE"
],
"networkMode": "awsvpc",
"runtimePlatform": null,
"cpu": "1024",
"revision": 7,
"status": "ACTIVE",
"inferenceAccelerators": null,
"proxyConfiguration": null,
"volumes": [
{
"fsxWindowsFileServerVolumeConfiguration": null,
"efsVolumeConfiguration": null,
"name": "host-data",
"host": {
"sourcePath": null
},
"dockerVolumeConfiguration": null
}
]
}

Answer 1

elasticsearch:7.13.0 has added a USER directive in the Dockerfile that ensures elasticsearch container to be run by elasticsearch user instead of root user. But AWS ECS still mounts volume owned by root user, that's why elasticsearch user can't write to it raising AccessDeniedException.
ECS docs suggests to add a VOLUME directive after USER directive in the Dockerfile to make the volume owned by the non-root user. I ended up with the following Dockerfile to build elasticsearch image myself and push it to ECR to make it work.
FROM docker.elastic.co/elasticsearch/elasticsearch:7.16.1
VOLUME ["/usr/share/elasticsearch/data"]