EC2 + CodePipeline/CodeDeploy - how to make application ready on instance refresh - amazon-ec2

Question

Please forgive me, I am learning DevOps.
I am using codedeploy/codepipeline to deploy a node.js app onto an ec2 instance.
I am using terraform to manage the infrastructure.
The pipeline works great, but...
Question 1: Deploy on Refresh
If I refresh the auto scaling group, then the instances are missing the application. Is this normal?
Question 2: Clean on deploy
As a workaround in step 1, I wrote a script that downloads the latest build from the codebuild s3 bucket, unzips it, and runs the application.
I then added the logic in the ASG launch configuration to run the script.
So far, so good. When I refresh the instance, the app is booted.
However, the problem arises when I try and then trigger a codepipeline deployment.
It fails in the codedeploy step because the project folder is already full.
In my appspec.yml I have already tried to make it clean out the application directory during the ApplicationStop phase, but I have not found any success.
Any tips/guidance would be most welcome.
Code
app_dir/appspec.yml
version: 0.0
os: linux
files:
- source: /
destination: /home/ec2-user/app_dir/
overwrite: true
permissions:
- object: /home/ec2-user/app_dir/
pattern: "**"
owner: ec2-user
group: ec2-user
hooks:
ApplicationStop:
- location: infrastructure/cleanup.sh
timeout: 60
runas: ec2-user
AfterInstall:
- location: infrastructure/install_dependencies.sh
timeout: 180
runas: ec2-user
- location: infrastructure/install_root_dependencies.sh
timeout: 30
runas: root
ApplicationStart:
- location: infrastructure/start.sh
timeout: 45
runas: ec2-user
app_dir/infrastructure/cleanup.sh
#!/bin/bash
export HOME=/home/ec2-user
pm2 stop all
pm2 delete all
rm -rf /home/ec2-user/app_dir
mkdir -p /home/ec2-user/app_dir
sudo chown ec2-user:ec2-user /home/ec2-user/app_dir
cd /home/ec2-user/app_dir
app_dir/infrastructure/install_root_dependencies.sh
#!/bin/bash
export HOME=/home/ec2-user
cd /home/ec2-user/app_dir
npm install -g pm2
app_dir/infrastructure/start.sh
#!/bin/bash
export HOME=/home/ec2-user
mkdir -p /home/ec2-user/Lexstep
. /home/ec2-user/Lexstep/infrastructure/parameter_store/load_ssm_parameters.sh
cd /home/ec2-user/app_dir || exit
sudo chown -R ec2-user:ec2-user /home/ec2-user/app_dir
mkdir -p /home/ec2-user/logs/app
sudo chown -R ec2-user:ec2-user /home/ec2-user/logs/app
sudo chmod a+rwx /home/ec2-user/logs/app
output=$(pm2 ls | awk '{print $2}')
startup_cmd=$(pm2 startup | awk '/^sudo/')
if [[ $output == *"0"* ]]; then
echo -e "\nProcess already defined. Deleting all processes."
pm2 restart ecosystem.config.js --update-env
else
echo -e "\nNo process is running currently. Creating a new process..\n"
if [[ "$DEPLOYMENT_GROUP_NAME" == "lexstep-prod"* ]] || [[ "$DEPLOYMENT_GROUP_NAME" == "lexstep-stage"* ]]; then
pm2 start ecosystem.config.js --env production
else
pm2 start ecosystem.config.js --env development
fi
eval "$startup_cmd"
fi
pm2 save #save current PM2 process list to start upon reboot
codepipeline terraform definitions
resource "aws_codebuild_project" "codebuild_project" {
name = "MY_APP-${var.env}-build-project"
description = "${var.env} Build Project for MY_APP"
build_timeout = "10"
service_role = aws_iam_role.codebuild_role.arn
source {
type = "CODEPIPELINE"
buildspec = "infrastructure/buildspec.yml"
git_clone_depth = 0
git_submodules_config {
fetch_submodules = false
}
}
environment {
compute_type = "BUILD_GENERAL1_SMALL"
image = "aws/codebuild/amazonlinux2-x86_64-standard:3.0"
type = "LINUX_CONTAINER"
image_pull_credentials_type = "CODEBUILD"
privileged_mode = false
}
vpc_config {
vpc_id = aws_vpc.vpc.id
subnets = [
aws_subnet.subnets_app[0].id,
aws_subnet.subnets_app[1].id,
]
security_group_ids = [
aws_security_group.app_sg.id,
]
}
artifacts {
type = "CODEPIPELINE"
artifact_identifier = "BuildArtifact"
}
secondary_artifacts {
type = "S3"
artifact_identifier = "TestArtifacts"
location = aws_s3_bucket.codepipeline_bucket.id
path = "MY_APP-${var.env}-pipeline/TestArtifacts/artifacts.zip"
name = "TestArtifacts"
packaging = "ZIP"
}
logs_config {
cloudwatch_logs {
group_name = local.app_build_log_group_name
stream_name = "MY_APP-${var.env}-build-app-log-stream"
}
}
cache {
type = "LOCAL"
modes = ["LOCAL_DOCKER_LAYER_CACHE", "LOCAL_SOURCE_CACHE"]
}
tags = merge(
var.additional_tags,
{
Name = "MY_APP-${var.env}-build-project"
},
)
}
resource "aws_codebuild_project" "codebuild_test_project" {
name = "MY_APP-${var.env}-build-test-project"
description = "Unit Test Build Project for MY_APP"
build_timeout = "10"
service_role = aws_iam_role.codebuild_test_role.arn
source {
type = "CODEPIPELINE"
buildspec = "infrastructure/buildspec_unit_test.yml"
git_clone_depth = 0
git_submodules_config {
fetch_submodules = false
}
}
environment {
compute_type = "BUILD_GENERAL1_SMALL"
image = "aws/codebuild/amazonlinux2-x86_64-standard:3.0"
type = "LINUX_CONTAINER"
image_pull_credentials_type = "CODEBUILD"
privileged_mode = false
}
vpc_config {
vpc_id = aws_vpc.vpc.id
subnets = [
aws_subnet.subnets_app[0].id,
aws_subnet.subnets_app[1].id,
]
security_group_ids = [
aws_security_group.app_sg.id,
]
}
artifacts {
type = "CODEPIPELINE"
}
logs_config {
cloudwatch_logs {
group_name = local.test_build_log_group_name
stream_name = "MY_APP-${var.env}-build-unit-test-log-stream"
}
}
cache {
type = "LOCAL"
modes = ["LOCAL_DOCKER_LAYER_CACHE", "LOCAL_SOURCE_CACHE"]
}
tags = merge(
var.additional_tags,
{
Name = "MY_APP-${var.env}-build-test-project"
},
)
}
resource "aws_codebuild_project" "codebuild_e2e_test_project" {
name = "MY_APP-${var.env}-build-e2e-test-project"
description = "Integration Test Build Project for MY_APP"
build_timeout = "10"
service_role = aws_iam_role.codebuild_test_role.arn
source {
type = "CODEPIPELINE"
buildspec = "infrastructure/buildspec_e2e_test.yml"
git_clone_depth = 0
git_submodules_config {
fetch_submodules = false
}
}
environment {
compute_type = "BUILD_GENERAL1_SMALL"
image = "aws/codebuild/amazonlinux2-x86_64-standard:3.0"
type = "LINUX_CONTAINER"
image_pull_credentials_type = "CODEBUILD"
privileged_mode = false
}
vpc_config {
vpc_id = aws_vpc.vpc.id
subnets = [
aws_subnet.subnets_app[0].id,
aws_subnet.subnets_app[1].id,
]
security_group_ids = [
aws_security_group.app_sg.id,
]
}
artifacts {
type = "CODEPIPELINE"
}
logs_config {
cloudwatch_logs {
group_name = local.e2e_test_build_log_group_name
stream_name = "MY_APP-${var.env}-build-test-e2e-log-stream"
}
}
cache {
type = "LOCAL"
modes = ["LOCAL_DOCKER_LAYER_CACHE", "LOCAL_SOURCE_CACHE"]
}
tags = merge(
var.additional_tags,
{
Name = "MY_APP-${var.env}-build-test-e2e-project"
},
)
}
resource "aws_codedeploy_app" "codedeploy_app" {
name = "MY_APP-${var.env}-deploy-app"
compute_platform = "Server"
}
resource "aws_codedeploy_deployment_group" "codedeploy_group" {
app_name = aws_codedeploy_app.codedeploy_app.name
deployment_group_name = "MY_APP-${var.env}-deploy-group"
service_role_arn = aws_iam_role.codedeploy_role.arn
autoscaling_groups = [aws_autoscaling_group.app_asg.name]
# deployment_config_name = "CodeDeployDefault.OneAtATime" # possible option: "CodeDeployDefault.AllAtOnce"
# TODO: change this to OneAtATime for production
deployment_config_name = "CodeDeployDefault.AllAtOnce" # possible option: "CodeDeployDefault.AllAtOnce"
deployment_style {
deployment_type = "IN_PLACE"
}
auto_rollback_configuration {
enabled = true
events = ["DEPLOYMENT_FAILURE"]
}
}
resource "aws_codedeploy_deployment_group" "codedeploy_schedule_group" {
app_name = aws_codedeploy_app.codedeploy_app.name
deployment_group_name = "MY_APP-${var.env}-schedule-deploy-group"
service_role_arn = aws_iam_role.codedeploy_role.arn
autoscaling_groups = [aws_autoscaling_group.app_schedule_asg.name]
deployment_config_name = "CodeDeployDefault.OneAtATime" # possible option: "CodeDeployDefault.AllAtOnce"
deployment_style {
deployment_type = "IN_PLACE"
}
auto_rollback_configuration {
enabled = true
events = ["DEPLOYMENT_FAILURE"]
}
}
resource "aws_codepipeline" "codepipeline" {
name = "MY_APP-${var.env}-pipeline"
role_arn = aws_iam_role.codepipeline_role.arn
artifact_store {
location = aws_s3_bucket.codepipeline_bucket.bucket
type = "S3"
}
stage {
name = "Source"
action {
name = "Source"
category = "Source"
owner = "AWS"
provider = "CodeStarSourceConnection"
version = "1"
output_artifacts = ["SourceArtifacts"]
namespace = "SourceVariables"
configuration = {
ConnectionArn = var.connection_arn
FullRepositoryId = "MY_APP/MY_APP-nest"
BranchName = "master" # change branch here "master"
}
}
}
stage {
name = "Build"
action {
name = "Build"
category = "Build"
owner = "AWS"
provider = "CodeBuild"
version = "1"
input_artifacts = ["SourceArtifacts"]
output_artifacts = ["BuildArtifact", "TestArtifacts"]
namespace = "BuildVariables"
configuration = {
ProjectName = aws_codebuild_project.codebuild_project.id
}
}
}
/* stage {
name = "Test"
action {
name = "UnitTest"
category = "Test"
owner = "AWS"
provider = "CodeBuild"
version = "1"
run_order = "1"
input_artifacts = ["TestArtifacts"]
configuration = {
ProjectName = aws_codebuild_project.codebuild_test_project.id
}
}
action {
name = "IntegrationTest"
category = "Test"
owner = "AWS"
provider = "CodeBuild"
version = "1"
run_order = "2"
input_artifacts = ["TestArtifacts"]
configuration = {
ProjectName = aws_codebuild_project.codebuild_e2e_test_project.id
}
}
} */
stage {
name = "Deploy"
dynamic "action" {
# a fake map for a conditional block
for_each = var.deployment_requires_approval ? { name = "ManualApproval" } : {}
content {
name = "ManualApproval"
category = "Approval"
owner = "AWS"
provider = "Manual"
version = "1"
run_order = "1"
configuration = {
NotificationArn = var.sns_notification_arn
}
}
}
action {
name = "Deploy"
category = "Deploy"
owner = "AWS"
provider = "CodeDeploy"
input_artifacts = ["BuildArtifact"]
version = "1"
run_order = "1"
namespace = "DeployVariables"
configuration = {
ApplicationName = aws_codedeploy_app.codedeploy_app.name
DeploymentGroupName = "MY_APP-${var.env}-deploy-group"
}
}
action {
name = "DeploySchedule"
category = "Deploy"
owner = "AWS"
provider = "CodeDeploy"
input_artifacts = ["BuildArtifact"]
version = "1"
run_order = "2"
namespace = "DeployScheduleVariables"
configuration = {
ApplicationName = aws_codedeploy_app.codedeploy_app.name
DeploymentGroupName = "MY_APP-${var.env}-schedule-deploy-group"
}
}
}
tags = merge(
var.additional_tags,
{
Name = "MY_APP-${var.env}-pipeline"
},
)
depends_on = [aws_codebuild_project.codebuild_project]
}
Terraform EC2 definition
#############################
####### EC2 INSTANCES #######
#############################
### APP instance Launch Configuration ###
resource "aws_launch_configuration" "app_lc" {
name_prefix = "app-${var.env}-lc"
image_id = var.app_ec2_ami
instance_type = var.app_instance_type
iam_instance_profile = aws_iam_instance_profile.app_instance_profile.name
key_name = var.key_name
enable_monitoring = true
ebs_optimized = false
security_groups = [aws_security_group.app_sg.id]
root_block_device {
volume_size = 25
volume_type = "gp3"
}
user_data = <<-EOF
#!/bin/bash
sudo yum update -y
sudo yum install git jq -y
amazon-linux-extras install epel -y
#############
# Node.js #
#############
${file("gists/install_node.sh")}
#############
# App #
#############
cat <<-'ENVFILE' | tee /home/ec2-user/.env
export DEPLOYMENT_GROUP_NAME="{var.env}-deploy-group"
export region="eu-west-2"
ENVFILE
source /home/ec2-user/.env
function get_app() {
sudo mkdir -p /home/ec2-user/logs/app
sudo chown -R ec2-user:ec2-user /home/ec2-user/logs
sudo chmod 777 -R /home/ec2-user/logs
sudo chmod +arwx -R /home/ec2-user/logs
echo "get_app: starting"
mkdir -p /home/ec2-user/app_dir
cd /home/ec2-user/app_dir
BUCKET="${aws_s3_bucket.codepipeline_bucket.bucket}"
BUCKET_KEY=`aws s3 ls $BUCKET --recursive | sort | tail -n 1 | awk '{print $4}'`
echo "downloading app.zip"
aws s3 cp s3://$BUCKET/$BUCKET_KEY ./app.zip
echo "unzipping app.zip"
unzip app.zip
rm -rf app.zip
echo "removing app.zip"
sudo chown -R ec2-user:ec2-user /home/ec2-user/Lexstep
echo "installing root dependencies"
bash /home/ec2-user/app_dir/infrastructure/install_root_dependencies.sh
echo "installing root dependencies"
bash /home/ec2-user/app_dir/infrastructure/install_dependencies.sh
echo "STARTING APP"
export NODE_ENV=production
export region=eu-west-2
bash /home/ec2-user/app_dir/infrastructure/start.sh
sudo chown ec2-user:ec2-user /home/ec2-user/.pm2/rpc.sock /home/ec2-user/.pm2/pub.sock
cd /home/ec2-user
echo "FINISHED get_app SCRIPT"
}
mkdir -p /home/ec2-user/logs && touch /home/ec2-user/logs/get_app.log && get_app 2>&1 | tee /home/ec2-user/logs/get_app.log
echo "Finished init of app_lc"
EOF
lifecycle {
create_before_destroy = true
}
}
### APP Autoscaling Group ###
resource "aws_autoscaling_group" "app_asg" {
name = "${var.env}-app"
launch_configuration = aws_launch_configuration.app_lc.name
min_size = 1
max_size = 2
desired_capacity = 2
health_check_type = "EC2"
health_check_grace_period = 240
vpc_zone_identifier = [aws_subnet.subnets_app[0].id, aws_subnet.subnets_app[1].id]
service_linked_role_arn = data.aws_iam_role.aws_service_linked_role.arn
target_group_arns = [aws_lb_target_group.tg.arn]
lifecycle {
create_before_destroy = true
}
tags = concat(
[
{
"key" = "Name"
"value" = "${var.env}-app"
"propagate_at_launch" = true
},
{
"key" = "Project"
"value" = var.additional_tags.Project
"propagate_at_launch" = true
},
{
"key" = "CreatedBy"
"value" = var.additional_tags.CreatedBy
"propagate_at_launch" = true
},
{
"key" = "Environment"
"value" = var.additional_tags.Environment
"propagate_at_launch" = true
},
])
}