AWS Cloudformation and Autoscaling keeps placing instances in same AZ - cluster-computing

Question

Hi all I have the following CF that create a RMQ cluster using RMQ autoclustering. It works. However every time I run this all the instances end up in the same AZ! I've verified that the stack variables are correct. The subnets are set correctly as well. It's all create in the correct account. Not sure what else to try. Im wondering if something is incorrect in the VPC that is supplied to me?
{
"AWSTemplateFormatVersion": "2010-09-09",
"Parameters": {
"EnvironmentValue": {
"AllowedValues": [
"Dev",
"Test"
],
"Default": "Dev",
"Description": "What environment is this?",
"Type": "String"
},
"RabbitMQErlangCookie": {
"Description": "The erlang cookie to propagate to all nodes in the cluster",
"Type": "String",
"MinLength": "20",
"MaxLength": "20",
"Default": "TGFBTKPLRTOYFHNVSTWN",
"AllowedPattern": "^[A-Z]*$",
"NoEcho": true
},
"RabbitMQAdminUserID": {
"Description": "The admin user name to create on the RabbitMQ cluster",
"Type": "String",
"MinLength": "5",
"MaxLength": "20",
"Default": "admin",
"AllowedPattern": "[a-zA-Z0-9]*",
"NoEcho": true
},
"RabbitMQAdminPassword": {
"Description": "The admin password for the admin account",
"Type": "String",
"MinLength": "5",
"MaxLength": "20",
"Default": "xxxxxx",
"AllowedPattern": "[a-zA-Z0-9!]*",
"NoEcho": true
},
"InstanceAvailabilityZones" : {
"Description" : "A list of avilability zones in which instances will be launched. ",
"Type" : "CommaDelimitedList",
"Default" : "us-east-1e,us-east-1d"
},
"Environment": {
"Description": "The environment to confgiure (dev, test, stage, prod",
"Type": "String",
"AllowedValues": [
"d",
"t"
],
"Default": "d",
"NoEcho": false
}
},
"Mappings": {
"Environments" : {
"Dev": {
"VPCProtectedApp":"vpc-protected-app",
"VPCProtectedDb":"vpc-protected-db",
"VPCProtectedFe":"vpc-protected-fe",
"ELB": "App-Dev",
"SecurityGroup": "sg-soa-db",
"Identifier": "d",
"Prefix": "Dev",
"RMQELB": "elb-soa-db-rmq-dev",
"RMQELBTargetGroup": "elb-soarmq-target-group-dev",
"RMQSubnets": "subnet-soa-db-1,subnet-soa-db-2",
"RMQSecurityGroup":"sg-soa-db",
"RMQClusterMin": "3",
"RMQClusterMax": "3",
"ConsulELB": "elb-soa-db-cons-dev",
"ConsulSubnets": "subnet-soa-db-1,subnet-soa-db-2",
"ConsulSecurityGroup":"sg-soa-db-cons",
"ConsulClusterMin": "3",
"ConsulClusterMax": "3"
},
"Test": {
"VPCProtectedApp":"vpc-protected-app",
"VPCProtectedDb":"vpc-protected-db",
"VPCProtectedFe":"vpc-protected-fe",
"ELB": "App-Dev",
"SecurityGroup": "sg-soa-db",
"Identifier": "t",
"Prefix": "Test",
"RMQELB": "elb-soa-db-rmq-test",
"RMQELBTargetGroup": "elb-soarmq-target-group-test",
"RMQSubnets": "subnet-soa-db-1,subnet-soa-db-2",
"RMQSecurityGroup":"sg-soa-db",
"RMQClusterMin": "3",
"RMQClusterMax": "3",
"ConsulELB": "elb-soa-db-cons-test",
"ConsulSubnets": "subnet-soa-db-1,subnet-soa-db-2",
"ConsulSecurityGroup":"sg-soa-db-cons",
"ConsulClusterMin": "3",
"ConsulClusterMax": "3"
}
}
},
"Resources": {
"RabbitMQRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": [
"ec2.amazonaws.com"
]
},
"Action": [
"sts:AssumeRole"
]
}
]
},
"Path": "/",
"Policies": [
{
"PolicyName": "root",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"autoscaling:DescribeAutoScalingInstances",
"ec2:DescribeInstances"
],
"Resource": [
"*"
]
},
{
"Effect": "Allow",
"Action": [
"ecs:CreateCluster",
"ecs:DeregisterContainerInstance",
"ecs:DiscoverPollEndpoint",
"ecs:Poll",
"ecs:RegisterContainerInstance",
"ecs:Submit*"
],
"Resource": "*"
},
{
"Effect": "Allow",
"Action": "ec2:DescribeInstances",
"Resource": "*"
},
{
"Effect": "Allow",
"Action": [
"logs:*"
],
"Resource": [
"arn:aws:logs:*:*:*"
]
}
]
}
}
]
}
},
"RabbitMQInstanceProfile": {
"Type": "AWS::IAM::InstanceProfile",
"Properties": {
"Path": "/",
"Roles": [
{
"Ref": "RabbitMQRole"
}
]
}
},
"ELBSOARabbitMQ": {
"Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
"Properties": {
"Name": {"Fn::FindInMap" : ["Environments", {"Ref" : "EnvironmentValue" },"RMQELB"]},
"Scheme": "internet-facing",
"Subnets": [
{
"Fn::ImportValue" : "subnet-soa-db-1"
},
{
"Fn::ImportValue" : "subnet-soa-db-2"
}
],
"SecurityGroups": [
{
"Fn::ImportValue" : "sg-soa-db"
}
]
}
},
"ELBSOARMQListener": {
"Type": "AWS::ElasticLoadBalancingV2::Listener",
"Properties": {
"DefaultActions": [
{
"TargetGroupArn": {
"Ref": "ELBSOARMQTargetGroup"
},
"Type": "forward"
}
],
"LoadBalancerArn": {
"Ref": "ELBSOARabbitMQ"
},
"Port": 80,
"Protocol": "HTTP"
}
},
"ELBSOARMQListenerRule": {
"Type": "AWS::ElasticLoadBalancingV2::ListenerRule",
"Properties": {
"Actions": [
{
"TargetGroupArn": {
"Ref": "ELBSOARMQTargetGroup"
},
"Type": "forward"
}
],
"Conditions": [
{
"Field": "path-pattern",
"Values": [
"/"
]
}
],
"ListenerArn": {
"Ref": "ELBSOARMQListener"
},
"Priority": 1
}
},
"ELBSOARMQTargetGroup": {
"Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
"Properties": {
"TargetType": "instance",
"HealthCheckIntervalSeconds": 30,
"HealthCheckPort": 15672,
"HealthCheckProtocol": "HTTP",
"HealthCheckTimeoutSeconds": 3,
"HealthyThresholdCount": 2,
"Name":{"Fn::FindInMap" : ["Environments", {"Ref" : "EnvironmentValue" },"RMQELBTargetGroup"]},
"Port": 15672,
"Protocol": "HTTP",
"UnhealthyThresholdCount": 2,
"VpcId": {
"Fn::ImportValue" : "vpc-protected-db"
}
}
},
"SOARMQServerGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"DependsOn": "ELBSOARabbitMQ",
"Properties": {
"LaunchConfigurationName": {
"Ref": "SOARMQEc2InstanceLC"
},
"MinSize": "3",
"MaxSize": "5",
"TargetGroupARNs": [
{
"Ref": "ELBSOARMQTargetGroup"
}
],
"Tags": [
{
"ResourceType": "auto-scaling-group",
"ResourceId": "my-asg",
"InstanceName": "rabbitmq",
"PropagateAtLaunch": true,
"Value": "test",
"Key": "environment"
},
{
"ResourceType": "auto-scaling-group",
"ResourceId": "my-asg",
"InstanceName": "rabbitmq",
"PropagateAtLaunch": true,
"Value": "vavd-soa-rmq",
"Key": "Name"
}
],
"AvailabilityZones" : { "Ref" : "InstanceAvailabilityZones" },
"VPCZoneIdentifier": [
{
"Fn::ImportValue": "subnet-soa-db-1"
},
{
"Fn::ImportValue": "subnet-soa-db-2"
}
]
}
},
"SOARMQEc2InstanceLC": {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"DependsOn": "ELBSOARabbitMQ",
"Properties": {
"IamInstanceProfile" : { "Ref" : "RabbitMQInstanceProfile" },
"ImageId": "ami-5e414e24",
"InstanceType": "m1.small",
"KeyName": "soa_dev_us_east_1",
"SecurityGroups": [
{
"Fn::ImportValue" : "sg-soa-db"
}
],
"UserData": {
"Fn::Base64": {
"Fn::Join": [
"",
[
"#!/bin/bash -xe\n",
"sudo su\n",
"exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n",
"echo \"1. Installing yum updates\"\n",
"sudo yum update -y\n",
"sudo yum install wget -y\n",
"sudo yum install socat -y\n",
"yum install -y aws-cfn-bootstrap\n",
"echo \"2. Downloading erlang distro and install\"\n",
"wget https://github.com/rabbitmq/erlang-rpm/releases/download/v20.3.0/erlang-20.3-1.el6.x86_64.rpm\n",
"sudo rpm -ivh erlang-20.3-1.el6.x86_64.rpm\n",
"export EC2_PUBLIC_IP=$(curl http://169.254.169.254/latest/meta-data/public-ipv4)\n",
"echo \"3. Downloading rabbitmq distro and installing\"\n",
"wget http://dl.bintray.com/rabbitmq/all/rabbitmq-server/3.7.4/rabbitmq-server-3.7.4-1.el6.noarch.rpm\n",
"sudo rpm -Uvh rabbitmq-server-3.7.4-1.el6.noarch.rpm\n",
"export RABBITMQ_USE_LONGNAME=true\n",
"echo \"4. Setting the erlang cookie for clustering\"\n",
"sudo sh -c \"echo ''",
{
"Ref": "RabbitMQErlangCookie"
},
"'' > /var/lib/rabbitmq/.erlang.cookie\"\n",
"sudo chown rabbitmq:rabbitmq /var/lib/rabbitmq/.erlang.cookie\n",
"sudo chmod 600 /var/lib/rabbitmq/.erlang.cookie\n",
"echo \"5. Writing the rabbitmq configurations for AWS Autocluster Group peer discovery\"\n",
"sudo cat << EOF > /etc/rabbitmq/rabbitmq.conf\n",
"cluster_formation.peer_discovery_backend = rabbit_peer_discovery_aws\n",
"cluster_formation.aws.region = us-east-1\n",
"cluster_formation.aws.use_autoscaling_group = true\n",
"log.console.level = debug\n",
"log.file.level = debug\n",
"EOF\n",
"echo \"6. Enable the management and peer discovery plugins\"\n",
"sudo rabbitmq-plugins enable rabbitmq_management\n",
"sudo rabbitmq-plugins --offline enable rabbitmq_peer_discovery_aws\n",
"echo \"7. Restart the service - stop the app prior to clustering\"\n",
"sudo service rabbitmq-server restart\n",
"sudo rabbitmqctl stop_app\n",
"sudo rabbitmqctl reset\n",
"echo \"8. Starting the application\"\n",
"sudo rabbitmqctl start_app\n",
"echo \"9. Adding admin user and setting permissions\"\n",
"sudo rabbitmqctl add_user ",
{
"Ref": "RabbitMQAdminUserID"
},
" ",
{
"Ref": "RabbitMQAdminPassword"
},
"\n",
"sudo rabbitmqctl set_user_tags ",
{
"Ref": "RabbitMQAdminUserID"
},
" administrator\n",
"sudo rabbitmqctl set_permissions -p / ",
{
"Ref": "RabbitMQAdminUserID"
},
" \".*\" \".*\" \".*\" \n",
"echo \"10. Configuration complete!\"\n"
]
]
}
}
}
}
}
}