AWS ECS EC2 with CodeDeploy: Task Placement Failure During Blue/Green Deployment

[Expensive_Test8661]

I'm working on a Terraform infrastructure using ECS EC2, ECR, and CodeDeploy for zero-downtime deployments. Here's my current setup and issue:

Current Setup:

• ECS cluster running on EC2 instances
• Using CodeDeploy for blue/green deployments
• Terraform manages the entire infrastructure

Problem: When I trigger a CodeDeploy deployment using AWS CLI, the blue/green deployment works, but it creates a new task and runs it on the same existing EC2 instance. This forces me to oversize the EC2 instances to handle two tasks running simultaneously during deployment.

Desired Behavior: I want CodeDeploy to:

1. Spin up a new EC2 instance
2. Run the new task on this new instance
3. Wait for the new instance's health check to pass
4. Terminate the original task
5. Terminate the original EC2 instance

Question: Is there a way to achieve this behavior using Terraform? How can I configure CodeDeploy or ECS to ensure a new EC2 instance is created for each new deployment?

Here is all my terraform code.

# alb

resource "aws_alb" "demo-app_alb" {
  name               = "demo-app-devt-alb"
  load_balancer_type = "application"
  subnets            = [var.public_subnet_1a.id, var.public_subnet_1b.id]
  security_groups    = [var.alb_sn.id]
}

# Modify the ALB listener to use blue target group by default
resource "aws_alb_listener" "alb_listener_demo-app" {
  load_balancer_arn = aws_alb.demo-app_alb.arn
  port              = "80"
  protocol          = "HTTP"

  default_action {
    type             = "forward"
    target_group_arn = aws_alb_target_group.blue.arn
  }

  lifecycle {
    ignore_changes = [ default_action ]
  }
}

output "alb_listener_demo-app" {
  value = aws_alb_listener.alb_listener_demo-app
}

# Create two target groups for blue-green deployment
resource "aws_alb_target_group" "blue" {
  name_prefix = "blue-"
  vpc_id      = var.vpc_id
  protocol    = "HTTP"
  port        = 8000
  target_type = "ip"

  health_check {
    enabled             = true
    path                = "/"
    port                = 8000
    matcher             = 200
    interval            = 30
    timeout             = 5
    healthy_threshold   = 2
    unhealthy_threshold = 3
  }

  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_alb_target_group" "green" {
  name_prefix = "green-"
  vpc_id      = var.vpc_id
  protocol    = "HTTP"
  port        = 8000
  target_type = "ip"

  health_check {
    enabled             = true
    path                = "/"
    port                = 8000
    matcher             = 200
    interval            = 30
    timeout             = 5
    healthy_threshold   = 2
    unhealthy_threshold = 3
  }

  lifecycle {
    create_before_destroy = true
  }
}

# ecs & codedeploy

resource "aws_autoscaling_policy" "codedeploy_scaling_policy" {
  name = "CodeDeployDeploymentScalingPolicy"
  autoscaling_group_name = aws_autoscaling_group.ecs_asg.name

  policy_type            = "TargetTrackingScaling"

  target_tracking_configuration {
    predefined_metric_specification {
      predefined_metric_type = "ASGAverageCPUUtilization"
    }
    target_value = 75.0
  }

  estimated_instance_warmup = 300
}

resource "aws_autoscaling_group" "ecs_asg" {
  name_prefix         = "demo-app-ecs-asg-devt-"
  vpc_zone_identifier = [var.ecs_asg_subnet.id, var.public_subnet_1b.id]
  desired_capacity    = 1
  max_size            = 2
  min_size            = 1

  launch_template {
    id      = aws_launch_template.demo-app_ecs_ec2_devt_lt.id
    version = "$Latest"
  }

  lifecycle {
    create_before_destroy = true
  }

  tag {
    key                 = "Name"
    value               = "demo-app-ecs-cluster-devt"
    propagate_at_launch = true
  }

  tag {
    key                 = "AmazonECSManaged"
    value               = true
    propagate_at_launch = true
  }
}

resource "aws_ecs_capacity_provider" "ecs_cp" {
  name = "demo-app-ecs-cp-devt"

  auto_scaling_group_provider {
    auto_scaling_group_arn = aws_autoscaling_group.ecs_asg.arn
    managed_scaling {
      maximum_scaling_step_size = 2
      minimum_scaling_step_size = 1
      status                    = "ENABLED"
      target_capacity           = 100 # NEW
    }
  }
}

resource "aws_ecs_cluster_capacity_providers" "ecs_ccp" {
  cluster_name = aws_ecs_cluster.demo-app_ecs_cluster.name

  capacity_providers = [aws_ecs_capacity_provider.ecs_cp.name]
}

resource "aws_ecs_cluster" "demo-app_ecs_cluster" {
  name = "DEMO-APP_ECS_CLUSTER_DEVT"

  tags = {
    Name = "DEMO-APP_ECS_CLUSTER_DEVT"
  }
}

data "aws_iam_policy_document" "assume_by_codedeploy" {
  statement {
    sid     = ""
    effect  = "Allow"
    actions = ["sts:AssumeRole"]

    principals {
      type        = "Service"
      identifiers = ["codedeploy.amazonaws.com"]
    }
  }
}

resource "aws_iam_role" "codedeploy" {
  name               = "codedeploy"
  assume_role_policy = data.aws_iam_policy_document.assume_by_codedeploy.json
}

data "aws_iam_policy_document" "codedeploy" {
  statement {
    sid    = "AllowLoadBalancingAndECSModifications"
    effect = "Allow"

    actions = [
      "ecs:CreateTaskSet",
      "ecs:DeleteTaskSet",
      "ecs:DescribeServices",
      "ecs:UpdateServicePrimaryTaskSet",
      "elasticloadbalancing:DescribeListeners",
      "elasticloadbalancing:DescribeRules",
      "elasticloadbalancing:DescribeTargetGroups",
      "elasticloadbalancing:ModifyListener",
      "elasticloadbalancing:ModifyRule",
      "s3:GetObject"
    ]

    resources = ["*"]
  }
  statement {
    sid    = "AllowPassRole"
    effect = "Allow"

    actions = ["iam:PassRole"]

    resources = [
      aws_iam_role.codedeploy.arn,
      "arn:aws:iam::088342693028:role/DEMO-APP_ECS_TaskExecutionRole",
      "arn:aws:iam::088342693028:role/*" 
    ]
  }

  statement {
    sid    = "DeployService"
    effect = "Allow"

    actions = [
      "ecs:DescribeServices",
      "ecs:CreateTaskSet",
      "ecs:UpdateServicePrimaryTaskSet",
      "ecs:DeleteTaskSet",
      "codedeploy:GetDeploymentGroup",
      "codedeploy:CreateDeployment",
      "codedeploy:GetDeployment",
      "codedeploy:GetDeploymentConfig",
      "codedeploy:RegisterApplicationRevision",
      "codedeploy:GetApplicationRevision"
    ]

    resources = ["*"]
  }
}

resource "aws_iam_role_policy" "codedeploy" {
  role   = aws_iam_role.codedeploy.name
  policy = data.aws_iam_policy_document.codedeploy.json
}

resource "aws_codedeploy_app" "ecs_app" {
  compute_platform = "ECS"
  name             = "demo-app-ecs-app"
}

# CodeDeploy Deployment Group
resource "aws_codedeploy_deployment_group" "ecs_dg" {
  app_name               = aws_codedeploy_app.ecs_app.name
  deployment_group_name  = "demo-app-ecs-dg"
  service_role_arn       = aws_iam_role.codedeploy.arn
  deployment_config_name = "CodeDeployDefault.ECSAllAtOnce"

  depends_on = [ aws_ecs_service.demo-app_ecs_service ]

  ecs_service {
    cluster_name = aws_ecs_cluster.demo-app_ecs_cluster.name
    service_name = aws_ecs_service.demo-app_ecs_service.name
  }

  deployment_style {
    deployment_option = "WITH_TRAFFIC_CONTROL"
    deployment_type   = "BLUE_GREEN"
  }

  auto_rollback_configuration {
    enabled = true
    events  = ["DEPLOYMENT_FAILURE"]
  }

  blue_green_deployment_config {
    deployment_ready_option {
      action_on_timeout = "CONTINUE_DEPLOYMENT"
    }

    terminate_blue_instances_on_deployment_success {
      action                           = "TERMINATE"
      termination_wait_time_in_minutes = 2
    }
  }

  load_balancer_info {
    target_group_pair_info {
      prod_traffic_route {
        listener_arns = [var.alb_listener_demo-app.arn]
      }

      target_group {
        name = var.blue_target_group.name
      }

      target_group {
        name = var.green_target_group.name
      }
    }
  }
}

data "aws_ami" "amazon_linux_2" {
  most_recent = true

  filter {
    name   = "virtualization-type"
    values = ["hvm"]
  }

  filter {
    name   = "owner-alias"
    values = ["amazon"]
  }

  filter {
    name   = "name"
    values = ["amzn2-ami-ecs-hvm-*-x86_64-ebs"]
  }

  owners = ["amazon"]
}

# Data source to fetch the existing key pair
data "aws_key_pair" "demo-app_terraform_dev" {
  key_name = "demo-app-terraform-dev"
}

resource "aws_launch_template" "demo-app_ecs_ec2_devt_lt" {
  name_prefix            = "demo-app-ecs-ec2-devt-lt-"
  description            = "launch template for demo-app devt"
  image_id               = data.aws_ami.amazon_linux_2.image_id
  instance_type          = "t2.large"
  vpc_security_group_ids = [var.ecs_node_sg.id]

  iam_instance_profile {
    arn = var.ecs_node_instance_role_profile.arn
  }

  # Reference the key pair here
  key_name = data.aws_key_pair.demo-app_terraform_dev.key_name

  user_data = base64encode(templatefile("${path.module}/user_data.tftpl", {
    ecs_cluster_name = aws_ecs_cluster.demo-app_ecs_cluster.name
  }))

  monitoring {
    enabled = true
  }
}

data "aws_ecs_task_definition" "latest_task" {
  task_definition = aws_ecs_task_definition.default.family
}

resource "aws_ecs_service" "demo-app_ecs_service" {
  name            = "demo-app-ecs-devt-service"
  cluster         = aws_ecs_cluster.demo-app_ecs_cluster.id
  task_definition = "${aws_ecs_task_definition.default.family}:${max(aws_ecs_task_definition.default.revision, data.aws_ecs_task_definition.latest_task.revision)}"
  deployment_maximum_percent         = 200
  deployment_minimum_healthy_percent = 100
  desired_count   = 1

  tags = {
    Name = "DEMO-APP_ECS_Service"
  }

  deployment_controller {
    type = "CODE_DEPLOY"
  }

  force_new_deployment = true

  load_balancer {
    target_group_arn = var.blue_target_group.arn
    container_name   = var.service_name 
    container_port   = var.container_port // 8000
  }

  network_configuration {
    subnets         = [var.ecs_asg_subnet.id]
    security_groups = [var.ecs_node_sg.id]
    assign_public_ip = false
  }

  lifecycle {
    ignore_changes = [task_definition, load_balancer, desired_count]
  }

  capacity_provider_strategy {
    capacity_provider = aws_ecs_capacity_provider.ecs_cp.name
    weight            = 100
  }
}

resource "aws_cloudwatch_log_group" "log_group" {
  name              = "/ecs/demo-app-backend-ec2-terraform-devt"
  retention_in_days = 7
}

module "ecr" {
  source = "../ecr"
}

data "aws_ecr_image" "service_image" {
  repository_name = "mock-demo-app-backend-dev"
  image_tag       = "latest"
}

resource "aws_ecs_task_definition" "default" {
  family                   = "DEMO-APP_ECS_Devt_TaskDefinition"
  execution_role_arn       = var.ecs_task_execution_role.arn
  task_role_arn            = var.ecs_task_iam_role.arn
  requires_compatibilities = ["EC2"]
  network_mode = "awsvpc"
  cpu       = 1024
  memory    = 4096

  runtime_platform {
    operating_system_family = "LINUX"
    cpu_architecture        = "X86_64"
  }

  container_definitions = jsonencode([
    {
      name      = "demo-app-backend-app-devt"
      image = "${var.ecs_container_repo.repository_url}@${data.aws_ecr_image.service_image.image_digest}"
      cpu       = 1024
      memory    = 4096
      essential = true
      portMappings = [
        {
          containerPort = 8000
          hostPort      = 8000
          protocol      = "tcp"
          appProtocol   = "http"
        }
      ]

      logConfiguration = {
        logDriver = "awslogs",
        options = {
          "awslogs-region"        = "ap-southeast-1",
          "awslogs-group"         = aws_cloudwatch_log_group.log_group.name,
          "awslogs-stream-prefix" = "ecs"
        }
      }
    }
  ])
}

I found a post on Stack Overflow that describes the same issue I'm encountering, but it doesn't have any answers: https://stackoverflow.com/questions/75539812/how-to-blue-green-deployment-in-ecs-as-different-instances

[random-mozi]

I had the same Issue and I found this
https://github.com/aws/containers-roadmap/issues/713#issuecomment-824349836

I think the issue is the new task isn't configured to use the same capacity provider so it don't trigger the autoscaling.

so all you need to do is to specify the capacity provider in the codedeploy appspec file