Terraform definition for Jenkins with ECS, EFS, CoreOS

cloud-config.yml

#cloud-config
write-files:
  - path: /etc/conf.d/nfs
    permissions: '0644'
    content: |
      OPTS_RPC_MOUNTD=""

coreos:
  units:
    - name: update-engine.service
      command: stop
    - name: mnt-jenkins.mount
      content: |
        [Mount]
        What=AZ_ZONE.${jenkins_efs_id}.efs.${aws_region}.amazonaws.com:/
        Where=/mnt/jenkins
        Type=nfs
    - name: amazon-ecs-agent.service
      command: start
      runtime: true
      content: |
        [Unit]
        Description=AWS ECS Agent
        Documentation=https://docs.aws.amazon.com/AmazonECS/latest/developerguide/
        Requires=docker.socket
        After=docker.socket
        [Service]
        Environment=ECS_CLUSTER=${ecs_cluster_name}
        Environment=ECS_LOGLEVEL=${ecs_log_level}
        Environment=ECS_VERSION=${ecs_agent_version}
        Restart=on-failure
        RestartSec=30
        RestartPreventExitStatus=5
        SyslogIdentifier=ecs-agent
        ExecStartPre=-/bin/mkdir -p /var/log/ecs /var/ecs-data /etc/ecs
        ExecStartPre=-/usr/bin/docker kill ecs-agent
        ExecStartPre=-/usr/bin/docker rm ecs-agent
        ExecStartPre=/usr/bin/docker pull amazon/amazon-ecs-agent:$${ECS_VERSION}
        ExecStart=/usr/bin/docker run --name ecs-agent \
                                      --volume=/var/run/docker.sock:/var/run/docker.sock \
                                      --volume=/var/log/ecs:/log \
                                      --volume=/var/ecs-data:/data \
                                      --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \
                                      --volume=/run/docker/execdriver/native:/var/lib/docker/execdriver/native:ro \
                                      --publish=127.0.0.1:51678:51678 \
                                      --env=ECS_LOGFILE=/log/ecs-agent.log \
                                      --env=ECS_LOGLEVEL=$${ECS_LOGLEVEL} \
                                      --env=ECS_DATADIR=/data \
                                      --env=ECS_CLUSTER=$${ECS_CLUSTER} \
                                      --env=ECS_AVAILABLE_LOGGING_DRIVERS='["awslogs"]' \
                                      --log-driver=awslogs \
                                      --log-opt awslogs-region=${aws_region} \
                                      --log-opt awslogs-group=${ecs_log_group_name} \
                                      amazon/amazon-ecs-agent:$${ECS_VERSION}
    - name: runcmd.service
      command: start
      content: |
        [Unit]
        Description=command
        [Service]
        Type=oneshot
        ExecStart=/bin/sh -c "AZ_ZONE=$(curl -L http://169.254.169.254/latest/meta-data/placement/availability-zone); sed -i \"s/AZ_ZONE/$AZ_ZONE/\" /etc/systemd/system/mnt-jenkins.mount; systemctl daemon-reload; systemctl restart mnt-jenkins.mount"
        ExecStartPost=/bin/sh -c "/bin/chmod 777 /mnt/jenkins"

jenkins-ci-policy.json

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Action": [
                "ec2:DescribeSpotInstanceRequests",
                "ec2:CancelSpotInstanceRequests",
                "ec2:GetConsoleOutput",
                "ec2:RequestSpotInstances",
                "ec2:RunInstances",
                "ec2:StartInstances",
                "ec2:StopInstances",
                "ec2:TerminateInstances",
                "ec2:CreateTags",
                "ec2:DeleteTags",
                "ec2:DescribeInstances",
                "ec2:DescribeKeyPairs",
                "ec2:DescribeRegions",
                "ec2:DescribeImages",
                "ec2:DescribeAvailabilityZones",
                "ec2:DescribeSecurityGroups",
                "ec2:DescribeSubnets"
            ],
            "Effect": "Allow",
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": "iam:PassRole",
            "Resource": "*"
        }
    ]
}

jenkins_ci_task_definition.json

[
  {
    "cpu": 1024,
    "essential": true,
    "image": "${image_url}",
    "memory": 1024,
    "name": "${container_name}",
    "portMappings": [
      {
        "containerPort": 8080,
        "hostPort": 8080
      },
      {
        "containerPort": 5000,
        "hostPort": 5000
      }
    ],
		"mountPoints": [
			{
				"sourceVolume": "jenkins-vol",
				"containerPath": "/var/jenkins_home"
			}
		],
    "logConfiguration": {
        "logDriver": "awslogs",
        "options": {
            "awslogs-group": "${log_group_name}",
            "awslogs-region": "${log_group_region}"
        }
    },
    "environment": [
      { 
        "name": "JAVA_OPTS",
        "value": "-Xmx1024m -XX:PermSize=256m -XX:MaxPermSize=1024m"
      }
    ]
  }
]

main.tf

// main.tf

provider "aws" {
  region = "${var.aws_region}"
}

## EFS
resource "aws_efs_file_system" "jenkins_efs" {
  creation_token = "jenkins-efs"
  tags {
    Name = "jenkins-efs"
  }
}

resource "aws_efs_mount_target" "jenkins-efs-mount-target" {
  count = "${var.aws_az_count}"
  file_system_id = "${aws_efs_file_system.jenkins_efs.id}"
  subnet_id = "${element(aws_subnet.subnet.*.id, count.index)}"
  security_groups = [
    "${aws_security_group.instance_sg.id}"
  ]
}

## EC2

### Network

data "aws_availability_zones" "available" {}

resource "aws_vpc" "vpc" {
  enable_dns_hostnames = true
  cidr_block = "10.10.0.0/16"
}

resource "aws_subnet" "subnet" {
  count = "${var.aws_az_count}"
  cidr_block = "${cidrsubnet(aws_vpc.vpc.cidr_block, 8, count.index)}"
  availability_zone = "${data.aws_availability_zones.available.names[count.index]}"
  vpc_id = "${aws_vpc.vpc.id}"
}

resource "aws_internet_gateway" "gw" {
  vpc_id = "${aws_vpc.vpc.id}"
}

resource "aws_route_table" "lower_r" {
  vpc_id = "${aws_vpc.vpc.id}"
  route {
      cidr_block = "0.0.0.0/0"
      gateway_id = "${aws_internet_gateway.gw.id}"
  }
}

resource "aws_route_table_association" "lower_a" {
  count = "${var.aws_az_count}"
  subnet_id = "${element(aws_subnet.subnet.*.id, count.index)}"
  route_table_id = "${aws_route_table.lower_r.id}"
}

### Compute

resource "aws_autoscaling_group" "app" {
  name = "app-asg"
  vpc_zone_identifier = ["${aws_subnet.subnet.*.id}"]
  min_size = "${var.asg_min}"
  max_size = "${var.asg_max}"
  desired_capacity = "${var.asg_min}"
  launch_configuration = "${aws_launch_configuration.app_launch_config.name}"

  tag {
    key = "Name"
    value = "app-ecs"
    propagate_at_launch = true
  }

  lifecycle {
    create_before_destroy = true
  }

  depends_on = [
    "aws_efs_file_system.jenkins_efs",
    "aws_efs_mount_target.jenkins-efs-mount-target"
  ]
}

## ECS

data "template_file" "jenkins_task_definition" {
  template = "${file("${path.module}/jenkins_ci_task_definition.json")}"
    vars {
        image_url = "jenkins:2.7.3"
        container_name = "jenkins-ci"
        log_group_region = "${var.aws_region}"
        log_group_name = "${aws_cloudwatch_log_group.app.name}"
    }
}

resource "aws_ecs_cluster" "app_ecs_cluster" {
  name = "app_ecs_cluster"
}

resource "aws_ecs_task_definition" "jenkins_ci" {
  family = "jenkins_ci"
  container_definitions = "${data.template_file.jenkins_task_definition.rendered}"

  volume {
    name = "jenkins-vol"
    host_path = "/mnt/jenkins"
  }
}

resource "aws_ecs_service" "jenkins_ci_ecs_service" {
  name = "jenkins_ci_service"
  cluster = "${aws_ecs_cluster.app_ecs_cluster.id}"
  task_definition = "${aws_ecs_task_definition.jenkins_ci.arn}"
  desired_count = 1
  iam_role = "${aws_iam_role.ecs_service.name}"

  load_balancer {
    target_group_arn = "${aws_alb_target_group.jenkins_ci_alb_tg.id}"
    container_name = "jenkins-ci"
    container_port = "8080"
  }
}


# ALB

resource "aws_alb" "jenkins_ci_alb" {
  name            = "jenkins-ci-ecs-alb"
  subnets         = ["${aws_subnet.subnet.*.id}"]
  security_groups = ["${aws_security_group.lb_sg.id}"]
}

resource "aws_alb_target_group" "jenkins_ci_alb_tg" {
  name = "jenkins-ci-ecs-alb-tg"
  port = 8080
  protocol = "HTTP"
  vpc_id = "${aws_vpc.vpc.id}"

  health_check {
    path = "/login"
    healthy_threshold = 2
    port = 8080
  }
}

resource "aws_alb_listener" "jenkins_ci_alb_listener" {
  load_balancer_arn = "${aws_alb.jenkins_ci_alb.id}"
  port = "80"
  protocol = "HTTP"

  default_action {
    target_group_arn = "${aws_alb_target_group.jenkins_ci_alb_tg.id}"
    type = "forward"
  }
}

data "template_file" "jenkins_ci_policy" {
  template = "${file("${path.module}/jenkins-ci-policy.json")}"
  vars {
    app_log_group_arn = "${aws_cloudwatch_log_group.app.arn}"
    ecs_log_group_arn = "${aws_cloudwatch_log_group.ecs.arn}"
  }
}

resource "aws_iam_role_policy" "jenkins_ci" {
    name = "jenkins-ci-policy"
    role = "${aws_iam_role.app_instance.name}"
    policy = "${data.template_file.jenkins_ci_policy.rendered}"
}

# EC2
data "template_file" "cloud_config" {
  template = "${file("${path.module}/cloud-config.yml")}"
 	vars {
 		aws_region = "${var.aws_region}"
 		ecs_cluster_name = "${aws_ecs_cluster.app_ecs_cluster.name}"
 		ecs_log_level = "info"
 		ecs_agent_version = "latest"
 		ecs_log_group_name = "${aws_cloudwatch_log_group.ecs.name}"
    jenkins_efs_id = "${aws_efs_file_system.jenkins_efs.id}"
 	}
}

data "aws_ami" "stable_coreos" {
  most_recent = true
  filter {
    name = "description"
    values = ["CoreOS stable *"]
  }
  filter {
    name = "architecture"
    values = ["x86_64"]
  }
  filter {
    name = "virtualization-type"
    values = ["hvm"]
  }
  owners = ["595879546273"] # CoreOS
}

resource "aws_launch_configuration" "app_launch_config" {
  name_prefix = "app-lc-"
  security_groups = [
    "${aws_security_group.instance_sg.id}"
  ]
  key_name = "${var.my_ec2_key}"
  image_id = "${data.aws_ami.stable_coreos.id}"
  instance_type = "${var.aws_instance_type}"
  iam_instance_profile = "${aws_iam_instance_profile.app.name}"
 	user_data = "${data.template_file.cloud_config.rendered}"

  // note: 'ebs_optimized' only available for certain ec2 instance types
  /* ebs_optimized = true */
  associate_public_ip_address = true
  lifecycle {
    create_before_destroy = true
  }
}

# Security
resource "aws_security_group" "lb_sg" {
  description = "controls access to the application ELB"

  vpc_id = "${aws_vpc.vpc.id}"
  name = "ecs-lbsg"

  ingress {
    protocol = "tcp"
    from_port = 80
    to_port = 80
    cidr_blocks = ["0.0.0.0/0"]
  }

  egress {
    from_port = 0
    to_port = 0
    protocol = "-1"
    cidr_blocks = [
      "0.0.0.0/0"
    ]
  }
}

resource "aws_security_group" "instance_sg" {
  description = "controls direct access to application instances"
  vpc_id = "${aws_vpc.vpc.id}"
  name = "env-instance-sg"

  ingress {
    protocol = "tcp"
    from_port = 2049
    to_port = 2049
    cidr_blocks = ["0.0.0.0/0"]
  }

  ingress {
    protocol = "tcp"
    from_port = 22
    to_port = 22
    cidr_blocks = ["0.0.0.0/0"]
  }
  ingress {
    protocol = "tcp"
    from_port = 8080
    to_port = 8080
    cidr_blocks = ["0.0.0.0/0"]
  }
  
  ingress {
    protocol = "tcp"
    from_port = 5000
    to_port = 5000
    cidr_blocks = ["0.0.0.0/0"]
  }

  egress {
    from_port = 0
    to_port = 0
    protocol = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

# CloudWatch Logs

resource "aws_cloudwatch_log_group" "ecs" {
  name = "ecs/ecs-agent"
}

resource "aws_cloudwatch_log_group" "app" {
  name = "app/app"
}

# IAM
resource "aws_iam_instance_profile" "app" {
  name = "app-instance-profile"
  roles = ["${aws_iam_role.app_instance.name}"]
}

resource "aws_iam_role" "app_instance" {
  name = "app-instance-role"
  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "",
      "Effect": "Allow",
      "Principal": {
        "Service": "ec2.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}
EOF
}

resource "aws_iam_role" "ecs_service" {
  name = "ecs_role"
  assume_role_policy = <<EOF
{
  "Version": "2008-10-17",
  "Statement": [
    {
      "Sid": "",
      "Effect": "Allow",
      "Principal": {
        "Service": "ecs.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}
EOF
}

resource "aws_iam_role_policy" "ecs_service" {
       	name = "ecs_policy"
       	role = "${aws_iam_role.ecs_service.name}"
       	policy = <<EOF
{
       	"Version": "2012-10-17",
       	"Statement": [
       		{
       			"Effect": "Allow",
       			"Action": [
       				"ec2:Describe*",
       				"elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
       				"elasticloadbalancing:DeregisterTargets",
       				"elasticloadbalancing:Describe*",
       				"elasticloadbalancing:RegisterInstancesWithLoadBalancer",
       				"elasticloadbalancing:RegisterTargets"
       			],
       			"Resource": "*"
       		}
       	]
}
EOF
}

variables.tf

// variables.tf

variable "aws_region" {
  default = "us-west-2"
}

variable "aws_key_name" {
  default = "my_ec2_key"
}

variable "aws_az_count" {
  default = 2
}

variable "aws_instance_type" {
  default = "t2.small"
}

variable "asg_min" {
  default = "1"
}

variable "asg_max" {
  default = "1"
}