CoreOS template for CloudFormation with two EBS disks on /var/lib/docker

README.md

This is a fork from the original CoreOS cloud formation template. It adds two EBS disks of 30G, mounted to /var/lib/docker.

This is intended to be used with an m3.large machine which comes with 30G of SSD.

The reason I add two of 30 and not one of 60 is because a BTRFS limitation.

You can use the "btrfs add" approach in the units to create a BTRFS raid.

coreos.json

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Description": "CoreOS on EC2: http://coreos.com/docs/running-coreos/cloud-providers/ec2/",
  "Mappings" : {
      "RegionMap" : {

          "ap-northeast-1" : {
              "AMI" : "ami-1fb9e61e"
          },

          "sa-east-1" : {
              "AMI" : "ami-8f57fe92"
          },

          "ap-southeast-2" : {
              "AMI" : "ami-874620bd"
          },

          "ap-southeast-1" : {
              "AMI" : "ami-d6d88084"
          },

          "us-east-1" : {
              "AMI" : "ami-04a2766c"
          },

          "us-west-2" : {
              "AMI" : "ami-3193e801"
          },

          "us-west-1" : {
              "AMI" : "ami-63eae826"
          },

          "eu-west-1" : {
              "AMI" : "ami-92ea39e5"
          }

      }
  },
  "Parameters": {
    "InstanceType" : {
      "Description" : "EC2 PV instance type (m3.medium, etc). Note: m1.small is not supported.",
      "Type" : "String",
      "Default" : "m3.medium",
      "AllowedValues" : ["m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m1.medium", "m1.large", "m1.xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c1.medium", "c1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "hi1.4xlarge", "hs1.8xlarge", "t1.micro"],
      "ConstraintDescription" : "Must be a valid EC2 PV instance type. Note: m1.small is not supported."
    },
    "ClusterSize": {
      "Default": "3",
      "MinValue": "3",
      "MaxValue": "12",
      "Description": "Number of nodes in cluster (3-12).",
      "Type": "Number"
    },
    "DiscoveryURL": {
      "Description": "An unique etcd cluster discovery URL. Grab a new token from https://discovery.etcd.io/new",
      "Type": "String"
    },
    "AllowSSHFrom": {
      "Description": "The net block (CIDR) that SSH is available to.",
      "Default": "0.0.0.0/0",
      "Type": "String"
    },
    "KeyPair" : {
      "Description" : "The name of an EC2 Key Pair to allow SSH access to the instance.",
      "Type" : "String"
    }
  },
  "Resources": {
    "CoreOSSecurityGroup": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "CoreOS SecurityGroup",
        "SecurityGroupIngress": [
          {"IpProtocol": "tcp", "FromPort": "22", "ToPort": "22", "CidrIp": {"Ref": "AllowSSHFrom"}}
        ]
      }
    },
    "IngressAll": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties": {
        "GroupName": {"Ref": "CoreOSSecurityGroup"}, "IpProtocol": "tcp", "FromPort": "0", "ToPort": "65535", "SourceSecurityGroupId": {
          "Fn::GetAtt" : [ "CoreOSSecurityGroup", "GroupId" ]
        }
      }
    },
    "CoreOSServerAutoScale": {
      "Type": "AWS::AutoScaling::AutoScalingGroup",
      "Properties": {
        "AvailabilityZones": {"Fn::GetAZs": ""},
        "LaunchConfigurationName": {"Ref": "CoreOSServerLaunchConfig"},
        "MinSize": "3",
        "MaxSize": "12",
        "DesiredCapacity": {"Ref": "ClusterSize"},
        "Tags": [
            {"Key": "Name", "Value": { "Ref" : "AWS::StackName" }, "PropagateAtLaunch": true}
        ]
      }
    },
    "CoreOSServerLaunchConfig": {
      "Type": "AWS::AutoScaling::LaunchConfiguration",
      "Properties": {
        "ImageId" : { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "AMI" ]},
        "InstanceType": {"Ref": "InstanceType"},
        "KeyName": {"Ref": "KeyPair"},
        "SecurityGroups": [{"Ref": "CoreOSSecurityGroup"}],
        "UserData" : { "Fn::Base64":
          { "Fn::Join": [ "", [
            "#cloud-config\n",
            "\n",
            "coreos:\n",
            "  etcd:\n",
            "    discovery: ", { "Ref": "DiscoveryURL" } , "\n",
            "    addr: $private_ipv4:4001\n",
            "    peer-addr: $private_ipv4:7001\n",
            "  units:\n",
            "    - name: etcd.service\n",
            "      command: start\n",
            "    - name: fleet.service\n",
            "      command: start\n",
            "    - name: format-disks.service\n",
            "      runtime: true\n",
            "      command: start\n",
            "      content: |\n",
            "        [Unit]\n",
            "        Description=Wipe the two EBS devices, only if the first one is not BTRFS yet.\n",
            "        [Service]\n",
            "        Type=oneshot\n",
            "        RemainAfterExit=yes\n",
            "        ExecStart=/bin/bash -c '(/usr/sbin/blkid -t TYPE=btrfs | grep /dev/xvdb) || (/usr/sbin/wipefs -fa /dev/xvdb && /usr/sbin/wipefs -fa /dev/xvdc && /usr/sbin/mkfs.btrfs -f /dev/xvdb)'\n",
            "    - name: var-lib-docker.mount\n",
            "      command: start\n",
            "      content: |\n",
            "        [Unit]\n",
            "        Description=Mount the first EBS device to /var/lib/docker\n",
            "        Requires=format-disks.service\n",
            "        After=format-disks.service\n",
            "        Before=docker.service\n",
            "        [Mount]\n",
            "        What=/dev/xvdb\n",
            "        Where=/var/lib/docker\n",
            "        Type=btrfs\n",
            "    - name: var-lib-docker-2nd.service\n",
            "      runtime: true\n",
            "      command: start\n",
            "      content: |\n",
            "        [Unit]\n",
            "        Description=Add the second device to /var/lib/docker\n",
            "        Requires=format-disks.service\n",
            "        After=format-disks.service\n",
            "        RequiresMountsFor=/var/lib/docker\n",
            "        [Service]\n",
            "        Type=oneshot\n",
            "        RemainAfterExit=yes\n",
            "        ExecStart=/usr/sbin/btrfs device add /dev/xvdc /var/lib/docker",
            "\n",
            "write_files:\n",
            "  - path: /home/core/.dockercfg\n",
            "    owner: core:core\n",
            "    permissions: 0644\n",
            "    content: |\n",
            "      {\n",
            "        \"docker.auth0.com\": {\n",
            "          \"auth\": \"YXV0aDA6UGFzc3cwcmQh\",\n",
            "          \"email\": \"[email protected]\"\n",
            "        }\n",
            "      }\n"
            ] ]
          }
        },
        "BlockDeviceMappings": [
          {
            "DeviceName" : "/dev/xvdb",
            "Ebs" : {
              "VolumeSize" : 30
            }
          },
          {
            "DeviceName" : "/dev/xvdc",
            "Ebs" : {
              "VolumeSize" : 30
            }
          }
        ]
      }
    }
  }
}

Just a heads up, you left your docker config creds in there. That being said, this is really useful / exactly what I was looking for. Thanks!