AWS CloudFormation Template For ElasticSearch Cluster

ElasticSearch.template

{
  "AWSTemplateFormatVersion" : "2010-09-09",

  "Description" : "ElasticSearch Cluster.",

  "Parameters" : {

    "KeyName": {
      "Description" : "The name of an existing key pair to enable SSH access to Amazon EC2 instances",
      "Type": "String",
      "MinLength": "1",
      "MaxLength": "255",
      "AllowedPattern" : "[\\x20-\\x7E]*",
      "ConstraintDescription" : "KeyPair name from 1 to 255 ASCII characters."
    },

    "InstanceType" : {
      "Description" : "Elasticsearch Node Type (EC2 Instance)",
      "Type" : "String",
      "Default" : "m1.large",
      "AllowedValues" : [
        "t1.micro",
        "m1.small",
        "m1.medium",
        "m1.large",
        "m1.xlarge",
        "m3.medium",
        "m3.large",
        "m3.xlarge",
        "m3.2xlarge",
        "c1.medium",
        "c1.xlarge",
        "c3.large",
        "c3.xlarge",
        "c3.2xlarge",
        "c3.4xlarge",
        "c3.8xlarge",
        "cc2.8xlarge",
        "m2.xlarge",
        "m2.2xlarge",
        "m2.4xlarge",
        "r3.large",
        "r3.xlarge",
        "r3.2xlarge",
        "r3.4xlarge",
        "r3.8xlarge",
        "cr1.8xlarge",
        "hi1.4xlarge",
        "hs1.8xlarge",
        "i2.xlarge",
        "i2.2xlarge",
        "i2.4xlarge",
        "i2.8xlarge"
      ],
      "ConstraintDescription" : "Must be a valid Amazon EC2 instance type."
    },

    "SSHLocation" : {
      "Description" : "The range of IP addresses with access to SSH into the instances",
      "Type": "String",
      "MinLength": "9",
      "MaxLength": "18",
      "Default": "0.0.0.0/0",
      "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
      "ConstraintDescription": "Must be a valid IP CIDR range of the form x.x.x.x/x."
    },

    "ClusterSize" : {
      "Description" : "The number of Elasticsearch instances to launch in the Auto Scaling group",
      "Type" : "Number",
      "Default" : "3"
    },

    "ElasticsearchVersion" : {
      "Description" : "The version number of Elasticsearch used throughout the application",
      "Type" : "String",
      "Default" : "1.4.2",
      "AllowedValues" : [ "1.4.2" ],
      "ConstraintDescription" : "Must be a supported version number."
    }

  },
  "Mappings" : {
    "AWSInstanceType2Arch" : {
      "t1.micro"     : { "Arch" : "64" },
      "m1.small"     : { "Arch" : "64" },
      "m1.medium"    : { "Arch" : "64" },
      "m1.large"     : { "Arch" : "64" },
      "m1.xlarge"    : { "Arch" : "64" },
      "m3.medium"    : { "Arch" : "64" },
      "m3.large"     : { "Arch" : "64" },
      "m3.xlarge"    : { "Arch" : "64" },
      "m3.2xlarge"   : { "Arch" : "64" },
      "c1.medium"    : { "Arch" : "64" },
      "c1.xlarge"    : { "Arch" : "64" },
      "c3.large"     : { "Arch" : "64" },
      "c3.xlarge"    : { "Arch" : "64" },
      "c3.2xlarge"   : { "Arch" : "64" },
      "c3.4xlarge"   : { "Arch" : "64" },
      "c3.8xlarge"   : { "Arch" : "64" },
      "cc2.8xlarge"  : { "Arch" : "64HVM" },
      "m2.xlarge"    : { "Arch" : "64" },
      "m2.2xlarge"   : { "Arch" : "64" },
      "m2.4xlarge"   : { "Arch" : "64" },
      "r3.large"     : { "Arch" : "64HVM" },
      "r3.xlarge"    : { "Arch" : "64HVM" },
      "r3.2xlarge"   : { "Arch" : "64HVM" },
      "r3.4xlarge"   : { "Arch" : "64HVM" },
      "r3.8xlarge"   : { "Arch" : "64HVM" },
      "cr1.8xlarge"  : { "Arch" : "64HVM" },
      "hi1.4xlarge"  : { "Arch" : "64HVM" },
      "hs1.8xlarge"  : { "Arch" : "64HVM" },
      "i2.xlarge"    : { "Arch" : "64HVM" },
      "i2.2xlarge"   : { "Arch" : "64HVM" },
      "i2.4xlarge"   : { "Arch" : "64HVM" },
      "i2.8xlarge"   : { "Arch" : "64HVM" }
    },
    "AWSRegionArch2AMI" : {
      "us-east-1"      : { "64" : "ami-fb8e9292", "64HVM" : "ami-978d91fe" },
      "us-west-1"      : { "64" : "ami-7aba833f", "64HVM" : "ami-5aba831f" },
      "us-west-2"      : { "64" : "ami-043a5034", "64HVM" : "ami-383a5008" },
      "eu-west-1"      : { "64" : "ami-2918e35e", "64HVM" : "ami-4b18e33c" },
      "sa-east-1"      : { "64" : "ami-215dff3c", "64HVM" : "ami-635dff7e" },
      "ap-southeast-1" : { "64" : "ami-b40d5ee6", "64HVM" : "ami-860d5ed4" },
      "ap-southeast-2" : { "64" : "ami-3b4bd301", "64HVM" : "ami-cf4ad2f5" },
      "ap-northeast-1" : { "64" : "ami-c9562fc8", "64HVM" : "ami-bb562fba" }
    },
    "ElasticsearchVersion2ServiceWrapperHash" : {
      "1.4.2" : { "Hash" : "4943d5a" }
    },
    "ElasticsearchVersion2AWSCloudPluginVersion" : {
      "1.4.2"   : { "Ver" : "2.4.1" }
    }
  },

  "Resources" : {

    "ElasticsearchDiscoveryRole" : {
      "Type" : "AWS::IAM::Role",
      "Properties" : {
        "AssumeRolePolicyDocument" : {
          "Version" : "2012-10-17",
          "Statement" : [ {
            "Effect" : "Allow",
            "Principal" : {
              "Service" : [ "ec2.amazonaws.com" ]
            },
            "Action" : [ "sts:AssumeRole" ]
          } ]
        },
        "Path" : "/"
      }
    },

    "ElasticsearchRolePolicies" : {
      "Type" : "AWS::IAM::Policy",
      "Properties" : {
        "PolicyName" : "esdicovery",
        "PolicyDocument": {
          "Version" : "2012-10-17",
          "Statement": [ {
              "Effect": "Allow",
              "Action": "ec2:*",
              "Resource": "*"
          } ]
        },
        "Roles": [ { "Ref": "ElasticsearchDiscoveryRole" } ]
      }
    },

    "ElasticsearchInstanceProfile" : {
      "Type": "AWS::IAM::InstanceProfile",
      "Properties": {
        "Path": "/",
        "Roles": [ { "Ref": "ElasticsearchDiscoveryRole" } ]
      }
    },

    "ElasticsearchServerGroup" : {
      "Type" : "AWS::AutoScaling::AutoScalingGroup",
      "Properties" : {
        "AvailabilityZones" : { "Fn::GetAZs" : ""},
        "LaunchConfigurationName" : { "Ref" : "ElasticsearchServer" },
        "MinSize" : "1",
        "MaxSize" : "12",
        "DesiredCapacity" : { "Ref" : "ClusterSize" },
        "Tags" : [ { "Key" : "type", "Value" : "elasticsearch", "PropagateAtLaunch" : "true" } ]
      }
    },

    "ElasticsearchServer": {
      "Type": "AWS::AutoScaling::LaunchConfiguration",

      "Metadata" : {
        "AWS::CloudFormation::Init" : {
          "config" : {
            "commands" : {
              "createfile" : {
              "command" : { "Fn::Join" : ["", [
                "echo \"",
                "cloud.aws.region:\n",
                " ", { "Ref" : "AWS::Region" }, "\n",
                "discovery:\n",
                " type: ec2\n",
                "\n",
                "discovery.ec2.groups: ", { "Ref" : "ElasticsearchSecurityGroup" }, "\n",
                "discovery.ec2.tag.type: elasticsearch\n",
                "\n",
                "cloud.node.auto_attributes: true\n",
                " \" > ",
                "/usr/local/elasticsearch/elasticsearch-", { "Ref" : "ElasticsearchVersion"  }, "/config/elasticsearch.yml"
              ]]}
              }
            }
          }
        }
      },

      "Properties": {
        "ImageId" : {
          "Fn::FindInMap" : [
            "AWSRegionArch2AMI",
            { "Ref" : "AWS::Region" },
            { "Fn::FindInMap" : [ "AWSInstanceType2Arch",
              { "Ref" : "InstanceType" }, "Arch" ] } ] },
        "InstanceType"   : { "Ref" : "InstanceType" },
        "SecurityGroups" : [
          {"Ref" : "ElasticsearchSecurityGroup"}
        ],
        "KeyName"        : { "Ref" : "KeyName" },
        "IamInstanceProfile" : { "Ref" : "ElasticsearchInstanceProfile" },
        "UserData"       : { "Fn::Base64" : { "Fn::Join" : ["", [
          "#!/bin/bash\n",
          "yum update -y aws-cfn-bootstrap\n",

          "# Helper function\n",
          "function error_exit\n",
          "{\n",
          "  /opt/aws/bin/cfn-signal -e 1 -r \"$1\" '", { "Ref" : "WaitHandle" }, "'\n",
          "  exit 1\n",
          "}\n",

          "# Install application\n",
          "#/opt/aws/bin/cfn-init -s ", { "Ref" : "AWS::StackId" }, " -r ElasticsearchServer ",
          "#    --region ", { "Ref" : "AWS::Region" }, "\n",

          "#get and unzip elasticsearch\n",
          "wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" , { "Ref" : "ElasticsearchVersion"  }, ".zip || error_exit \"Failed to retrieve elasticsearch archive\"\n",
          "unzip elasticsearch-" , { "Ref" : "ElasticsearchVersion"  }, ".zip -d /usr/local/elasticsearch\n",

          "#install aws plugin\n",
          "cd /usr/local/elasticsearch/elasticsearch-" , { "Ref" : "ElasticsearchVersion"  }, "\n",
          "res=$(bin/plugin -install org.elasticsearch/elasticsearch-cloud-aws/", { "Fn::FindInMap" : [ "ElasticsearchVersion2AWSCloudPluginVersion", { "Ref" : "ElasticsearchVersion" }, "Ver"] }, ")\n",
          "if [ \"$?\" -ne \"0\" ]; then\n",
          "   error_exit \"Failed to install aws plugin: ${res}\"\n",
          "fi\n",

          "# Install elasticsearch config.yml\n",
          "/opt/aws/bin/cfn-init -s ", { "Ref" : "AWS::StackId" }, " -r ElasticsearchServer ",
          "    --region ", { "Ref" : "AWS::Region" }, "|| error_exit \"failed to run cfn-init\"\n",

          "#install elasticsearch servicewrapper daemon\n",
          "cd ~\n",
          "wget https://github.com/elasticsearch/elasticsearch-servicewrapper/zipball/", { "Fn::FindInMap" : [ "ElasticsearchVersion2ServiceWrapperHash", { "Ref" : "ElasticsearchVersion" }, "Hash"] }, "\n",
          "unzip ", { "Fn::FindInMap" : [ "ElasticsearchVersion2ServiceWrapperHash", { "Ref" : "ElasticsearchVersion" }, "Hash"] }, "\n",
          "mv elasticsearch-elasticsearch-servicewrapper-", { "Fn::FindInMap" : [ "ElasticsearchVersion2ServiceWrapperHash", { "Ref" : "ElasticsearchVersion" }, "Hash"] }, "/service/ /usr/local/elasticsearch/elasticsearch-" , { "Ref" : "ElasticsearchVersion"  }, "/bin/\n",
          "cd /usr/local/elasticsearch/elasticsearch-" , { "Ref" : "ElasticsearchVersion"  }, "\n",
          "sed -i 's#set.default.ES_HOME=.*#set.default.ES_HOME='$PWD'#g' bin/service/elasticsearch.conf\n",
          "#changing default heap size for smaller instances\n",
          "sed -i 's#set.default.ES_HEAP_SIZE=.*#set.default.ES_HEAP_SIZE=512#g' bin/service/elasticsearch.conf\n",
          "bin/service/elasticsearch64 install || error_exit \"Failed install elasticsearch daemon\"\n",
          "res=$(bin/service/elasticsearch64 start)\n",
          "if [ \"$?\" -ne \"0\" ]; then\n",
          "   error_exit \"Failed to start elasticsearch servicewrapper: ${res}\"\n",
          "fi\n",

          "# All is well so signal success\n",
          "/opt/aws/bin/cfn-signal -e $? '", { "Ref" : "WaitHandle" }, "'\n"
        ]]}}
      }
    },

    "WaitHandle" : {
      "Type" : "AWS::CloudFormation::WaitConditionHandle"
    },

    "WaitCondition" : {
      "Type" : "AWS::CloudFormation::WaitCondition",
      "DependsOn" : "ElasticsearchServer",
      "Properties" : {
        "Handle" : {"Ref" : "WaitHandle"},
        "Timeout" : "600"
      }
    },

    "ElasticsearchSecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroup",
      "Properties" : {
        "GroupDescription" : "Enable Elasticsearch access",
        "SecurityGroupIngress" : [
          {"IpProtocol" : "tcp", "FromPort" : "9200", "ToPort" : "9200", "CidrIp" : "0.0.0.0/0"},
          {"IpProtocol" : "tcp", "FromPort" : "9300", "ToPort" : "9300", "CidrIp" : "0.0.0.0/0"},
          {"IpProtocol" : "tcp", "FromPort" : "22", "ToPort" : "22", "CidrIp" : { "Ref" : "SSHLocation"}}
        ]
      }
    }
  }
}