Created
July 15, 2012 03:53
-
-
Save pcn/3114862 to your computer and use it in GitHub Desktop.
gertrude launch template
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "AWSTemplateFormatVersion" : "2010-09-09", | |
| "Description" : "Creates a Gertrude cluster", | |
| "Parameters" : { | |
| "Environment" : { | |
| "Description" : "The environment where this stack belongs. 'Production' and 'Staging' are reserved words. Do NOT use them for testing.", | |
| "Type" : "String" | |
| }, | |
| "EnvironmentClass" : { | |
| "Description" : "The environment class where this stack belongs. 'Production' and 'Staging' are reserved words. Do NOT use them for testing.", | |
| "Type" : "String", | |
| "Default" : "Dev" | |
| }, | |
| "BillingEnvironment": { | |
| "Description": "The AWS billing environment where this stack belongs.", | |
| "Type": "String", | |
| "Default": "staging", | |
| "AllowedValues": [ | |
| "production", "staging", "utility", "test" | |
| ] | |
| }, | |
| "KeyName" : { | |
| "Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instances", | |
| "Type" : "String", | |
| "Default" : "CF-STAGING-01", | |
| "MinLength": "1", | |
| "MaxLength": "64", | |
| "AllowedPattern" : "[-_ a-zA-Z0-9]*", | |
| "ConstraintDescription" : "can contain only alphanumeric characters, spaces, dashes and underscores." | |
| }, | |
| "InstanceType" : { | |
| "Description" : "Cassanda EC2 instance type is used for all instances", | |
| "Type" : "String", | |
| "Default" : "c1.medium", | |
| "AllowedValues" : [ "t1.micro", "m1.small", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "c1.medium", "c1.xlarge", "cc1.4xlarge" ], | |
| "ConstraintDescription" : "must be a valid EC2 instance type." | |
| }, | |
| "Group1Count" : { | |
| "Description" : "Instances desired in us-east-1a", | |
| "Type" : "String", | |
| "Default" : "1" | |
| }, | |
| "Group2Count" : { | |
| "Description" : "Instances desired in us-east-1b", | |
| "Type" : "String", | |
| "Default" : "0" | |
| }, | |
| "ImageId" : { | |
| "Description" : "AMI Id to use (should be natty or later release)", | |
| "Type" : "String", | |
| "Default" : "ami-04bc666d" | |
| }, | |
| "ClusterSize" : { | |
| "Description" : "The size of the Cassandra cluster. How many nodes to launch under the Autoscale group.", | |
| "Type" : "Number", | |
| "Default" : "3" | |
| }, | |
| "PREREQZooKeeperDNSList" : { | |
| "Description" : "A comma separated list of zookeeper 'host:port' server instances without spaces between commas (e.g., 'zk1.knewton.com:2181' or 'zk1.knewton.com:2181,zk2.knewton.com:2181,zk3.knewton.com:2181')", | |
| "Type" : "String" | |
| }, | |
| "GertrudeVersion" : { | |
| "Description" : "Version of Gertrude build", | |
| "Type" : "String" | |
| }, | |
| "SystemsVersion": { | |
| "Description": "The systems library that does various house keeping tasks on the box", | |
| "Type": "String", | |
| "Default": "0.0.12" | |
| }, | |
| "DistRev": { | |
| "Description": "The version of the Dist package for scripts used to bootstrap a system", | |
| "Type": "String", | |
| "Default": "11" | |
| }, | |
| "JavaDistRev": { | |
| "Description": "The version of the Dist package for scripts used to bootstrap a system", | |
| "Type": "String", | |
| "Default": "0.0.2" | |
| } | |
| }, | |
| "Mappings": { | |
| "JavaHeapByInstance": { | |
| "m1.small" : {"default" : "-Xmx800m"}, | |
| "m1.medium" : {"default" : "-Xmx2600m"}, | |
| "m1.large" : {"default" : "-Xmx4800m"}, | |
| "m1.xlarge" : {"default" : "-Xmx11000m"}, | |
| "m2.xlarge" : {"default" : "-Xmx14000m"}, | |
| "m2.2xlarge" : {"default" : "-Xmx25000m"}, | |
| "m2.4xlarge" : {"default" : "-Xmx55000m"}, | |
| "c1.medium" : {"default" : "-Xmx800m"}, | |
| "c1.xlarge" : {"default" : "-Xmx4800m"}, | |
| "cc1.4xlarge" : {"default" : "-Xmx18000m"}, | |
| "cc1.8xlarge" : {"default" : "-Xmx50000m"} | |
| } | |
| }, | |
| "Resources" : { | |
| "CfnUser" : { | |
| "Type" : "AWS::IAM::User", | |
| "Properties" : { | |
| "Path": "/", | |
| "Policies": [{ | |
| "PolicyName" : "gertrude", | |
| "PolicyDocument": { | |
| "Statement":[{ | |
| "Effect" : "Allow", | |
| "Action" : [ "cloudformation:DescribeStackResource" ], | |
| "Resource" :"*" | |
| }] | |
| } | |
| }] | |
| } | |
| }, | |
| "CfnKeys" : { | |
| "Type" : "AWS::IAM::AccessKey", | |
| "Properties" : { | |
| "UserName" : {"Ref": "CfnUser"} | |
| } | |
| }, | |
| "BuildBucketPolicy": { | |
| "Type": "AWS::S3::BucketPolicy", | |
| "DependsOn" : "RawBucketPolicy", | |
| "Properties": { | |
| "PolicyDocument": { | |
| "Version": "2008-10-17", | |
| "Id": { "Fn::Join" : [ "", [ | |
| "BuildPolicy-", | |
| { "Fn::GetAtt": [ "CfnUser", "Arn" ] } | |
| ]]}, | |
| "Statement": [{ | |
| "Sid": "ReadAccess", | |
| "Action": [ "s3:GetObject" ], | |
| "Effect": "Allow", | |
| "Resource": { "Fn::Join" : [ "", [ | |
| "arn:aws:s3:::", | |
| "knewton-", | |
| { "Ref": "BillingEnvironment" }, | |
| "-build", | |
| "/*" | |
| ]]}, | |
| "Principal": { "AWS": { | |
| "Fn::GetAtt": [ "CfnUser", "Arn" ] | |
| }} | |
| }] | |
| }, | |
| "Bucket" : { "Fn::Join": [ "", [ | |
| "knewton-", | |
| { "Ref": "BillingEnvironment" }, | |
| "-build" | |
| ]]} | |
| } | |
| }, | |
| "RawBucketPolicy": { | |
| "Type": "AWS::S3::BucketPolicy", | |
| "Properties": { | |
| "PolicyDocument": { | |
| "Version": "2008-10-17", | |
| "Id": { "Fn::Join" : [ "", [ | |
| "RawPolicy-", | |
| { "Fn::GetAtt": [ "CfnUser", "Arn" ] } | |
| ]]}, | |
| "Statement": [{ | |
| "Sid": "ReadAccess", | |
| "Action": [ | |
| "s3:GetObject", | |
| "s3:PutObject" | |
| ], | |
| "Effect": "Allow", | |
| "Resource": { "Fn::Join" : [ "", [ | |
| "arn:aws:s3:::", | |
| "knewton-rawdata-", | |
| { "Ref": "BillingEnvironment" }, | |
| "/*" | |
| ]]}, | |
| "Principal": { "AWS": { | |
| "Fn::GetAtt": [ "CfnUser", "Arn" ] | |
| }} | |
| }] | |
| }, | |
| "Bucket": { "Fn::Join": [ "", [ | |
| "knewton-rawdata-", | |
| { "Ref": "BillingEnvironment" } | |
| ]]} | |
| } | |
| } | |
| , | |
| "GertrudeConfig1": { | |
| "Type": "AWS::AutoScaling::LaunchConfiguration", | |
| "DependsOn" : "BuildBucketPolicy", | |
| "Metadata" : { | |
| "AWS::CloudFormation::Init" : { | |
| "configSets" : { | |
| "inorder" : [ | |
| "systems", | |
| "dist", | |
| "gertrude" | |
| ] | |
| }, | |
| "systems": { | |
| "sources": { | |
| "/var/tmp/knewton": { "Fn::Join": [ "", [ | |
| "http://s3.amazonaws.com/", | |
| "knewton-", | |
| { "Ref": "BillingEnvironment" }, | |
| "-build", | |
| "/Systems-", { "Ref": "SystemsVersion" }, | |
| ".tar.gz" | |
| ]]} | |
| }, | |
| "commands": { | |
| "01_runSystemsBundle": { | |
| "command": "sudo ./systems_bundle.sh /var/tmp/knewton/cookbooks/ base", | |
| "cwd": "/var/tmp/knewton" | |
| } | |
| } | |
| }, | |
| "dist": { | |
| "sources": { | |
| "/var/tmp/knewton": { "Fn::Join" : [ "", [ | |
| "http://s3.amazonaws.com/", | |
| "knewton-", | |
| { "Ref": "BillingEnvironment" }, | |
| "-build", | |
| "/Dist-", { "Ref": "DistRev" }, ".tar.gz" | |
| ]]} | |
| }, | |
| "commands": { | |
| "01_installDist": { | |
| "command": { "Fn::Join": ["", [ | |
| "mgmt/common/virtualenv.sh -a Dist ;", | |
| "source /opt/virtualenvs/Dist/bin/activate ;", | |
| "./setup.py install" | |
| ]]}, | |
| "cwd": "/var/tmp/knewton/dist/" | |
| }, | |
| "02_runDist": { | |
| "command": { "Fn::Join": ["", [ | |
| "source /opt/virtualenvs/Dist/bin/activate ;", | |
| "./box-config", | |
| " -c ", {"Ref": "EnvironmentClass"}, | |
| " -n ", {"Ref": "Environment"}, | |
| " -b ", {"Ref": "BillingEnvironment"}, | |
| " -s Gertrude", | |
| " -r Platform", | |
| " -A ", { "Ref": "CfnKeys" }, | |
| " -S ", { "Fn::GetAtt": [ | |
| "CfnKeys", "SecretAccessKey" | |
| ]}, | |
| " -w '", { "Ref": "WaitHandle1" }, | |
| "' || /var/tmp/knewton/notify-failed.sh" | |
| ]]}, | |
| "cwd": "/var/tmp/knewton/dist/bin" | |
| } | |
| } | |
| }, | |
| "gertrude" : { | |
| "sources" : { | |
| "/var/tmp/knewton" : { | |
| "Fn::Join" : [ "", | |
| [ "http://s3.amazonaws.com/", | |
| "knewton-", | |
| { "Ref": "BillingEnvironment" }, | |
| "-build", | |
| "/Gertrude-", | |
| { "Ref" : "GertrudeVersion" }, | |
| ".tar.gz" ]] | |
| }, | |
| "/var/tmp/knewton/" : { | |
| "Fn::Join" : [ "", | |
| [ "http://s3.amazonaws.com/", | |
| "knewton-", | |
| { "Ref": "BillingEnvironment" }, | |
| "-build", | |
| "/Javadist-", | |
| {"Ref" : "JavaDistRev" }, | |
| ".tar.gz" ]] | |
| } | |
| }, | |
| "files" : { | |
| "/etc/knewton/discovery/zookeeper/platform.yml" : { | |
| "content" : "Just need to create the file. Content will be overridden.", | |
| "mode" : "000644", | |
| "owner" : "root", | |
| "group" : "root" | |
| } | |
| }, | |
| "commands" : { | |
| "00_populateServersInConfig" : { | |
| "command" : { "Fn::Join" : ["", ["echo ", {"Ref" : "PREREQZooKeeperDNSList"}, " | perl -pe 's/,/}, {header: {service_class: zookeeper}, host: /g' | perl -pe 's/:([0-9])/, port: $1/g' | perl -pe 's/(.*)/{server_list: [{header: {service_class: zookeeper}, host: $1}]}/' > /etc/knewton/discovery/zookeeper/platform.yml"]]}, | |
| "cwd" : "/var/tmp/knewton" | |
| }, | |
| "01_installGertrudeDiscoveryConfigs": { | |
| "command": { | |
| "Fn::Join": [" ; ", [ | |
| "mv gertrude/conf/configuration/* /etc/knewton/configuration" | |
| ]] | |
| }, | |
| "cwd" : "/var/tmp/knewton" | |
| }, | |
| "02_installGertrude" : { | |
| "command" : { "Fn::Join" : ["", [ | |
| "/usr/bin/env ", | |
| "SERVICE_NAME=gertrude", | |
| "SERVICE_MAIN_CLASS=com.knewton.recommendation.platform.GertrudeMain ", | |
| "SERVICE_PARAMS='--reader --writer --intake' ", | |
| "JVM_ARGS=", | |
| {"Fn::FindInMap": [ | |
| "JavaHeapByInstance", | |
| {"Ref" : "InstanceType"}, | |
| "default"]}, | |
| " bash ./javadist/deploy.sh"]]} , | |
| "cwd" : "/var/tmp/knewton" | |
| } | |
| } | |
| }, | |
| "AWS::CloudFormation::Authentication" : { | |
| "S3AccessCreds" : { | |
| "type" : "S3", | |
| "accessKeyId" : { "Ref" : "CfnKeys" }, | |
| "secretKey" : {"Fn::GetAtt": ["CfnKeys", "SecretAccessKey"]}, | |
| "buckets" : [{"Fn::Join" : ["", [ "knewton-", | |
| { "Ref": "BillingEnvironment" }, | |
| "-build"]] | |
| }] | |
| } | |
| } | |
| } | |
| }, | |
| "Properties": { | |
| "ImageId" : "ami-04bc666d", | |
| "InstanceType" : { "Ref" : "InstanceType" }, | |
| "SecurityGroups" : [ | |
| {"Fn::Join" : ["", [{"Ref" : "Environment"}, "_KCS"]]}, | |
| {"Fn::Join": ["", ["GErtrude_",{"Ref":"Environment"},"_KCS"]]}, | |
| "Base" | |
| ], | |
| "KeyName" : { | |
| "Ref" : "KeyName" | |
| }, | |
| "UserData": { | |
| "Fn::Base64": { "Fn::Join": [ "", [ | |
| "#!/bin/bash\n", | |
| "# Install application\n", | |
| "/usr/local/bin/cfn-init -v -s ", { | |
| "Ref": "AWS::StackName" | |
| }, | |
| " -r GertrudeConfig1 -c inorder", | |
| " --access-key ", { "Ref": "CfnKeys" }, | |
| " --secret-key ", { "Fn::GetAtt": [ "CfnKeys", "SecretAccessKey" ]}, | |
| " --region ", { "Ref": "AWS::Region" }, "\n", | |
| "# Signal result\n", | |
| "/usr/local/bin/cfn-signal -e $? -r 'GertrudeConfig1' '", | |
| { "Ref": "WaitHandle1" }, "'\n" | |
| ]]} | |
| } | |
| } | |
| }, | |
| "GertrudeConfig2": { | |
| "Type": "AWS::AutoScaling::LaunchConfiguration", | |
| "DependsOn" : "BuildBucketPolicy", | |
| "Metadata" : { | |
| "AWS::CloudFormation::Init" : { | |
| "configSets" : { | |
| "inorder" : [ | |
| "systems", | |
| "dist", | |
| "gertrude" | |
| ] | |
| }, | |
| "systems": { | |
| "sources": { | |
| "/var/tmp/knewton": { "Fn::Join": [ "", [ | |
| "http://s3.amazonaws.com/", | |
| "knewton-", | |
| { "Ref": "BillingEnvironment" }, | |
| "-build", | |
| "/Systems-", { "Ref": "SystemsVersion" }, | |
| ".tar.gz" | |
| ]]} | |
| }, | |
| "commands": { | |
| "01_runSystemsBundle": { | |
| "command": "sudo ./systems_bundle.sh /var/tmp/knewton/cookbooks/ base", | |
| "cwd": "/var/tmp/knewton" | |
| } | |
| } | |
| }, | |
| "dist": { | |
| "sources": { | |
| "/var/tmp/knewton": { "Fn::Join" : [ "", [ | |
| "http://s3.amazonaws.com/", | |
| "knewton-", | |
| { "Ref": "BillingEnvironment" }, | |
| "-build", | |
| "/Dist-", { "Ref": "DistRev" }, ".tar.gz" | |
| ]]} | |
| }, | |
| "commands": { | |
| "01_installDist": { | |
| "command": { "Fn::Join": ["", [ | |
| "mgmt/common/virtualenv.sh -a Dist ;", | |
| "source /opt/virtualenvs/Dist/bin/activate ;", | |
| "./setup.py install" | |
| ]]}, | |
| "cwd": "/var/tmp/knewton/dist/" | |
| }, | |
| "02_runDist": { | |
| "command": { "Fn::Join": ["", [ | |
| "source /opt/virtualenvs/Dist/bin/activate ;", | |
| "./box-config", | |
| " -c ", {"Ref": "EnvironmentClass"}, | |
| " -n ", {"Ref": "Environment"}, | |
| " -b ", {"Ref": "BillingEnvironment"}, | |
| " -s Gertrude", | |
| " -r Platform", | |
| " -A ", { "Ref": "CfnKeys" }, | |
| " -S ", { "Fn::GetAtt": [ | |
| "CfnKeys", "SecretAccessKey" | |
| ]}, | |
| " -w '", { "Ref": "WaitHandle2" }, | |
| "' || /var/tmp/knewton/notify-failed.sh" | |
| ]]}, | |
| "cwd": "/var/tmp/knewton/dist/bin" | |
| } | |
| } | |
| }, | |
| "gertrude" : { | |
| "sources" : { | |
| "/var/tmp/knewton" : { | |
| "Fn::Join" : [ "", | |
| [ "http://s3.amazonaws.com/", | |
| "knewton-", | |
| { "Ref": "BillingEnvironment" }, | |
| "-build", | |
| "/Gertrude-", | |
| { "Ref" : "GertrudeVersion" }, | |
| ".tar.gz" ]] | |
| }, | |
| "/var/tmp/knewton/" : { | |
| "Fn::Join" : [ "", | |
| [ "http://s3.amazonaws.com/", | |
| "knewton-", | |
| { "Ref": "BillingEnvironment" }, | |
| "-build", | |
| "/Javadist-", | |
| {"Ref" : "JavaDistRev" }, | |
| ".tar.gz" ]] | |
| } | |
| }, | |
| "files" : { | |
| "/etc/knewton/discovery/zookeeper/platform.yml" : { | |
| "content" : "Just need to create the file. Content will be overridden.", | |
| "mode" : "000644", | |
| "owner" : "root", | |
| "group" : "root" | |
| } | |
| }, | |
| "commands" : { | |
| "00_populateServersInConfig" : { | |
| "command" : { "Fn::Join" : ["", ["echo ", {"Ref" : "PREREQZooKeeperDNSList"}, " | perl -pe 's/,/}, {header: {service_class: zookeeper}, host: /g' | perl -pe 's/:([0-9])/, port: $1/g' | perl -pe 's/(.*)/{server_list: [{header: {service_class: zookeeper}, host: $1}]}/' > /etc/knewton/discovery/zookeeper/platform.yml"]]}, | |
| "cwd" : "/var/tmp/knewton" | |
| }, | |
| "01_installGertrudeDiscoveryConfigs": { | |
| "command": { | |
| "Fn::Join": [" ; ", [ | |
| "mv gertrude/conf/configuration/* /etc/knewton/configuration" | |
| ]] | |
| }, | |
| "cwd" : "/var/tmp/knewton" | |
| }, | |
| "02_installGertrude" : { | |
| "command" : { "Fn::Join" : ["", [ | |
| "/usr/bin/env ", | |
| "SERVICE_NAME=gertrude", | |
| "SERVICE_MAIN_CLASS=com.knewton.recommendation.platform.GertrudeMain ", | |
| "SERVICE_PARAMS='--reader --writer --intake' ", | |
| "JVM_ARGS=", | |
| {"Fn::FindInMap": [ | |
| "JavaHeapByInstance", | |
| {"Ref" : "InstanceType"}, | |
| "default"]}, | |
| " bash ./javadist/deploy.sh"]]} , | |
| "cwd" : "/var/tmp/knewton" | |
| } | |
| } | |
| }, | |
| "AWS::CloudFormation::Authentication" : { | |
| "S3AccessCreds" : { | |
| "type" : "S3", | |
| "accessKeyId" : { "Ref" : "CfnKeys" }, | |
| "secretKey" : {"Fn::GetAtt": ["CfnKeys", "SecretAccessKey"]}, | |
| "buckets" : [{"Fn::Join" : ["", [ "knewton-", | |
| { "Ref": "BillingEnvironment" }, | |
| "-build"]] | |
| }] | |
| } | |
| } | |
| } | |
| }, | |
| "Properties": { | |
| "ImageId" : "ami-04bc666d", | |
| "InstanceType" : { "Ref" : "InstanceType" }, | |
| "SecurityGroups" : [ | |
| {"Fn::Join" : ["", [{"Ref" : "Environment"}, "_KCS"]]}, | |
| {"Fn::Join": ["", ["GErtrude_",{"Ref":"Environment"},"_KCS"]]}, | |
| "Base" | |
| ], | |
| "KeyName" : { | |
| "Ref" : "KeyName" | |
| }, | |
| "UserData": { | |
| "Fn::Base64": { "Fn::Join": [ "", [ | |
| "#!/bin/bash\n", | |
| "# Install application\n", | |
| "/usr/local/bin/cfn-init -v -s ", { | |
| "Ref": "AWS::StackName" | |
| }, | |
| " -r GertrudeConfig2 -c inorder", | |
| " --access-key ", { "Ref": "CfnKeys" }, | |
| " --secret-key ", { "Fn::GetAtt": [ "CfnKeys", "SecretAccessKey" ]}, | |
| " --region ", { "Ref": "AWS::Region" }, "\n", | |
| "# Signal result\n", | |
| "/usr/local/bin/cfn-signal -e $? -r 'GertrudeConfig2' '", | |
| { "Ref": "WaitHandle2" }, "'\n" | |
| ]]} | |
| } | |
| } | |
| }, | |
| "GertrudeGroup1" : { | |
| "Type" : "AWS::AutoScaling::AutoScalingGroup", | |
| "Properties" : { | |
| "AvailabilityZones" : ["us-east-1a","us-east-1b"], | |
| "LaunchConfigurationName" : { "Ref" : "GertrudeConfig1" }, | |
| "MinSize" : { "Ref" : "Group1Count" }, | |
| "MaxSize" : { "Ref" : "Group1Count" } | |
| } | |
| }, | |
| "GertrudeGroup2" : { | |
| "Type" : "AWS::AutoScaling::AutoScalingGroup", | |
| "Properties" : { | |
| "AvailabilityZones" : ["us-east-1c","us-east-1d","us-east-1e"], | |
| "LaunchConfigurationName" : { "Ref" : "GertrudeConfig2" }, | |
| "MinSize" : { "Ref" : "Group2Count" }, | |
| "MaxSize" : { "Ref" : "Group2Count" } | |
| } | |
| }, | |
| "WaitHandle1" : { | |
| "Type" : "AWS::CloudFormation::WaitConditionHandle" | |
| }, | |
| "WaitHandle2" : { | |
| "Type" : "AWS::CloudFormation::WaitConditionHandle" | |
| }, | |
| "WaitCondition1" : { | |
| "Type" : "AWS::CloudFormation::WaitCondition", | |
| "DependsOn" : "GertrudeGroup1", | |
| "Properties" : { | |
| "Handle" : {"Ref" : "WaitHandle1"}, | |
| "Timeout" : "900" | |
| } | |
| }, | |
| "WaitCondition2" : { | |
| "Type" : "AWS::CloudFormation::WaitCondition", | |
| "DependsOn" : "GertrudeGroup2", | |
| "Properties" : { | |
| "Handle" : {"Ref" : "WaitHandle2"}, | |
| "Timeout" : "900" | |
| } | |
| } | |
| }, | |
| "Outputs" : { | |
| "Services" : { | |
| "Value" : "Gertrude", | |
| "Description" : "Gertrude, now with more G." | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment