Skip to content

Instantly share code, notes, and snippets.

@thoughtpolice

thoughtpolice/foundationdb-beta1-getting-started.template

Created February 7, 2019 03:14
Show Gist options
  • Save thoughtpolice/34f7ce0dd077e95fc67012917ddce47a to your computer and use it in GitHub Desktop.
Save thoughtpolice/34f7ce0dd077e95fc67012917ddce47a to your computer and use it in GitHub Desktop.
Download ZIP
Ancient CloudFormation FoundationDB template, discovered by completing an ancient trial
Raw
foundationdb-beta1-getting-started.template
{
"AWSTemplateFormatVersion" : "2010-09-09",
"Description" : "FoundationDB CloudFormation Test. **WARNING** This template creates one or more Amazon EC2 instances. You will be billed for the AWS resources used if you create a stack from this template.",
"Parameters" : {
"InstanceType" : {
"Description" : "EC2 instance type",
"Type" : "String",
"Default" : "m1.large",
"AllowedValues" : [ "m1.large","m1.xlarge","m2.xlarge","m2.2xlarge","m2.4xlarge","cc1.4xlarge","cc2.8xlarge" ],
"ConstraintDescription" : "must be one of the following EC2 instance types - m1.large, m1.xlarge, m2.xlarge, m2.2xlarge, m2.4xlarge, cc1.4xlarge, cc2.8xlarge."
},
"KeyName" : {
"Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instances",
"Type" : "String"
},
"InstanceCount" : {
"Description" : "Number of FoundationDB instances to start",
"Type" : "Number",
"MinValue" : 1
},
"LicenseAgreement" : {
"Description" : "Type 'accept' if you have read and accept the FoundationDB Beta Software License Agreement (http://foundationdb.com/BetaLicenseAgreement.pdf)",
"ConstraintDescription" : "To use this template (which downloads and runs FoundationDB) you must have read and accepted the FoundationDB Beta Software License Agreement that you can find at http://foundationdb.com/BetaLicenseAgreement.pdf",
"Type" : "String",
"AllowedValues" : [ "accept" ]
}
},
"Mappings" : {
"AWSRegion2AMI" : {
"us-east-1" : { "AMI" : "ami-cdc072a4" },
"us-west-1" : { "AMI" : "ami-fb5176be" },
"us-west-2" : { "AMI" : "ami-b47af484" },
"eu-west-1" : { "AMI" : "ami-3dcacb49" },
"sa-east-1" : { "AMI" : "ami-00f22b1d" },
"ap-southeast-1" : { "AMI" : "ami-a86424fa" },
"ap-southeast-2" : { "AMI" : "ami-818611bb" },
"ap-northeast-1" : { "AMI" : "ami-9405b995" }
},
"InstanceTypeDefaults" : {
"m1.large" : { "TotalMem" : "7000", "StorageMem" : "3500", "Processes" : "1" },
"m1.xlarge" : { "TotalMem" : "4000", "StorageMem" : "2000", "Processes" : "3" },
"m2.xlarge" : { "TotalMem" : "7000", "StorageMem" : "3500", "Processes" : "2" },
"m2.2xlarge" : { "TotalMem" : "8000", "StorageMem" : "4500", "Processes" : "4" },
"m2.4xlarge" : { "TotalMem" : "8000", "StorageMem" : "4500", "Processes" : "8" },
"cc1.4xlarge" : { "TotalMem" : "5000", "StorageMem" : "2500", "Processes" : "5" },
"cc2.8xlarge" : { "TotalMem" : "7500", "StorageMem" : "4000", "Processes" : "16" },
"cg1.4xlarge" : { "TotalMem" : "5000", "StorageMem" : "2500", "Processes" : "5" }
}
},
"Resources" : {
"CfnUser" : {
"Type" : "AWS::IAM::User",
"Properties" : {
"Path" : "/",
"Policies" : [{
"PolicyName" : "root",
"PolicyDocument" : { "Statement": [{
"Effect" : "Allow",
"Action" : [
"cloudformation:DescribeStackResource",
"ec2:DescribeInstances"
],
"Resource" : "*"
}]}
}]
}
},
"HostKeys" : {
"Type" : "AWS::IAM::AccessKey",
"Properties" : {
"UserName" : { "Ref" : "CfnUser" }
}
},
"FDBServerGroup" : {
"Type" : "AWS::AutoScaling::AutoScalingGroup",
"Properties" : {
"AvailabilityZones" : { "Fn::GetAZs" : "" },
"LaunchConfigurationName" : { "Ref" : "LaunchConfig" },
"MinSize" : { "Ref" : "InstanceCount" },
"MaxSize" : { "Ref" : "InstanceCount" }
}
},
"InstanceSecurityGroup" : {
"Type" : "AWS::EC2::SecurityGroup",
"Properties" : {
"GroupDescription" : "Enable SSH access and internal FoundationDB communications",
"SecurityGroupIngress" :
[ { "IpProtocol" : "tcp", "FromPort" : "22", "ToPort" : "22", "CidrIp" : "0.0.0.0/0" } ]
}
},
"FoundationDBServerIngress" : {
"Type" : "AWS::EC2::SecurityGroupIngress",
"Properties" : {
"GroupName" : { "Ref" : "InstanceSecurityGroup" },
"IpProtocol" : "tcp",
"FromPort" : "4500",
"ToPort" : "4600",
"SourceSecurityGroupName" : { "Ref" : "InstanceSecurityGroup" }
}
},
"CoordinationIngress" : {
"Type" : "AWS::EC2::SecurityGroupIngress",
"Properties" : {
"GroupName" : { "Ref" : "InstanceSecurityGroup" },
"IpProtocol" : "tcp",
"FromPort" : "4800",
"ToPort" : "4800",
"SourceSecurityGroupName" : { "Ref" : "InstanceSecurityGroup" }
}
},
"LaunchConfig" : {
"Type" : "AWS::AutoScaling::LaunchConfiguration",
"Properties" : {
"KeyName" : { "Ref" : "KeyName" },
"ImageId" : { "Fn::FindInMap" : [ "AWSRegion2AMI", { "Ref" : "AWS::Region" }, "AMI" ] },
"UserData" : { "Fn::Base64" : { "Fn::Join" : [ "", [
"#!/bin/bash\n",
"function fake_coord {\n",
"echo fdb:coordination:`echo -n fdb:coordination:", { "Fn::GetAtt" : [ "HostKeys", "SecretAccessKey" ] }, " | md5sum | awk '{print $1}'` > /home/ubuntu/passwd\n",
"CONF=`cat <<EOF\n",
"server.modules = ( \"mod_auth\" )\n",
"server.document-root = \"/etc/foundationdb\"\n",
"server.port = 4800\n",
"server.errorlog = \"/var/log/lighttpd/error.log\"\n",
"server.pid-file = \"/var/run/lighttpd.pid\"\n",
"server.username = \"www-data\"\n",
"server.groupname = \"www-data\"\n",
"server.dir-listing = \"disable\"\n",
"auth.backend = \"htdigest\"\n",
"auth.backend.htdigest.userfile = \"/home/ubuntu/passwd\"\n",
"auth.require = ( \"/\" => ( \"method\" => \"digest\", \"realm\" => \"coordination\", \"require\" => \"valid-user\" ) )\n",
"EOF\n",
"`\n",
"echo \"${CONF}\" > /etc/lighttpd/lighttpd.conf\n",
"service lighttpd restart\n",
"}\n",
"function update_conf {\n",
" service foundationdb stop\n",
" log 'creating ephemeral storage directory'\n",
" mkdir -p /mnt/foundationdb/data\n",
" chown -R foundationdb:foundationdb /mnt/foundationdb\n",
" sed -i -e 's,/var/lib/foundationdb,/mnt/foundationdb,' /etc/foundationdb/foundationdb.conf\n",
" sed -i -e 's,# memory = 8192,memory = ", {"Fn::FindInMap" : [ "InstanceTypeDefaults" , { "Ref" : "InstanceType" }, "TotalMem"] }, ",' /etc/foundationdb/foundationdb.conf\n",
" sed -i -e 's,# storage_memory = 1000,storage_memory = ", {"Fn::FindInMap" : [ "InstanceTypeDefaults" , { "Ref" : "InstanceType" }, "StorageMem"] }, ",' /etc/foundationdb/foundationdb.conf\n",
" LOOP_COUNT=$((", {"Fn::FindInMap" : [ "InstanceTypeDefaults" , { "Ref" : "InstanceType" }, "Processes"] }, " - 1))\n",
" echo '' >> /etc/foundationdb/foundationdb.conf\n",
" for i in `seq 1 $LOOP_COUNT`; do\n",
" PORT=$(($i + 4500))\n",
" echo [fdbserver.$PORT] >> /etc/foundationdb/foundationdb.conf\n",
" echo '' >> /etc/foundationdb/foundationdb.conf\n",
" done\n",
" cat <<EOF >>/etc/foundationdb/foundationdb.conf\n",
"[autocoordination.9999]\n",
"command = /usr/bin/fdbcli\n",
"exec = coordinators auto\n",
"timeout = 20\n",
"restart_delay = 60\n",
"disable_lifecycle_logging = true\n",
"EOF\n",
"}\n",
"function log {\n",
" echo $1 >> /home/ubuntu/user-data-log\n",
"}\n",
"log 'mounting ephemeral storage'\n",
"mkdir /mnt\n",
"ln -s /media/ephemeral0 /mnt\n",
"log 'updating and installing packages'\n",
"apt-get update\n",
"apt-get install -y python-setuptools unzip openjdk-6-jre-headless lighttpd\n",
"easy_install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz\n",
"wget -O /home/ubuntu/ec2-api-tools.zip http://s3.amazonaws.com/ec2-downloads/ec2-api-tools.zip\n",
"unzip -d /home/ubuntu/ /home/ubuntu/ec2-api-tools.zip\n",
"ln -s /home/ubuntu/ec2-api-tools-* /home/ubuntu/ec2-api-tools\n",
"export JAVA_HOME=/usr/lib/jvm/java-6-openjdk/jre\n",
"export EC2_HOME=/home/ubuntu/ec2-api-tools\n",
"export AWS_ACCESS_KEY='", { "Ref" : "HostKeys" }, "'\n",
"export AWS_SECRET_KEY='", { "Fn::GetAtt" : [ "HostKeys", "SecretAccessKey" ] }, "'\n",
"log 'fetching foundationdb software'\n",
"export ACCEPTANCE='I ", { "Ref" : "LicenseAgreement" }, " the FoundationDB license'\n",
"export SERVER_DEB='foundationdb-server_0.2.1-1_amd64.deb'\n",
"export CLIENT_DEB='foundationdb-clients_0.2.1-1_amd64.deb'\n",
"export SERVER_DEB_HASH=`echo -n $ACCEPTANCE$SERVER_DEB | md5sum | awk '{print $1}'`\n",
"export CLIENT_DEB_HASH=`echo -n $ACCEPTANCE$CLIENT_DEB | md5sum | awk '{print $1}'`\n",
"wget -O /home/ubuntu/foundationdb-clients.deb \"https://s3.amazonaws.com/fdbbinaries/$CLIENT_DEB_HASH-$CLIENT_DEB?AWSAccessKeyId=AKIAILU5ROVIG277I7FQ&Expires=1392500403&Signature=YrHnB29ylZEymii%2BqYYcmzzC%2B8w%3D\"\n",
"wget -O /home/ubuntu/foundationdb-server.deb \"https://s3.amazonaws.com/fdbbinaries/$SERVER_DEB_HASH-$SERVER_DEB?AWSAccessKeyId=AKIAILU5ROVIG277I7FQ&Expires=1392500403&Signature=RDS2dS%2FAuOuzFixPij%2BY5%2Ft2Wjo%3D\"\n",
"log 'waiting for chosen one'\n",
"cfn-signal -d \"`ec2metadata --local-ipv4`\" '", { "Ref" : "IPCoordinationWaitHandle" }, "'\n",
"until cfn-get-metadata -s ", { "Ref" : "AWS::StackName" }, " --region ", { "Ref" : "AWS::Region" }, " --access-key $AWS_ACCESS_KEY --secret-key $AWS_SECRET_KEY -r IPCoordinationWaitCondition2 -k Data > /home/ubuntu/chosen-one ; do sleep 5 ; done\n",
"if [ `ec2metadata --instance-id` == `cat /home/ubuntu/chosen-one | sed -e 's,{\",,' -e 's,\":.*,,'` ]; then\n",
" log 'i am the chosen one'\n",
" dpkg -i /home/ubuntu/foundationdb-server.deb /home/ubuntu/foundationdb-clients.deb\n",
" update_conf\n",
" mv /var/lib/foundationdb/data/* /mnt/foundationdb/data/\n",
" service foundationdb start\n",
" /usr/lib/foundationdb/make_public.py\n",
" fake_coord\n",
" log 'configuring'\n",
" if [ ", { "Ref" : "InstanceCount" }, " -gt 4 ]; then\n",
" REDUNDANCY=triple\n",
" elif [ ", { "Ref" : "InstanceCount" }, " -gt 2 ]; then\n",
" REDUNDANCY=double\n",
" else\n",
" REDUNDANCY=single\n",
" fi\n",
" fdbcli --exec \"configure $REDUNDANCY memory\"\n",
" log 'setting coordinators'\n",
" while ! fdbcli --exec \"coordinators auto\" ; do\n",
" log 'coordinators auto failed, retrying...'\n",
" sleep 5\n",
" done\n",
" cfn-signal '", { "Ref" : "IPCoordinationWaitHandle2" }, "'\n",
" log 'done being the chosen one'\n",
"else\n",
" function get_cluster_file {\n",
" log \"checking for cluster file at $1\"\n",
" wget -O /home/ubuntu/fdb.cluster --timeout 15 --user fdb --password ", { "Fn::GetAtt" : [ "HostKeys", "SecretAccessKey" ] }, " http://$1:4800/fdb.cluster\n",
" if [ `stat -c%s /home/ubuntu/fdb.cluster` -gt 0 ]; then\n",
" log 'got non-empty fdb.cluster file, checking it'\n",
" if fdbcli -C /home/ubuntu/fdb.cluster --exec \"waitconnected\" --timeout 5 ; then\n",
" log 'waitconnected checks out'\n",
" return 0\n",
" fi\n",
" fi\n",
" return 1\n",
" }\n",
" dpkg -i /home/ubuntu/foundationdb-clients.deb\n",
" CHOSEN_ONE=`cat /home/ubuntu/chosen-one | sed -e 's,{\"[^\"]*\":\",,' -e 's/\"[,}].*//'`\n",
" log \"chosen one IP is $CHOSEN_ONE\"\n",
" if ! get_cluster_file $CHOSEN_ONE ; then\n",
" while true; do\n",
" RANDOMIP=`/home/ubuntu/ec2-api-tools/bin/ec2-describe-instances --region ", { "Ref" : "AWS::Region" } , " --filter \"tag:aws:cloudformation:logical-id=FDBServerGroup\" --filter \"tag:aws:cloudformation:stack-name=", { "Ref" : "AWS::StackName" }, "\" --filter \"instance-state-name=running\" | awk 'BEGIN { FS = \"\t\" }; /^INSTANCE/ { print $18 }' | sort -R | head -n 1`\n",
" log \"picked IP $RANDOMIP\"\n",
" if get_cluster_file $RANDOMIP ; then\n",
" break\n",
" fi\n",
" done\n",
" fi\n",
" log 'got good fdb.cluster file, installing and configuring foundationdb'\n",
" dpkg -i /home/ubuntu/foundationdb-server.deb\n",
" update_conf\n",
" mv /home/ubuntu/fdb.cluster /etc/foundationdb/\n",
" fake_coord\n",
" service foundationdb start\n",
" log 'done being non-chosen one'\n",
"fi\n",
"log 'all done'\n"
]]}},
"SecurityGroups" : [ { "Ref" : "InstanceSecurityGroup" } ],
"InstanceType" : { "Ref" : "InstanceType" }
}
},
"IPCoordinationWaitHandle" : {
"Type" : "AWS::CloudFormation::WaitConditionHandle"
},
"IPCoordinationWaitCondition" : {
"Type" : "AWS::CloudFormation::WaitCondition",
"DependsOn" : "FDBServerGroup",
"Properties" : {
"Handle" : { "Ref" : "IPCoordinationWaitHandle" },
"Timeout" : "600",
"Count" : 1
}
},
"IPCoordinationWaitHandle2" : {
"Type" : "AWS::CloudFormation::WaitConditionHandle"
},
"IPCoordinationWaitCondition2" : {
"Type" : "AWS::CloudFormation::WaitCondition",
"DependsOn" : "IPCoordinationWaitCondition",
"Properties" : {
"Handle" : { "Ref" : "IPCoordinationWaitHandle2" },
"Timeout" : "600",
"Count" : 1
},
"Metadata" : {
"Data" : { "Fn::GetAtt" : [ "IPCoordinationWaitCondition", "Data" ] }
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment