Last active
June 25, 2018 08:27
-
-
Save Yggdrasil/d10cf4f2070b69c005bfd55ac33eb4cf to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| AWSTemplateFormatVersion: '2010-09-09' | |
| Description: 'Spotinst Elastigroup for Amazon EKS' | |
| Parameters: | |
| AccessToken: | |
| Type: String | |
| Description: Provide Spotinst API Token | |
| AccountID: | |
| Type: String | |
| Description: Provide Spotinst Account ID | |
| KeyName: | |
| Description: The EC2 Key Pair to allow SSH access to the instances | |
| Type: AWS::EC2::KeyPair::KeyName | |
| ClusterOrientation: | |
| Type: String | |
| AllowedValues: | |
| - balanced | |
| - availabilityOriented | |
| - costOriented | |
| Default: balanced | |
| Description: Specify Cluster Orientation | |
| SpotPercentage: | |
| Type: Number | |
| MinValue: '0' | |
| MaxValue: '100' | |
| Default: '100' | |
| Description: Spot Instances Percentage in the Cluster | |
| ConstraintDescription: Select a Number Between 0 and 100 | |
| DetailedMonitoring: | |
| Type: String | |
| AllowedValues: | |
| - 'True' | |
| - 'False' | |
| Default: 'False' | |
| Description: Allow Detailed CloudWatch Monitoring? | |
| OnDemandInstanceType: | |
| Description: Which on-demand instance type should we use to base Elastigroup upon? | |
| Type: String | |
| Default: t2.medium | |
| AllowedValues: | |
| - t2.small | |
| - t2.medium | |
| - t2.large | |
| - t2.xlarge | |
| - t2.2xlarge | |
| - m3.medium | |
| - m3.large | |
| - m3.xlarge | |
| - m3.2xlarge | |
| - m4.large | |
| - m4.xlarge | |
| - m4.2xlarge | |
| - m4.4xlarge | |
| - m4.10xlarge | |
| - m5.large | |
| - m5.xlarge | |
| - m5.2xlarge | |
| - m5.4xlarge | |
| - m5.12xlarge | |
| - m5.24xlarge | |
| - c4.large | |
| - c4.xlarge | |
| - c4.2xlarge | |
| - c4.4xlarge | |
| - c4.8xlarge | |
| - c5.large | |
| - c5.xlarge | |
| - c5.2xlarge | |
| - c5.4xlarge | |
| - c5.9xlarge | |
| - c5.18xlarge | |
| - i3.large | |
| - i3.xlarge | |
| - i3.2xlarge | |
| - i3.4xlarge | |
| - i3.8xlarge | |
| - i3.16xlarge | |
| - r3.xlarge | |
| - r3.2xlarge | |
| - r3.4xlarge | |
| - r3.8xlarge | |
| - r4.large | |
| - r4.xlarge | |
| - r4.2xlarge | |
| - r4.4xlarge | |
| - r4.8xlarge | |
| - r4.16xlarge | |
| - x1.16xlarge | |
| - x1.32xlarge | |
| - p2.xlarge | |
| - p2.8xlarge | |
| - p2.16xlarge | |
| - p3.2xlarge | |
| - p3.8xlarge | |
| - p3.16xlarge | |
| ConstraintDescription: must be a valid EC2 instance type | |
| SpotInstancesType: | |
| Type: CommaDelimitedList | |
| Default: c4.large,c5.large | |
| Description: Provide a comma separated list of Spot Instances Available to Elastigroup | |
| CapacityTarget: | |
| Type: Number | |
| Description: Desired Amount of Instances in the Cluster | |
| CapacityMin: | |
| Type: Number | |
| Description: Minimum Amount of Instances in the Cluster | |
| CapacityMax: | |
| Type: Number | |
| Description: Maximum Amount of Instances in the Cluster | |
| EKSClusterName: | |
| Description: The cluster name provided when the cluster was created. If it is incorrect, nodes will not be able to join the cluster. | |
| Type: String | |
| ElastigroupName: | |
| Type: String | |
| Description: Provide a Name for the Elastigroup | |
| ClusterControlPlaneSecurityGroup: | |
| Description: The security group of the cluster control plane. | |
| Type: AWS::EC2::SecurityGroup::Id | |
| VpcId: | |
| Description: The VPC of the worker instances | |
| Type: AWS::EC2::VPC::Id | |
| SubnetIds: | |
| Type: List<AWS::EC2::Subnet::Id> | |
| Description: Provide Subnet IDs for the Cluster (Must be from selected VPC) | |
| Mappings: | |
| AWSRegionToAMI: | |
| us-east-2: | |
| AMI: ami-956e52f0 | |
| us-east-1: | |
| AMI: ami-dea4d5a1 | |
| us-west-2: | |
| AMI: ami-73a6e20b | |
| us-west-1: | |
| AMI: ami-6b81980b | |
| eu-west-3: | |
| AMI: ami-ca75c4b7 | |
| eu-west-2: | |
| AMI: ami-3622cf51 | |
| eu-west-1: | |
| AMI: ami-c91624b0 | |
| eu-central-1: | |
| AMI: ami-10e6c8fb | |
| ap-northeast-2: | |
| AMI: ami-7c69c112 | |
| ap-northeast-1: | |
| AMI: ami-f3f8098c | |
| ap-southeast-2: | |
| AMI: ami-bc04d5de | |
| ap-southeast-1: | |
| AMI: ami-b75a6acb | |
| ca-central-1: | |
| AMI: ami-da6cecbe | |
| ap-south-1: | |
| AMI: ami-c7072aa8 | |
| sa-east-1: | |
| AMI: ami-a1e2becd | |
| Metadata: | |
| AWS::CloudFormation::Interface: | |
| ParameterGroups: | |
| - Label: | |
| default: Spotinst Configuraiton | |
| Parameters: | |
| - AccountID | |
| - AccessToken | |
| - Label: | |
| default: "EKS Cluster" | |
| Parameters: | |
| - EKSClusterName | |
| - ClusterControlPlaneSecurityGroup | |
| - Label: | |
| default: "Elastigroup for Worker Node Configuration" | |
| Parameters: | |
| - ElastigroupName | |
| - CapacityTarget | |
| - CapacityMin | |
| - CapacityMax | |
| - OnDemandInstanceType | |
| - SpotInstancesType | |
| - KeyName | |
| - ClusterOrientation | |
| - SpotPercentage | |
| - DetailedMonitoring | |
| - Label: | |
| default: "Worker Network Configuration" | |
| Parameters: | |
| - VpcId | |
| - SubnetIds | |
| Resources: | |
| NodeInstanceProfile: | |
| Type: AWS::IAM::InstanceProfile | |
| Properties: | |
| Path: "/" | |
| Roles: | |
| - !Ref NodeInstanceRole | |
| NodeInstanceRole: | |
| Type: AWS::IAM::Role | |
| Properties: | |
| AssumeRolePolicyDocument: | |
| Version: '2012-10-17' | |
| Statement: | |
| - Effect: Allow | |
| Principal: | |
| Service: | |
| - ec2.amazonaws.com | |
| Action: | |
| - sts:AssumeRole | |
| Path: "/" | |
| ManagedPolicyArns: | |
| - arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy | |
| - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy | |
| - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly | |
| NodeSecurityGroup: | |
| Type: AWS::EC2::SecurityGroup | |
| Properties: | |
| GroupDescription: Security group for all nodes in the cluster | |
| VpcId: | |
| !Ref VpcId | |
| Tags: | |
| - Key: !Sub "kubernetes.io/cluster/${EKSClusterName}" | |
| Value: 'owned' | |
| NodeSecurityGroupIngress: | |
| Type: AWS::EC2::SecurityGroupIngress | |
| DependsOn: NodeSecurityGroup | |
| Properties: | |
| Description: Allow node to communicate with each other | |
| GroupId: !Ref NodeSecurityGroup | |
| SourceSecurityGroupId: !Ref NodeSecurityGroup | |
| IpProtocol: '-1' | |
| FromPort: 0 | |
| ToPort: 65535 | |
| NodeSecurityGroupFromControlPlaneIngress: | |
| Type: AWS::EC2::SecurityGroupIngress | |
| DependsOn: NodeSecurityGroup | |
| Properties: | |
| Description: Allow worker Kubelets and pods to receive communication from the cluster control plane | |
| GroupId: !Ref NodeSecurityGroup | |
| SourceSecurityGroupId: !Ref ClusterControlPlaneSecurityGroup | |
| IpProtocol: tcp | |
| FromPort: 1025 | |
| ToPort: 65535 | |
| ControlPlaneEgressToNodeSecurityGroup: | |
| Type: AWS::EC2::SecurityGroupEgress | |
| DependsOn: NodeSecurityGroup | |
| Properties: | |
| Description: Allow the cluster control plane to communicate with worker Kubelet and pods | |
| GroupId: !Ref ClusterControlPlaneSecurityGroup | |
| DestinationSecurityGroupId: !Ref NodeSecurityGroup | |
| IpProtocol: tcp | |
| FromPort: 1025 | |
| ToPort: 65535 | |
| ClusterControlPlaneSecurityGroupIngress: | |
| Type: AWS::EC2::SecurityGroupIngress | |
| DependsOn: NodeSecurityGroup | |
| Properties: | |
| Description: Allow pods to communicate with the cluster API Server | |
| GroupId: !Ref ClusterControlPlaneSecurityGroup | |
| SourceSecurityGroupId: !Ref NodeSecurityGroup | |
| IpProtocol: tcp | |
| ToPort: 443 | |
| FromPort: 443 | |
| SpotinstElastigroup: | |
| Type: Custom::elastigroup | |
| Properties: | |
| ServiceToken: !Sub 'arn:aws:lambda:${AWS::Region}:178579023202:function:spotinst-cloudformation' | |
| accessToken: !Ref 'AccessToken' | |
| accountId: !Ref 'AccountID' | |
| group: | |
| name: !Ref 'ElastigroupName' | |
| region: !Ref 'AWS::Region' | |
| strategy: | |
| risk: !Ref 'SpotPercentage' | |
| availabilityVsCost: !Ref 'ClusterOrientation' | |
| capacity: | |
| target: !Ref 'CapacityTarget' | |
| minimum: !Ref 'CapacityMin' | |
| maximum: !Ref 'CapacityMax' | |
| scaling: {} | |
| compute: | |
| instanceTypes: | |
| ondemand: !Ref 'OnDemandInstanceType' | |
| spot: !Ref 'SpotInstancesType' | |
| subnetIds: !Ref 'SubnetIds' | |
| launchSpecification: | |
| tags: | |
| - tagKey: Name | |
| tagValue: !Sub '${EKSClusterName}-${ElastigroupName}-Node' | |
| - tagKey: !Sub 'kubernetes.io/cluster/${EKSClusterName}' | |
| tagValue: owned | |
| monitoring: !Ref 'DetailedMonitoring' | |
| imageId: !FindInMap [AWSRegionToAMI, !Ref "AWS::Region", AMI] | |
| keyPair: !Ref 'KeyName' | |
| networkInterfaces: | |
| - deviceIndex: 0 | |
| associatePublicIpAddress: true | |
| deleteOnTermination: true | |
| associateIpv6Address: false | |
| securityGroupIds: [ !Ref NodeSecurityGroup ] | |
| iamRole: | |
| arn: !GetAtt 'NodeInstanceProfile.Arn' | |
| userData: | |
| Fn::Base64: | |
| Fn::Join: [ | |
| "", | |
| [ | |
| "#!/bin/bash -xe\n", | |
| "CA_CERTIFICATE_DIRECTORY=/etc/kubernetes/pki", "\n", | |
| "CA_CERTIFICATE_FILE_PATH=$CA_CERTIFICATE_DIRECTORY/ca.crt", "\n", | |
| "MODEL_DIRECTORY_PATH=~/.aws/eks", "\n", | |
| "MODEL_FILE_PATH=$MODEL_DIRECTORY_PATH/eks-2017-11-01.normal.json", "\n", | |
| "mkdir -p $CA_CERTIFICATE_DIRECTORY", "\n", | |
| "mkdir -p $MODEL_DIRECTORY_PATH", "\n", | |
| "curl -o $MODEL_FILE_PATH https://s3-us-west-2.amazonaws.com/amazon-eks/1.10.3/2018-06-05/eks-2017-11-01.normal.json", "\n", | |
| "aws configure add-model --service-model file://$MODEL_FILE_PATH --service-name eks", "\n", | |
| "aws eks describe-cluster --region=", { Ref: "AWS::Region" }," --name=", { Ref: EKSClusterName }," --query 'cluster.{certificateAuthorityData: certificateAuthority.data, endpoint: endpoint}' > /tmp/describe_cluster_result.json", "\n", | |
| "cat /tmp/describe_cluster_result.json | grep certificateAuthorityData | awk '{print $2}' | sed 's/[,\"]//g' | base64 -d > $CA_CERTIFICATE_FILE_PATH", "\n", | |
| "MASTER_ENDPOINT=$(cat /tmp/describe_cluster_result.json | grep endpoint | awk '{print $2}' | sed 's/[,\"]//g')", "\n", | |
| "INTERNAL_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)", "\n", | |
| "INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type)", "\n", | |
| "sed -i s,MASTER_ENDPOINT,$MASTER_ENDPOINT,g /var/lib/kubelet/kubeconfig", "\n", | |
| "sed -i s,CLUSTER_NAME,", { Ref: EKSClusterName }, ",g /var/lib/kubelet/kubeconfig", "\n", | |
| "sed -i s,REGION,", { Ref: "AWS::Region" }, ",g /etc/systemd/system/kubelet.service", "\n", | |
| "if [ \"$INSTANCE_TYPE\" == \"c4.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"c4.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"c5.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"c5.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"i3.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"i3.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m3.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m4.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m4.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m5.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m5.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"p2.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"p3.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r3.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r3.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r4.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r4.2xlarge\" ]" , "\n" , | |
| "then" , "\n", | |
| "sed -i s,MAX_PODS,", 58, ",g /etc/systemd/system/kubelet.service", "\n", | |
| "elif [ \"$INSTANCE_TYPE\" == \"c4.large\" ] || [ \"$INSTANCE_TYPE\" == \"c5.large\" ] || [ \"$INSTANCE_TYPE\" == \"i3.large\" ] || [ \"$INSTANCE_TYPE\" == \"m3.large\" ] || [ \"$INSTANCE_TYPE\" == \"m5.large\" ] || [ \"$INSTANCE_TYPE\" == \"r4.large\" ]" , "\n", | |
| "then" , "\n", | |
| "sed -i s,MAX_PODS,", 29, ",g /etc/systemd/system/kubelet.service", "\n", | |
| "elif [ \"$INSTANCE_TYPE\" == \"c4.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"c4.8xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"c5.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"c5.9xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"i3.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"i3.8xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m4.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m4.10xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m5.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m5.12xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"p2.8xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"p2.16xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"p3.8xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"p3.16xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r3.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r3.8xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r4.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r4.8xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"x1.16xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"x1.32xlarge\" ]" , "\n", | |
| "then" , "\n", | |
| "sed -i s,MAX_PODS,", 234, ",g /etc/systemd/system/kubelet.service", "\n", | |
| "elif [ \"$INSTANCE_TYPE\" == \"c5.18xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"i3.16xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m5.24xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r4.16xlarge\"]" , "\n", | |
| "then" , "\n", | |
| "sed -i s,MAX_PODS,", 737, ",g /etc/systemd/system/kubelet.service", "\n", | |
| "elif [ \"$INSTANCE_TYPE\" == \"m3.medium\" ]" , "\n", | |
| "then" , "\n", | |
| "sed -i s,MAX_PODS,", 12, ",g /etc/systemd/system/kubelet.service", "\n", | |
| "elif [ \"$INSTANCE_TYPE\" == \"m3.2xlarge\" ]" , "\n", | |
| "then" , "\n", | |
| "sed -i s,MAX_PODS,", 118, ",g /etc/systemd/system/kubelet.service", "\n", | |
| "elif [ \"$INSTANCE_TYPE\" == \"m4.large\" ]" , "\n", | |
| "then" , "\n", | |
| "sed -i s,MAX_PODS,", 20, ",g /etc/systemd/system/kubelet.service", "\n", | |
| "elif [ \"$INSTANCE_TYPE\" == \"t2.small\" ]" , "\n", | |
| "then" , "\n", | |
| "sed -i s,MAX_PODS,", 8, ",g /etc/systemd/system/kubelet.service", "\n", | |
| "elif [ \"$INSTANCE_TYPE\" == \"t2.medium\" ]" , "\n", | |
| "then" , "\n", | |
| "sed -i s,MAX_PODS,", 17, ",g /etc/systemd/system/kubelet.service", "\n", | |
| "elif [ \"$INSTANCE_TYPE\" == \"t2.large\" ]" , "\n", | |
| "then" , "\n", | |
| "sed -i s,MAX_PODS,", 35, ",g /etc/systemd/system/kubelet.service", "\n", | |
| "elif [ \"$INSTANCE_TYPE\" == \"t2.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"t2.2xlarge\" ]" , "\n", | |
| "then" , "\n", | |
| "sed -i s,MAX_PODS,", 44, ",g /etc/systemd/system/kubelet.service", "\n", | |
| "else" , "\n", | |
| "sed -i s,MAX_PODS,", 58, ",g /etc/systemd/system/kubelet.service", "\n", | |
| "fi" , "\n", | |
| "sed -i s,MASTER_ENDPOINT,$MASTER_ENDPOINT,g /etc/systemd/system/kubelet.service", "\n", | |
| "sed -i s,INTERNAL_IP,$INTERNAL_IP,g /etc/systemd/system/kubelet.service", "\n", | |
| "DNS_CLUSTER_IP=10.100.0.10", "\n", | |
| "if [[ $INTERNAL_IP == 10.* ]] ; then DNS_CLUSTER_IP=172.20.0.10; fi", "\n", | |
| "sed -i s,DNS_CLUSTER_IP,$DNS_CLUSTER_IP,g /etc/systemd/system/kubelet.service", "\n", | |
| "sed -i s,CERTIFICATE_AUTHORITY_FILE,$CA_CERTIFICATE_FILE_PATH,g /var/lib/kubelet/kubeconfig" , "\n", | |
| "sed -i s,CLIENT_CA_FILE,$CA_CERTIFICATE_FILE_PATH,g /etc/systemd/system/kubelet.service" , "\n", | |
| "systemctl daemon-reload", "\n", | |
| "systemctl restart kubelet", "\n", | |
| "/opt/aws/bin/cfn-signal -e $? ", | |
| " --stack ", { Ref: "AWS::StackName" }, | |
| " --resource NodeGroup ", | |
| " --region ", { Ref: "AWS::Region" }, "\n" | |
| ] | |
| ] | |
| product: Linux/UNIX | |
| scheduling: {} | |
| thirdPartiesIntegration: | |
| kubernetes: | |
| integrationMode: pod | |
| clusterIdentifier: !Ref 'ElastigroupName' | |
| autoScale: | |
| isEnabled: true | |
| isAutoConfig: true | |
| Outputs: | |
| NodeInstanceRole: | |
| Description: The node instance role | |
| Value: !GetAtt NodeInstanceRole.Arn | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment