3-node setup for k8s cluster w/ cloud provider support

aws-cfn-3-node-k8s-cluster.yaml

AWSTemplateFormatVersion: 2010-09-09
Description: A 3-node setup for k8s cluster w/ cloud provider support
##!/bin/bash
#set -ex
#if [ "${1}" = "create" ]; then
#    aws cloudformation create-stack \
#    --template-body file://cfn-elk-cluster.yaml \
#    --stack-name k8s-test-cluster \
#    --capabilities CAPABILITY_NAMED_IAM \
#    --parameters \
#    ParameterKey=KeyName,ParameterValue=kaas-ci
#elif [ "${1}" = "wait" ]; then
#    aws cloudformation wait stack-create-complete --stack-name k8s-test-cluster
#elif [ "${1}" = "delete" ]; then
#    aws cloudformation delete-stack --stack-name k8s-test-cluster
#fi

Parameters:
  VPNPublicIp:
    Description: Public IP of VPN used to connect to Web UI
    Type: String
    Default: 0.0.0.0/0  # All
  VPCCidr:
    Description: IPv4 CIDR of AWS VPC
    Type: String
    Default: 10.0.0.0/16
  KeyName:
    Description: EC2 key-pair to SSH on instance
    Type: AWS::EC2::KeyPair::KeyName
  ClusterInstanceType:
    Description: EC2 instance type for Master/Worker nodes
    Type: String
    Default: c5d.large
  BaseImage:
    Description: Image ID for creating instance
    Type: String
    Default: ami-033a0960d9d83ead0 

Resources:
  # VPC infra############################################
  VnaumovVPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: !Ref VPCCidr
      EnableDnsSupport: true
      EnableDnsHostnames: true
      InstanceTenancy: default
      Tags:
       - Key: Name
         Value: vnaumov-test
       # Needed for AWS cloud provider
       - Key: kubernetes.io/cluster/kubernetes
         Value: owned
  GatewayVPC:
    Type: AWS::EC2::InternetGateway
    Properties:
      Tags:
       # Needed for AWS cloud provider
       - Key: kubernetes.io/cluster/kubernetes
         Value: owned
  GatewayAttachment:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      VpcId: !Ref VnaumovVPC
      InternetGatewayId: !Ref GatewayVPC
  # VPC routing
  RouteTableVPC:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref VnaumovVPC
      Tags:
       # Needed for AWS cloud provider
       - Key: kubernetes.io/cluster/kubernetes
         Value: owned
  InternetRouteVPC:
    Type: AWS::EC2::Route
    DependsOn: GatewayVPC
    Properties:
      DestinationCidrBlock: 0.0.0.0/0
      GatewayId: !Ref GatewayVPC
      RouteTableId: !Ref RouteTableVPC
  # VPC Subnet
  SubnetVPC:
    Type: AWS::EC2::Subnet
    Properties:
      AvailabilityZone: 'us-east-2a'
      CidrBlock: !Ref VPCCidr
      VpcId: !Ref VnaumovVPC
      MapPublicIpOnLaunch: true
      Tags:
       - Key: Name
         Value: vnaumov-k8s-subnet
       # Needed for AWS cloud provider
       - Key: kubernetes.io/cluster/kubernetes
         Value: owned
  SubnetARouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId: !Ref RouteTableVPC
      SubnetId: !Ref SubnetVPC
  # VPC infra############################################
  # IAM roles############################################
  IAMK8sGroup:
    Type: AWS::IAM::Group
    Properties:
      GroupName: k8s-cluster-test-iam-group
  IAMMasterPolicy:
    Type: AWS::IAM::ManagedPolicy
    DependsOn: IAMK8sGroup
    Properties:
      ManagedPolicyName: k8s-cluster-iam-master-policy
      Groups:
        - k8s-cluster-test-iam-group
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - 'autoscaling:DescribeAutoScalingGroups'
              - 'autoscaling:DescribeLaunchConfigurations'
              - 'autoscaling:DescribeTags'
              - 'ec2:DescribeInstances'
              - 'ec2:DescribeRegions'
              - 'ec2:DescribeRouteTables'
              - 'ec2:DescribeSecurityGroups'
              - 'ec2:DescribeSubnets'
              - 'ec2:DescribeVolumes'
              - 'ec2:CreateSecurityGroup'
              - 'ec2:CreateTags'
              - 'ec2:CreateVolume'
              - 'ec2:ModifyInstanceAttribute'
              - 'ec2:ModifyVolume'
              - 'ec2:AttachVolume'
              - 'ec2:AuthorizeSecurityGroupIngress'
              - 'ec2:CreateRoute'
              - 'ec2:DeleteRoute'
              - 'ec2:DeleteSecurityGroup'
              - 'ec2:DeleteVolume'
              - 'ec2:DetachVolume'
              - 'ec2:RevokeSecurityGroupIngress'
              - 'ec2:DescribeVpcs'
              - 'elasticloadbalancing:AddTags'
              - 'elasticloadbalancing:AttachLoadBalancerToSubnets'
              - 'elasticloadbalancing:ApplySecurityGroupsToLoadBalancer'
              - 'elasticloadbalancing:CreateLoadBalancer'
              - 'elasticloadbalancing:CreateLoadBalancerPolicy'
              - 'elasticloadbalancing:CreateLoadBalancerListeners'
              - 'elasticloadbalancing:ConfigureHealthCheck'
              - 'elasticloadbalancing:DeleteLoadBalancer'
              - 'elasticloadbalancing:DeleteLoadBalancerListeners'
              - 'elasticloadbalancing:DescribeLoadBalancers'
              - 'elasticloadbalancing:DescribeLoadBalancerAttributes'
              - 'elasticloadbalancing:DetachLoadBalancerFromSubnets'
              - 'elasticloadbalancing:DeregisterInstancesFromLoadBalancer'
              - 'elasticloadbalancing:ModifyLoadBalancerAttributes'
              - 'elasticloadbalancing:RegisterInstancesWithLoadBalancer'
              - 'elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer'
              - 'elasticloadbalancing:AddTags'
              - 'elasticloadbalancing:CreateListener'
              - 'elasticloadbalancing:CreateTargetGroup'
              - 'elasticloadbalancing:DeleteListener'
              - 'elasticloadbalancing:DeleteTargetGroup'
              - 'elasticloadbalancing:DescribeListeners'
              - 'elasticloadbalancing:DescribeLoadBalancerPolicies'
              - 'elasticloadbalancing:DescribeTargetGroups'
              - 'elasticloadbalancing:DescribeTargetHealth'
              - 'elasticloadbalancing:ModifyListener'
              - 'elasticloadbalancing:ModifyTargetGroup'
              - 'elasticloadbalancing:RegisterTargets'
              - 'elasticloadbalancing:SetLoadBalancerPoliciesOfListener'
              - 'iam:CreateServiceLinkedRole'
              - 'kms:DescribeKey'
            Resource: '*'
  IAMWorkerPolicy:
    Type: AWS::IAM::ManagedPolicy
    DependsOn: IAMK8sGroup
    Properties:
      ManagedPolicyName: k8s-cluster-iam-worker-policy
      Groups:
        - k8s-cluster-test-iam-group
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - 'ec2:DescribeInstances'
              - 'ec2:DescribeRegions'
              - 'ecr:GetAuthorizationToken'
              - 'ecr:BatchCheckLayerAvailability'
              - 'ecr:GetDownloadUrlForLayer'
              - 'ecr:GetRepositoryPolicy'
              - 'ecr:DescribeRepositories'
              - 'ecr:ListImages'
              - 'ecr:BatchGetImage'
            Resource: '*'
  IAMMasterRole:
    Type: AWS::IAM::Role
    DependsOn: IAMMasterPolicy
    Properties:
      Description: 'Cloud provider permissons for k8s control plane node'
      # EC2 only
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
              - ec2.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      ManagedPolicyArns:
        - !Ref IAMMasterPolicy
      RoleName: k8s-cluster-iam-master-role
      Tags:
       - Key: Name
         Value: k8s-test-iam
  IAMWorkerRole:
    Type: AWS::IAM::Role
    DependsOn: IAMWorkerPolicy
    Properties:
      Description: 'Cloud provider permissons for k8s worker node'
      # EC2 only
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
              - ec2.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      ManagedPolicyArns:
        - !Ref IAMWorkerPolicy
      RoleName: k8s-cluster-iam-worker-role
      Tags:
       - Key: Name
         Value: k8s-test-iam
  # pack roles into instace profiles to assign to instance
  MasterInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    DependsOn: IAMMasterRole
    Properties:
      InstanceProfileName: k8s-instance-profile-master
      Roles:
        - k8s-cluster-iam-master-role
  WorkerInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    DependsOn: IAMWorkerRole
    Properties:
      InstanceProfileName: k8s-instance-profile-worker
      Roles:
        - k8s-cluster-iam-worker-role
  # IAM roles############################################
  # Nodes
  MasterNode:
    Type: AWS::EC2::Instance
    DependsOn: 
      - VnaumovVPC
      - IAMMasterRole
    Properties:
      SubnetId: !Ref SubnetVPC
      KeyName: !Ref KeyName
      InstanceType: !Ref ClusterInstanceType
      ImageId: !Ref BaseImage
      SecurityGroupIds:
        - !Ref K8sClusterSecurityGroup
        - !Ref SSHSecurityGroup
      IamInstanceProfile: k8s-instance-profile-master 
      Tags:
       - Key: Name
         Value: K8s-Master
  WorkerNode1:
    Type: AWS::EC2::Instance
    DependsOn: 
      - VnaumovVPC
      - IAMWorkerRole
    Properties:
      SubnetId: !Ref SubnetVPC
      KeyName: !Ref KeyName
      InstanceType: !Ref ClusterInstanceType
      ImageId: !Ref BaseImage
      SecurityGroupIds:
        - !Ref K8sClusterSecurityGroup
        - !Ref SSHSecurityGroup
      IamInstanceProfile: k8s-instance-profile-worker
      Tags:
       - Key: Name
         Value: K8s-Worker1
  WorkerNode2:
    Type: AWS::EC2::Instance
    DependsOn: 
      - VnaumovVPC
      - IAMWorkerRole
    Properties:
      SubnetId: !Ref SubnetVPC
      KeyName: !Ref KeyName
      InstanceType: !Ref ClusterInstanceType
      ImageId: !Ref BaseImage
      SecurityGroupIds:
        - !Ref K8sClusterSecurityGroup
        - !Ref SSHSecurityGroup
      IamInstanceProfile: k8s-instance-profile-worker
      Tags:
       - Key: Name
         Value: K8s-Worker2
  MasterNodeIPAddress:
    Type: AWS::EC2::EIP
    DependsOn: VnaumovVPC
    Properties:
      Domain: vpc
  MasterNodeIPAssoc:
    Type: AWS::EC2::EIPAssociation
    DependsOn: VnaumovVPC
    Properties:
      InstanceId: !Ref MasterNode
      EIP: !Ref MasterNodeIPAddress
  WorkerNode1IPAddress:
    Type: AWS::EC2::EIP
    DependsOn: VnaumovVPC
    Properties:
      Domain: vpc
  WorkerNode1IPAssoc:
    Type: AWS::EC2::EIPAssociation
    DependsOn: VnaumovVPC
    Properties:
      InstanceId: !Ref WorkerNode1
      EIP: !Ref WorkerNode1IPAddress
  WorkerNode2IPAddress:
    Type: AWS::EC2::EIP
    DependsOn: VnaumovVPC
    Properties:
      Domain: vpc
  WorkerNode2IPAssoc:
    Type: AWS::EC2::EIPAssociation
    DependsOn: VnaumovVPC
    Properties:
      InstanceId: !Ref WorkerNode2
      EIP: !Ref WorkerNode2IPAddress
  # Security groups
  SSHSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    DependsOn: VnaumovVPC
    Properties:
      VpcId: !Ref VnaumovVPC
      GroupDescription: Enable SSH access
      SecurityGroupIngress:
        - CidrIp: !Ref VPNPublicIp
          IpProtocol: tcp
          FromPort: 22
          ToPort: 22
  K8sClusterSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    DependsOn: VnaumovVPC
    Properties:
      VpcId: !Ref VnaumovVPC
      GroupDescription: Allow all k8s used ports for master/workers
      Tags:
       - Key: Name
         Value: k8s-cluster-default
      SecurityGroupIngress:
        # pingable
        - CidrIp: !Ref VPCCidr
          IpProtocol: icmp
          FromPort: -1
          ToPort: -1
        # k8s apiserver
        - CidrIp: !Ref VPCCidr
          IpProtocol: tcp
          FromPort: 6443
          ToPort: 6443
        # k8s etcd
        - CidrIp: !Ref VPCCidr
          IpProtocol: tcp
          FromPort: 2379
          ToPort: 2380
        # k8s kubelet health
        - CidrIp: !Ref VPCCidr
          IpProtocol: tcp
          FromPort: 10250
          ToPort: 10250
        # k8s kubelet api
        - CidrIp: !Ref VPCCidr
          IpProtocol: tcp
          FromPort: 10255
          ToPort: 10255
        # k8s controller manager
        - CidrIp: !Ref VPCCidr
          IpProtocol: tcp
          FromPort: 10252
          ToPort: 10252
        # k8s scheduler
        - CidrIp: !Ref VPCCidr
          IpProtocol: tcp
          FromPort: 10251
          ToPort: 10251
Outputs:
  MasterIPAddress:
    Description: IP address of the newly created k8s controlplane instance
    Value: !Ref MasterNodeIPAddress
  Worker1IPAddress:
    Description: IP address of the newly created k8s worker instance
    Value: !Ref WorkerNode1IPAddress
  Worker2IPAddress:
    Description: IP address of the newly created k8s worker instance
    Value: !Ref WorkerNode2IPAddress