Private EKS Worker Nodes - CloudFormation

private-eks-nodes.yaml

###
#
# CloudFormation Template to create EKS Worker Nodes without access to an IGW
#
# The following CloudFormation template is based upon the AWS-provided CloudFormation template for creating 
# EKS worker nodes in your VPC.  It differs in that it has user-data modified to configure a node to operate 
# within a VPC that has no IGW.  
#
# To use this template you will need a VPC with an EKS cluster already deployed and private access enabled. 
# You will also need a web proxy configured to allow the worker nodes access to the AWS EKS service API.  This
# need for a proxy server can be removed if you provide the bootstrap.sh call invoked in the USER DATA with
# the cluster CA and endpoint URL.  This script also assumes that the following VPC endpoints are configured
# in your VPC: ECR, ECR Docker, and EC2.
#
# The usual EKS deployment steps in the AWS documentation still apply, simply use this CloudFormation template
# in place of the AWS provided one when deploying the worker nodes.
# In summary the process is:
#   1. Create a VPC with only private subnets
#   2. Create VPC endpoints for the ecr, dkr.ecr, and ec2 APIs
#   3. Provide a web proxy for the EKS service API
#   4. Deploy an EKS cluster into the VPC with private access enabled
#   5. Edit the aws-node daemonset to only pull images if not present
#      e.g. $ kubectl edit ds/aws-node -n kube-system
#   6. Deploy this template, specifying proxy url and security group granting access to VPC endpoints
#   7. Add the worker instance role to the authentiation config map for the cluster
#      e.g. $ kubectl apply -f aws-auth-cm.yaml
#
# NOTE:
# This CloudFormation template can be enhanced by providing the EKS cluster CA and endpoint url to the 
# bootstrap.sh script invoked on the worker node by the user data.  
# 
# This script has been tested using EKS v1.11.  It may be possible to get the Docker daemon working with ECR
# via proxy server or the VPC endpoint however I have not tested this.  
#
###

AWSTemplateFormatVersion: 2010-09-09
Description: Amazon EKS - Node Group
Parameters:
  KeyName:
    Description: The EC2 Key Pair to allow SSH access to the instances
    Type: 'AWS::EC2::KeyPair::KeyName'
  NodeImageId:
    Description: AMI id for the node instances.
    Type: 'AWS::EC2::Image::Id'
  NodeInstanceType:
    Description: EC2 instance type for the node instances
    Type: String
    Default: t3.medium
    ConstraintDescription: Must be a valid EC2 instance type
    AllowedValues:
      - t2.small
      - t2.medium
      - t2.large
      - t2.xlarge
      - t2.2xlarge
      - t3.nano
      - t3.micro
      - t3.small
      - t3.medium
      - t3.large
      - t3.xlarge
      - t3.2xlarge
      - m3.medium
      - m3.large
      - m3.xlarge
      - m3.2xlarge
      - m4.large
      - m4.xlarge
      - m4.2xlarge
      - m4.4xlarge
      - m4.10xlarge
      - m5.large
      - m5.xlarge
      - m5.2xlarge
      - m5.4xlarge
      - m5.12xlarge
      - m5.24xlarge
      - c4.large
      - c4.xlarge
      - c4.2xlarge
      - c4.4xlarge
      - c4.8xlarge
      - c5.large
      - c5.xlarge
      - c5.2xlarge
      - c5.4xlarge
      - c5.9xlarge
      - c5.18xlarge
      - i3.large
      - i3.xlarge
      - i3.2xlarge
      - i3.4xlarge
      - i3.8xlarge
      - i3.16xlarge
      - r3.xlarge
      - r3.2xlarge
      - r3.4xlarge
      - r3.8xlarge
      - r4.large
      - r4.xlarge
      - r4.2xlarge
      - r4.4xlarge
      - r4.8xlarge
      - r4.16xlarge
      - x1.16xlarge
      - x1.32xlarge
      - p2.xlarge
      - p2.8xlarge
      - p2.16xlarge
      - p3.2xlarge
      - p3.8xlarge
      - p3.16xlarge
      - p3dn.24xlarge
      - r5.large
      - r5.xlarge
      - r5.2xlarge
      - r5.4xlarge
      - r5.12xlarge
      - r5.24xlarge
      - r5d.large
      - r5d.xlarge
      - r5d.2xlarge
      - r5d.4xlarge
      - r5d.12xlarge
      - r5d.24xlarge
      - z1d.large
      - z1d.xlarge
      - z1d.2xlarge
      - z1d.3xlarge
      - z1d.6xlarge
      - z1d.12xlarge
  NodeAutoScalingGroupMinSize:
    Description: Minimum size of Node Group ASG.
    Type: Number
    Default: 1
  NodeAutoScalingGroupMaxSize:
    Description: >-
      Maximum size of Node Group ASG. Set to at least 1 greater than
      NodeAutoScalingGroupDesiredCapacity.
    Type: Number
    Default: 4
  NodeAutoScalingGroupDesiredCapacity:
    Description: Desired capacity of Node Group ASG.
    Type: Number
    Default: 3
  NodeVolumeSize:
    Description: Node volume size
    Type: Number
    Default: 20
  ClusterName:
    Description: >-
      The cluster name provided when the cluster was created. If it is
      incorrect, nodes will not be able to join the cluster.
    Type: String
  BootstrapArguments:
    Description: >-
      Arguments to pass to the bootstrap script. See files/bootstrap.sh in
      https://github.com/awslabs/amazon-eks-ami
    Type: String
    Default: ''
  NodeGroupName:
    Description: Unique identifier for the Node Group.
    Type: String
  ClusterControlPlaneSecurityGroup:
    Description: The security group of the cluster control plane.
    Type: 'AWS::EC2::SecurityGroup::Id'
  WorkerSecurityGroup:
    Description: Additional security group to grant to worker nodes.
    Type: 'AWS::EC2::SecurityGroup::Id'
  VpcId:
    Description: The VPC of the worker instances
    Type: 'AWS::EC2::VPC::Id'
  Subnets:
    Description: The subnets where workers can be created.
    Type: 'List<AWS::EC2::Subnet::Id>'
  HttpsProxyUrl:
    Description: Web proxy for communication with EKS service API
    Type: String
Metadata:
  'AWS::CloudFormation::Interface':
    ParameterGroups:
      - Label:
          default: EKS Cluster
        Parameters:
          - ClusterName
          - ClusterControlPlaneSecurityGroup
      - Label:
          default: Worker Node Configuration
        Parameters:
          - NodeGroupName
          - NodeAutoScalingGroupMinSize
          - NodeAutoScalingGroupDesiredCapacity
          - NodeAutoScalingGroupMaxSize
          - NodeInstanceType
          - NodeImageId
          - NodeVolumeSize
          - KeyName
          - BootstrapArguments
      - Label:
          default: Worker Network Configuration
        Parameters:
          - VpcId
          - Subnets
  'AWS::CloudFormation::Designer':
    85cad2f7-dadb-421a-ab98-852b69cf5d47:
      size:
        width: 60
        height: 60
      position:
        x: 60
        'y': 90
      z: 1
      embeds: []
    e81df3bf-a893-43de-9b24-97c277f6c0b6:
      source:
        id: 85cad2f7-dadb-421a-ab98-852b69cf5d47
      target:
        id: 85cad2f7-dadb-421a-ab98-852b69cf5d47
      z: 1
    04f70bb4-6911-4180-8c2c-89cddc4e1ee1:
      size:
        width: 60
        height: 60
      position:
        x: 180
        'y': 90
      z: 1
      embeds: []
    356363fa-6f7a-487a-a00a-3ebb3da047ca:
      size:
        width: 60
        height: 60
      position:
        x: 60
        'y': 210
      z: 1
      embeds: []
      isassociatedwith:
        - 04f70bb4-6911-4180-8c2c-89cddc4e1ee1
    339f765a-701a-43f3-8548-7fa5646e6a20:
      size:
        width: 60
        height: 60
      position:
        x: 180
        'y': 210
      z: 1
      embeds: []
      isassociatedwith:
        - 85cad2f7-dadb-421a-ab98-852b69cf5d47
    be1e0bbc-031f-4747-940e-c06ed3f61918:
      size:
        width: 60
        height: 60
      position:
        x: 300
        'y': 90
      z: 1
      embeds: []
      isassociatedwith:
        - 339f765a-701a-43f3-8548-7fa5646e6a20
Resources:
  NodeInstanceProfile:
    Type: 'AWS::IAM::InstanceProfile'
    Properties:
      Path: /
      Roles:
        - !Ref NodeInstanceRole
    Metadata:
      'AWS::CloudFormation::Designer':
        id: 356363fa-6f7a-487a-a00a-3ebb3da047ca
  NodeInstanceRole:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service: ec2.amazonaws.com
            Action: 'sts:AssumeRole'
      Path: /
      ManagedPolicyArns:
        - 'arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy'
        - 'arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy'
        - 'arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly'
    Metadata:
      'AWS::CloudFormation::Designer':
        id: 04f70bb4-6911-4180-8c2c-89cddc4e1ee1
  NodeSecurityGroup:
    Type: 'AWS::EC2::SecurityGroup'
    Properties:
      GroupDescription: Security group for all nodes in the cluster
      VpcId: !Ref VpcId
      Tags:
        - Key: !Sub 'kubernetes.io/cluster/${ClusterName}'
          Value: owned
    Metadata:
      'AWS::CloudFormation::Designer':
        id: 85cad2f7-dadb-421a-ab98-852b69cf5d47
  NodeSecurityGroupIngress:
    Type: 'AWS::EC2::SecurityGroupIngress'
    DependsOn: NodeSecurityGroup
    Properties:
      Description: Allow node to communicate with each other
      GroupId: !Ref NodeSecurityGroup
      SourceSecurityGroupId: !Ref NodeSecurityGroup
      IpProtocol: -1
      FromPort: 0
      ToPort: 65535
    Metadata:
      'AWS::CloudFormation::Designer':
        id: e81df3bf-a893-43de-9b24-97c277f6c0b6
  NodeSecurityGroupFromControlPlaneIngress:
    Type: 'AWS::EC2::SecurityGroupIngress'
    DependsOn: NodeSecurityGroup
    Properties:
      Description: >-
        Allow worker Kubelets and pods to receive communication from the cluster
        control plane
      GroupId: !Ref NodeSecurityGroup
      SourceSecurityGroupId: !Ref ClusterControlPlaneSecurityGroup
      IpProtocol: tcp
      FromPort: 1025
      ToPort: 65535
  ControlPlaneEgressToNodeSecurityGroup:
    Type: 'AWS::EC2::SecurityGroupEgress'
    DependsOn: NodeSecurityGroup
    Properties:
      Description: >-
        Allow the cluster control plane to communicate with worker Kubelet and
        pods
      GroupId: !Ref ClusterControlPlaneSecurityGroup
      DestinationSecurityGroupId: !Ref NodeSecurityGroup
      IpProtocol: tcp
      FromPort: 1025
      ToPort: 65535
  NodeSecurityGroupFromControlPlaneOn443Ingress:
    Type: 'AWS::EC2::SecurityGroupIngress'
    DependsOn: NodeSecurityGroup
    Properties:
      Description: >-
        Allow pods running extension API servers on port 443 to receive
        communication from cluster control plane
      GroupId: !Ref NodeSecurityGroup
      SourceSecurityGroupId: !Ref ClusterControlPlaneSecurityGroup
      IpProtocol: tcp
      FromPort: 443
      ToPort: 443
  ControlPlaneEgressToNodeSecurityGroupOn443:
    Type: 'AWS::EC2::SecurityGroupEgress'
    DependsOn: NodeSecurityGroup
    Properties:
      Description: >-
        Allow the cluster control plane to communicate with pods running
        extension API servers on port 443
      GroupId: !Ref ClusterControlPlaneSecurityGroup
      DestinationSecurityGroupId: !Ref NodeSecurityGroup
      IpProtocol: tcp
      FromPort: 443
      ToPort: 443
  ClusterControlPlaneSecurityGroupIngress:
    Type: 'AWS::EC2::SecurityGroupIngress'
    DependsOn: NodeSecurityGroup
    Properties:
      Description: Allow pods to communicate with the cluster API Server
      GroupId: !Ref ClusterControlPlaneSecurityGroup
      SourceSecurityGroupId: !Ref NodeSecurityGroup
      IpProtocol: tcp
      ToPort: 443
      FromPort: 443
  NodeGroup:
    Type: 'AWS::AutoScaling::AutoScalingGroup'
    Properties:
      DesiredCapacity: !Ref NodeAutoScalingGroupDesiredCapacity
      LaunchConfigurationName: !Ref NodeLaunchConfig
      MinSize: !Ref NodeAutoScalingGroupMinSize
      MaxSize: !Ref NodeAutoScalingGroupMaxSize
      VPCZoneIdentifier: !Ref Subnets
      Tags:
        - Key: Name
          Value: !Sub '${ClusterName}-${NodeGroupName}-Node'
          PropagateAtLaunch: true
        - Key: !Sub 'kubernetes.io/cluster/${ClusterName}'
          Value: owned
          PropagateAtLaunch: true
    UpdatePolicy:
      AutoScalingRollingUpdate:
        MaxBatchSize: 1
        MinInstancesInService: !Ref NodeAutoScalingGroupDesiredCapacity
        PauseTime: PT5M
    Metadata:
      'AWS::CloudFormation::Designer':
        id: be1e0bbc-031f-4747-940e-c06ed3f61918
  NodeLaunchConfig:
    Type: 'AWS::AutoScaling::LaunchConfiguration'
    Properties:
      AssociatePublicIpAddress: false
      IamInstanceProfile: !Ref NodeInstanceProfile
      ImageId: !Ref NodeImageId
      InstanceType: !Ref NodeInstanceType
      KeyName: !Ref KeyName
      SecurityGroups:
        - !GetAtt NodeSecurityGroup.GroupId
        - !Ref WorkerSecurityGroup
      BlockDeviceMappings:
        - DeviceName: /dev/xvda
          Ebs:
            VolumeSize: !Ref NodeVolumeSize
            VolumeType: gp2
            DeleteOnTermination: true
      UserData:
        'Fn::Base64': !Sub |
          #!/bin/bash

          set -o xtrace

          `aws ecr get-login --region eu-west-1 --no-include-email --registry-ids 602401143452`
          docker pull 602401143452.dkr.ecr.eu-west-1.amazonaws.com/amazon-k8s-cni:v1.5.0
          docker pull 602401143452.dkr.ecr.eu-west-1.amazonaws.com/eks/kube-proxy:v1.11.5
          docker pull 602401143452.dkr.ecr.eu-west-1.amazonaws.com/eks/coredns:v1.1.3
          docker pull 602401143452.dkr.ecr.eu-west-1.amazonaws.com/eks/pause-amd64:3.1

          NO_PROXY=169.254.169.254 https_proxy=${HttpsProxyUrl} /etc/eks/bootstrap.sh ${ClusterName} ${BootstrapArguments}
          /opt/aws/bin/cfn-signal --exit-code $? \
                   --stack  ${AWS::StackName} \
                   --resource NodeGroup  \
                   --region ${AWS::Region}
    Metadata:
      'AWS::CloudFormation::Designer':
        id: 339f765a-701a-43f3-8548-7fa5646e6a20
Outputs:
  NodeInstanceRole:
    Description: The node instance role
    Value: !GetAtt NodeInstanceRole.Arn
  NodeSecurityGroup:
    Description: The security group for the node group
    Value: !Ref NodeSecurityGroup