Deploy k8s cluster with Ubuntu nodes using kops

kops-ubuntu.sh

#!/usr/bin/env bash

export CLUSTER_FQDN="${CLUSTER_FQDN:-ruzickap-kops-ubuntu.k8s.mylabs.dev}"
export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-eu-central-1}"
export KUBECONFIG="/tmp/kubeconfig-${CLUSTER_FQDN}.conf"
export KOPS_STATE_STORE="s3://${CLUSTER_FQDN}"

set -euxo pipefail

## Create S3 bucket first
# aws s3api create-bucket --bucket "${CLUSTER_FQDN}" --create-bucket-configuration LocationConstraint=${AWS_DEFAULT_REGION}
# aws s3api put-public-access-block --bucket "${CLUSTER_FQDN}" --public-access-block-configuration "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true"
# aws s3api put-bucket-versioning --bucket "${CLUSTER_FQDN}" --versioning-configuration Status=Enabled

cat > "/tmp/kops-${CLUSTER_FQDN}.yaml" << EOF
apiVersion: kops.k8s.io/v1alpha2
kind: Cluster
metadata:
  name: ${CLUSTER_FQDN}
spec:
  api:
    loadBalancer:
      class: Network
      type: Public
  authorization:
    rbac: {}
  certManager:
    enabled: true
  channel: stable
  cloudLabels:
    Cluster: ${CLUSTER_FQDN}
    Environment: dev
    Owner: [email protected]
  cloudProvider: aws
  containerRuntime: containerd
  clusterAutoscaler:
    enabled: true
    balanceSimilarNodeGroups: true
    scaleDownUtilizationThreshold: "0.65"
    skipNodesWithLocalStorage: false
    cpuRequest: 100m
    memoryRequest: 800Mi
  etcdClusters:
  - cpuRequest: 200m
    etcdMembers:
    - encryptedVolume: true
      instanceGroup: master-${AWS_DEFAULT_REGION}a
      name: a
      volumeSize: 10
    - encryptedVolume: true
      instanceGroup: master-${AWS_DEFAULT_REGION}b
      name: b
      volumeSize: 10
    - encryptedVolume: true
      instanceGroup: master-${AWS_DEFAULT_REGION}c
      name: c
      volumeSize: 10
    memoryRequest: 100Mi
    name: main
  - cpuRequest: 100m
    etcdMembers:
    - encryptedVolume: true
      instanceGroup: master-${AWS_DEFAULT_REGION}a
      name: a
      volumeSize: 9
    - encryptedVolume: true
      instanceGroup: master-${AWS_DEFAULT_REGION}b
      name: b
      volumeSize: 9
    - encryptedVolume: true
      instanceGroup: master-${AWS_DEFAULT_REGION}c
      name: c
      volumeSize: 9
    memoryRequest: 100Mi
    name: events
  externalPolicies:
    master:
    - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
    node:
    - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
  iam:
    legacy: false
  instanceMetadata:
    httpTokens: required
  kubelet:
    anonymousAuth: false
    featureGates:
      EphemeralContainers: "true"
      GracefulNodeShutdown: "true"
      HPAContainerMetrics: "true"
      HPAScaleToZero: "true"
  kubernetesApiAccess:
  - 0.0.0.0/0
  - ::/0
  kubernetesVersion: 1.28.1
  masterPublicName: api.${CLUSTER_FQDN}
  metricsServer:
    enabled: true
  kubeAPIServer:
    featureGates:
      EphemeralContainers: "true"
      GracefulNodeShutdown: "true"
      HPAContainerMetrics: "true"
      HPAScaleToZero: "true"
  networkCIDR: 172.20.0.0/16
  networking:
    calico: {}
  nonMasqueradeCIDR: 100.64.0.0/10
  subnets:
  - cidr: 172.20.32.0/19
    name: ${AWS_DEFAULT_REGION}a
    type: Public
    zone: ${AWS_DEFAULT_REGION}a
  - cidr: 172.20.64.0/19
    name: ${AWS_DEFAULT_REGION}b
    type: Public
    zone: ${AWS_DEFAULT_REGION}b
  - cidr: 172.20.96.0/19
    name: ${AWS_DEFAULT_REGION}c
    type: Public
    zone: ${AWS_DEFAULT_REGION}c

---

apiVersion: kops.k8s.io/v1alpha2
kind: SSHCredential
metadata:
  labels:
    kops.k8s.io/cluster: ${CLUSTER_FQDN}
spec:
  publicKey: "$(cat ~/.ssh/id_rsa.pub)"

---

apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  labels:
    kops.k8s.io/cluster: ${CLUSTER_FQDN}
  name: master-${AWS_DEFAULT_REGION}a
spec:
  # aws ec2 describe-images --region eu-central-1 --output table --owners 099720109477 --query "sort_by(Images, &CreationDate)[*].[CreationDate,Name,ImageId]" --filters "Name=name,Values=ubuntu/images/hvm-ssd/ubuntu-*-22.04-amd64-*"
  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20231121
  machineType: t3.medium
  maxSize: 1
  minSize: 1
  nodeLabels:
    kops.k8s.io/instancegroup: master-${AWS_DEFAULT_REGION}a
  role: Master
  rootVolumeSize: 10
  subnets:
  - ${AWS_DEFAULT_REGION}a

---

apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  labels:
    kops.k8s.io/cluster: ${CLUSTER_FQDN}
  name: master-${AWS_DEFAULT_REGION}b
spec:
  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20231121
  machineType: t3.medium
  maxSize: 1
  minSize: 1
  nodeLabels:
    kops.k8s.io/instancegroup: master-${AWS_DEFAULT_REGION}b
  role: Master
  rootVolumeSize: 10
  subnets:
  - ${AWS_DEFAULT_REGION}b

---

apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  labels:
    kops.k8s.io/cluster: ${CLUSTER_FQDN}
  name: master-${AWS_DEFAULT_REGION}c
spec:
  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20231121
  machineType: t3.medium
  maxSize: 1
  minSize: 1
  nodeLabels:
    kops.k8s.io/instancegroup: master-${AWS_DEFAULT_REGION}c
  role: Master
  rootVolumeSize: 10
  subnets:
  - ${AWS_DEFAULT_REGION}c

---

apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  labels:
    kops.k8s.io/cluster: ${CLUSTER_FQDN}
  name: nodes-${AWS_DEFAULT_REGION}
spec:
  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20231121
  machineType: t3.large
  maxSize: 5
  minSize: 2
  nodeLabels:
    kops.k8s.io/instancegroup: nodes-${AWS_DEFAULT_REGION}
  role: Node
  rootVolumeSize: 10
  subnets:
  - ${AWS_DEFAULT_REGION}a
  - ${AWS_DEFAULT_REGION}b
  - ${AWS_DEFAULT_REGION}c
  cloudLabels:
    k8s.io/cluster-autoscaler/enabled: "true"
EOF

kops create -f "/tmp/kops-${CLUSTER_FQDN}.yaml"
kops update cluster --name "${CLUSTER_FQDN}" --yes --admin
kops export kubeconfig "${CLUSTER_FQDN}" --kubeconfig "${KUBECONFIG}" --admin

sleep 300

kops validate cluster --name "${CLUSTER_FQDN}" --wait 10m

################################################################################

cat << EOF

*** Cluster DELETE
export AWS_DEFAULT_REGION="eu-central-1"
export KOPS_STATE_STORE="s3://${CLUSTER_FQDN}"

kops delete cluster --name="${CLUSTER_FQDN}" --yes

EOF

echo -e "\n*****\n  export KUBECONFIG=${KUBECONFIG} \n*****\n"