Karpenter with Custom Launch Templates

karpenter-launch-templates.tf

data "aws_ami" "aws_eks_gpu_optimized" {
  owners      = ["602401143452"]
  most_recent = true

  filter {
    name   = "name"
    values = ["amazon-eks-gpu-node-${var.eks_cluster_version}-v20220112*"]
  }
}

data "template_file" "launch_template_userdata" {
  template = file("${path.module}/userdata.sh.tpl")

  vars = {
    cluster_name        = var.eks_cluster_name
    endpoint            = var.eks_cluster_endpoint
    cluster_auth_base64 = var.eks_cluster_ca_cert_data

    bootstrap_extra_args = ""
    kubelet_extra_args   = ""
  }
}

resource "aws_iam_role_policy_attachment" "karpenter_ssm_policy" {
  role       = var.eks_worker_iam_role_name
  policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
}

resource "aws_iam_instance_profile" "karpenter" {
  name = "KarpenterNodeInstanceProfile-${var.eks_cluster_name}"
  role = var.eks_worker_iam_role_name
}

resource "aws_launch_template" "karpenter_provisioner_gpu" {
  name_prefix = "karpenter-provisioner-gpu-"

  block_device_mappings {
    device_name = "/dev/xvda"

    ebs {
      volume_size = 100
      volume_type = "gp3"
      iops        = 3000
      throughput  = 125
    }
  }

  image_id = data.aws_ami.aws_eks_gpu_optimized.id

  iam_instance_profile {
    name = aws_iam_instance_profile.karpenter.name
  }

  vpc_security_group_ids = [var.eks_worker_security_group_id]

  user_data = base64encode(
    data.template_file.launch_template_userdata.rendered,
  )

  monitoring {
    enabled = true
  }
}

karpenter-provisioner.tf

resource "kubernetes_manifest" "karpenter_provisioner_gpu" {
  manifest = {
    apiVersion = "karpenter.sh/v1alpha5"
    kind       = "Provisioner"

    metadata = {
      name = "gpu"
    }

    spec = {
      requirements = [
        {
          key      = "kubernetes.io/arch"
          operator = "In"
          values   = ["amd64"]
        },
        {
          key      = "karpenter.sh/capacity-type"
          operator = "In"
          values   = ["on-demand"]
        },
      ]

      taints = [
        {
          key    = "nvidia.com/gpu"
          value  = "true"
          effect = "NoSchedule"
        }
      ]

      # It is required to add apiVersion, kind and securityGroupSelector or else the
      # kubernetes provider will fail on apply.
      # https://github.com/hashicorp/terraform-provider-kubernetes/issues/1545
      provider = {
        apiVersion = "extensions.karpenter.sh/v1alpha1"
        kind       = "AWS"

        instanceProfile = aws_iam_instance_profile.karpenter.name
        launchTemplate  = aws_launch_template.karpenter_provisioner_gpu.name
        subnetSelector = {
          "kubernetes.io/cluster/${var.eks_cluster_name}" : "*"
        }
        securityGroupSelector = {
          "kubernetes.io/cluster/${var.eks_cluster_name}" : "*"
        }
      }

      ttlSecondsAfterEmpty = "120"
    }
  }
}

userdata.sh.tpl

MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="//"

--//
Content-Type: text/x-shellscript; charset="us-ascii"
#!/bin/bash

exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1

set -xe

# Bootstrap and join the cluster
/etc/eks/bootstrap.sh '${cluster_name}' ${bootstrap_extra_args} \
--b64-cluster-ca '${cluster_auth_base64}' \
--apiserver-endpoint '${endpoint}' \
--kubelet-extra-args "${kubelet_extra_args}"
echo "Bootstrap Complete!"

--//--