A Karpenter provisioner for general purpose workloads, with focus on cost control.

general-cpu-provisioner.yaml

# The default provisioner used by Karpenter.

apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
  name: general-cpu
spec:
  # Terminate nodes x seconds after
  # the last pod is removed from it.
  ttlSecondsAfterEmpty: 30

  # Upper limit of resources that the provisioner can launch.
  limits:
    resources:
      cpu: "50"
      memory: 500Gi

  # Configure the characteristics of the nodes.
  requirements:
    # Prioritize spot, fallback to OD. Default behaviour of
    # Karpenter when both Spot and OD are specified.
    - key: karpenter.sh/capacity-type
      operator: In
      values: ["spot", "on-demand"]

    # Exclude instances in these categories.
    # We prevent instances with a GPU, etc from being provisioned...
    # Because the moneys.
    - key: karpenter.k8s.aws/instance-category
      operator: NotIn
      values:
        - g    # GPUs
        - p    # Large GPUs
        - inf  # Inferentia acclerator
        - dl   # Gaudi accelerator
        - cc   # Cluster compute, for HPC
        - d    # Dense storage, for parallel data processing
        - i    # Storage optimized
        - im   # Storage optimized, Graviton
        - is   # Storage optimized, Graviton
        - h    # Storage optimized, high throughput
        - u    # Absurdly high memory, for in-memory DBs
        - vt   # Video transcoding
        - x    # Xeon processors, high memory
        - z    # High mem, high CPU, high frequency

    # Exclude instances in these families.
    - key: karpenter.k8s.aws/instance-family
      operator: NotIn
      values:
        - c1 # 1st gen
        - c3 # 3rd gen
        - c6i # Intel processors, costlier (in some regions)
        - c6id
        - m1 # 1st gen
        - m2 # 2nd gen
        - m3 # 3rd gen
        - m5zn # High frequency
        - m6i # Intel processors
        - m6id
        - r3 # 3rd gen
        - r6i # Intel processors
        - r6id
        - t1 # 1st gen
        - g3 # 3rd gen, old GPU
        - g3s # 3rd gen, old GPU

    # We'd rather have smaller instances to encourage aggressive deprovisioning.
    # Prevent instances with more than 50 GB of memory from being launched.
    - key: karpenter.k8s.aws/instance-memory
      operator: Lt  # Lower than. Gt -> Greater than.
      values:
        - "50000"
    # Prevent instances with more than 24 CPUs from being launched.
    - key: karpenter.k8s.aws/instance-cpu
      operator: Lt
      values:
        - "25"

    # Allow dynamic node group provisioning.
    # Refer to bit.ly/3CDVkbj for details.
    - key: company.com/user
      operator: Exists

  # K8s Labels for the nodes launched by this provisioner.
  labels:
    node-type: general-cpu

  # Allow only pods with the matching toleration to use
  # nodes launched by this provisioner.
  taints:
    - key: workload-type
      value: general-cpu
      effect: NoSchedule

  # Link this provisioner to an AWSNodeTemplate.
  providerRef:
    name: general-cpu