pod-schedulable-nodes.sh

#!/bin/bash

set -e

# handle flags -e for eligible nodes, -c for conflict nodes
SHOW="all"

while getopts "ec" opt; do
  case $opt in
    e) SHOW="eligible" ;;
    c) SHOW="conflict" ;;
    *) echo "Usage: $0 [-e] [-c] <pod.yaml>" && exit 1 ;;
  esac
done
shift $((OPTIND - 1))


MANIFEST="$1"
[ -z "$MANIFEST" ] && echo "Usage: $0 <pod.yaml>" && exit 1

POD_JSON=$(kubectl create -f "$MANIFEST" --dry-run=client -o json 2>/dev/null)
[ -z "$POD_JSON" ] && echo "Error: Invalid Manifest" && exit 1

ANTI_AFFINITY_SELECTORS=$(echo "$POD_JSON" | jq -rc '
  def get_comparison_sign($op):
    if $op == "In" then "="
    elif $op == "NotIn" then "!="
    else empty end;

  def get_sign_prefix($op):
    if $op == "Exists" then ""
    elif $op == "DoesNotExist" then "!"
    else empty end;

  def parse_label_selector_expression($op; $key; $values):
    if $op == "In" or $op == "NotIn" then
      $values | map("\($key)\(get_comparison_sign($op))\(.)") | join(",")
    else
      "\(get_sign_prefix($op))\($key)"
    end;

  (
    .spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution // []
    | map(.labelSelector.matchLabels // {})
    | add // {}
    | to_entries
    | map("\(.key)=\(.value)")
  )+(
    .spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution // []
    | map(.labelSelector.matchExpressions // [])
    | add // []
    | map(parse_label_selector_expression(.operator; .key; .values // []))
  ) | unique | join(",")'
)

[[ ANTI_AFFINITY_SELECTORS != "" ]] &&
  (
    echo "Anti-Affinity Selectors: $ANTI_AFFINITY_SELECTORS";
    kubectl get pods -A -l "${ANTI_AFFINITY_SELECTORS}" -o json > /tmp/conflict_pods.json
  ) ||
  echo '{"items": []}' > /tmp/conflict_pods.json

echo "Evaluating node eligibility for pod (showing $SHOW nodes):"
kubectl get nodes -o json | jq --slurpfile conflicts /tmp/conflict_pods.json --argjson pod "$POD_JSON" --arg show "$SHOW" '
  def is_tolerated($taint; $tolerations):
    any($tolerations[];
      (.key == $taint.key or (.operator == "Exists" and .key == null)) and
      (.effect == $taint.effect or (.effect == null and .key != null)) and
      (.value == $taint.value or .operator == "Exists")
    );

  def check_selector($node_labels; $pod_selector):
    if ($pod_selector | length) == 0 then true
    else
      all($pod_selector | to_entries[]; $node_labels[.key] == .value)
    end;

  def check_match_expression($node_labels; $expr):
    if $expr.operator == "In" then
      ($node_labels[$expr.key] // "") as $val | ($expr.values | index($val))
    elif $expr.operator == "NotIn" then
      ($node_labels[$expr.key] // "") as $val | ($expr.values | index($val) | not)
    elif $expr.operator == "Exists" then
      $node_labels | has($expr.key)
    elif $expr.operator == "DoesNotExist" then
      $node_labels | has($expr.key) | not
    else
      true 
    end;

  def check_affinity($node_labels; $affinity):
    if $affinity == null then true
    else
      any($affinity.nodeSelectorTerms[];
        # TODO: check matchFields as well
        all(.matchExpressions[]; check_match_expression($node_labels; .))
      )
    end;

  def check_conflict($node_name; $node_labels; $topology_key; $conflict_list):
    if ($node_labels | has($topology_key))
    then
       any($conflict_list[0].items[]; .spec.nodeName == $node_name)
    else
      false
    end;

  # converts data units to bytes, and cpu units to cores
  def k8s_unit_to_number($str):
    if $str == null then 0
    elif ($str | endswith("Ki")) then ($str | rtrimstr("Ki") | tonumber) * pow(1024; 1)
    elif ($str | endswith("Mi")) then ($str | rtrimstr("Mi") | tonumber) * pow(1024; 2)
    elif ($str | endswith("Gi")) then ($str | rtrimstr("Gi") | tonumber) * pow(1024; 3)
    elif ($str | endswith("Ti")) then ($str | rtrimstr("Ti") | tonumber) * pow(1024; 4)
    elif ($str | endswith("K")) then ($str | rtrimstr("K") | tonumber) * pow(1000; 1)
    elif ($str | endswith("M")) then ($str | rtrimstr("M") | tonumber) * pow(1000; 2)
    elif ($str | endswith("G")) then ($str | rtrimstr("G") | tonumber) * pow(1000; 3)
    elif ($str | endswith("T")) then ($str | rtrimstr("T") | tonumber) * pow(1000; 4)
    elif ($str | endswith("m")) then ($str | rtrimstr("m") | tonumber) * 0.001
    # last case will convert 123e6 to number as well
    else $str | tonumber end;

  .items[] |
  {
    name: .metadata.name,
    labels: (.metadata.labels // {}),
    taints: (.spec.taints // []),    
    allocatable: .status.allocatable,
    pod_req: $pod.spec.containers[0].resources.requests
  } as $node | $node |
  {
    name: .name,
    failed_taints: [
      .taints[]? |
      select(.effect == "NoSchedule" or .effect == "NoExecute") |
      select(is_tolerated(.; $pod.spec.tolerations // []) | not)
    ],
    selector_match: check_selector(.labels; $pod.spec.nodeSelector // {}),
    affinity_match: check_affinity(.labels; $pod.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution),
    has_conflict: any($pod.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[];
      check_conflict($node.name; $node.labels; .topologyKey; $conflicts)
    ),
    cpu_ok: (
       if .pod_req.cpu then
         (k8s_unit_to_number(.allocatable.cpu) >= k8s_unit_to_number(.pod_req.cpu))
       else true end
    ),
    mem_ok: (
       if .pod_req.memory then
         (k8s_unit_to_number(.allocatable.memory) >= k8s_unit_to_number(.pod_req.memory))
       else true end
    ),
    storage_ok: (
       if .pod_req["ephemeral-storage"] then
         (k8s_unit_to_number(.allocatable["ephemeral-storage"]) >= k8s_unit_to_number(.pod_req["ephemeral-storage"]))
       else true end
    ),
    cpu_msg: (
      if .pod_req.cpu then
        "CPU req=" + (k8s_unit_to_number(.pod_req.cpu) | tostring) + ", avail=" + (k8s_unit_to_number(.allocatable.cpu) | tostring)
      else "CPU req=none" end
    ),
    mem_msg: (
      if .pod_req.memory then
        "Mem req=" + (k8s_unit_to_number(.pod_req.memory) / pow(1024; 3) | floor | tostring) + "Mi, avail=" +
        (k8s_unit_to_number(.allocatable.memory) / pow(1024; 3) | floor | tostring) + "Mi"
      else "Mem req=none" end
    ),
    storage_msg: (
      if .pod_req["ephemeral-storage"] then
        "Eph req=" + (k8s_unit_to_number(.pod_req["ephemeral-storage"]) / pow(1024; 3) | floor | tostring) + "Gi, avail=" +
          (k8s_unit_to_number(.allocatable["ephemeral-storage"]) / pow(1024; 3) | floor | tostring) + "Gi"
      else "Eph req=none" end
    )
  } | 
  if (.failed_taints | length) > 0 then
    if $show == "all" or $show == "conflict" then
    "❌ \(.name): Tainted \(.failed_taints[].key)"
    else empty end
  elif .selector_match == false then
    if $show == "all" or $show == "conflict" then
    "❌ \(.name): Node Selector Mismatch"
    else empty end
  elif .affinity_match == false then
    if $show == "all" or $show == "conflict" then
    "❌ \(.name): Node Affinity Mismatch"
    else empty end
  elif .has_conflict then
    if $show == "all" or $show == "conflict" then
    "❌ \(.name): Pod Anti-Affinity Conflict"
    else empty end
  elif .cpu_ok == false then
    if $show == "all" or $show == "conflict" then
     "❌ \(.name): Insufficient CPU Capacity (" + .cpu_msg + ")"
    else empty end
  elif .mem_ok == false then
    if $show == "all" or $show == "conflict" then
     "❌ \(.name): Insufficient Memory Capacity (" + .mem_msg + ")"
    else empty end
  elif .storage_ok == false then
    if $show == "all" or $show == "conflict" then
     "❌ \(.name): Insufficient Ephemeral Storage (" + .storage_msg + ")"
    else empty end
  else
    if $show == "all" or $show == "eligible" then
    "✅ \(.name): Eligible"
    else empty end
  end
' -r