gke-snapshot.sh

#!/bin/bash

NODE_COMMANDS=( \
  "uptime"                           \
  "df --all --inodes"                \
  "ip addr"                          \
  "sudo iptables-save --counters"    \
  "mount"                            \
  "ip route list table all"          \
  "top -bn1"                         \
  "sudo docker ps -a"                \
  "ps -edF"                          \
  "sudo conntrack --count"           \
  "ps -eo pid,tid,ppid,class,rtprio,ni,pri,psr,pcpu,stat,wchan:14,comm,args,cgroup" \
  "dmesg"                            \
  "systemctl status -l docker"       \
  "sudo journalctl --utc -u docker"       \
  "sudo journalctl --utc -u docker-monitor.service"  \
  "systemctl status -l kubelet"      \
  "sudo journalctl --utc -u kubelet"      \
  "sudo journalctl --utc -u kubelet-monitor.service" \
  "sudo journalctl --utc --boot --dmesg"  \
  "sudo journalctl --utc -u node-problem-detector"   \
)

NODE_FILES=( \
  "/proc/sys/fs/file-nr"             \
  "/proc/sys/net/nf_conntrack_max"   \
)

ROOT=$(mktemp -d -t gke-XXXXXX)
KUBECTL_ROOT=${ROOT}/kubectlCommands
NODE_ROOT=${ROOT}/nodes
NODES=$(kubectl get node | awk '{print $1}' | tail -n +2)

mkdir -p ${KUBECTL_ROOT}
mkdir -p ${NODE_ROOT}

function print_nodes() {
  echo "k8s node list:"
  for node in ${NODES}; do
    echo "- ${node}"
  done
}

function fetch_kubectl_logs() {
  local comp=$1
  local labels=$2

  echo "collecting ${comp} logs ..."

  kubectl -n kube-system logs -l${labels} --all-containers=true > ${KUBECTL_ROOT}/${comp}.log
}

function fetch_node_file() {
  local node=$1
  local file=$2

  # TODO(ygui): hardcoded zone/region (projects)
  gcloud compute ssh ${node} --zone us-central1-c --command "sudo cat ${file}" > "${NODE_ROOT}/${node}/files/${file//\//_}"
}

function run_node_command() {
  local node=$1
  local command=$2

  gcloud compute ssh ${node} --zone us-central1-c --command "${command}" > "${NODE_ROOT}/${node}/commands/${command// /_}"
}

# kubectl logs
fetch_kubectl_logs "kube-proxy" "component=kube-proxy"
fetch_kubectl_logs "kube-dns" "k8s-app=kube-dns"

# Node logs
print_nodes
for node in ${NODES}
do
  echo "collecting logs from ${node} ..."
  mkdir -p ${NODE_ROOT}/${node}/{files,commands}
  for cmd in "${NODE_COMMANDS[@]}"; do
    run_node_command "${node}" "${cmd}"
  done
  for file in "${NODE_FILES[@]}"; do
    fetch_node_file "${node}" "${file}"
  done
done

# Make tarball
tar zcvf snapshot.tar.gz ${ROOT}