kube-volume-freezer complex example.

kvf-complex-example.sh

#!/bin/bash
# This script performs a live snapshotting of 3 Volumes from 3 different deployments simultaneously.

set -Eeuo pipefail

GCLOUD=gcloud
KUBECTL=kubectl
KVFCTL="docker run --net host --rm wikiwi/kube-volume-freezer kvfctl"

NAMESPACE=default
GITLAB_DEPLOYMENT_NAME=gitlab
GITLAB_RUNNER_DEPLOYMENT_NAME=gitlab-runner
POSTGRES_DEPLOYMENT_NAME=gitlab-postgresql

KVF_NAMESPACE=default
KVF_SERVICE_NAME=kube-volume-freezer
KVF_SECRET=kube-volume-freezer-secret
KVF_PORT=8080
KVF_LOCAL_PORT=35642
KVF_TOKEN=$(${KUBECTL} --namespace "${KVF_NAMESPACE}" get secret "${KVF_SECRET}" -o 'go-template={{ index .data "apiserver-token" }}' | base64 -d)
KVF_ADDRESS=http://localhost:${KVF_LOCAL_PORT}

GITLAB_VOLUME=data
GITLAB_RUNNER_VOLUME=homegitlabrunner
POSTGRES_VOLUME=data

get_deployment_pd_name() {
  name=$("${KUBECTL}" --namespace "$1" get deployments "$2" --output-version="extensions/v1beta1" --output=jsonpath="{.spec.template.spec.volumes[?(@.name==\"$3\")].gcePersistentDisk.pdName}")
  if [ -z "${name}" ]; then
    echo "Volume $3 of Pod $2 in Namespace $1 not found" 1>&2
    return 1
  fi
  echo "${name}"
}

get_deployment_pods() {
  sel=$("${KUBECTL}" --namespace "$1" get deployments "$2" --output-version="extensions/v1beta1" --output=json | jq -r '.spec.selector.matchLabels | to_entries | .[] | "\(.key)=\(.value),"' | sed ':a;N;$!ba;s/\n//g')
  sel=${sel%?} # Remove trailing comma
  "${KUBECTL}" --namespace "$1" get pods --output-version="v1" --selector="${sel}" --output=jsonpath={.items..metadata.name}
}

get_service_pods() {
  sel=$("${KUBECTL}" --namespace "$1" get svc "$2" --output-version="v1" --output=json | jq -r '.spec.selector | to_entries | .[] | "\(.key)=\(.value),"' | sed ':a;N;$!ba;s/\n//g')
  sel=${sel%?} # Remove trailing comma
  "${KUBECTL}" --namespace "$1" get pods --output-version="v1" --selector="${sel}" --output=jsonpath={.items..metadata.name}
}

get_pod_zone() {
  node=$("${KUBECTL}" --namespace "$1" get pods "$2" --output-version="v1" --output=jsonpath={.spec.nodeName})
  zone=$("${KUBECTL}" get nodes "${node}" --output-version="v1" --output='go-template={{index .metadata.labels "failure-domain.beta.kubernetes.io/zone"}}')
  if [[ "${zone}" == "<no value>" ]]; then
    echo "Zone of Pod $2 in Namespace $1 not found" 1>&2
    return 1
  fi
  echo "${zone}"
}

wait_till_200() {
  for i in `seq 1 "$2"`;
  do
    curl --fail "$1" > /dev/null 2>&1 && return
    sleep 1
  done
  echo "Timeout waiting for $1" 1>&2
  return 1
}

trapError() {
  _name="$0"                # name of the script
  _lastline="$1"            # argument 1: last line of error occurence
  _lasterr="$2"             # argument 2: error code of last command
  echo "${_name}: line ${_lastline}: exit status of last command: ${_lasterr}" 1>&2
  trap - ERR
  exit "${_lasterr}"
}

cleanup() {
  KVF_PID=${KVF_PID:-}
  if [ -z "${KVF_PID}" ]; then
    return
  fi

  echo "=> Unfreeze..."
  ${KVFCTL} --address "${KVF_ADDRESS}" --namespace "${NAMESPACE}" --token "${KVF_TOKEN}" thaw ${gitlab_pods} ${GITLAB_VOLUME} || true
  ${KVFCTL} --address "${KVF_ADDRESS}" --namespace "${NAMESPACE}" --token "${KVF_TOKEN}" thaw ${gitlab_runner_pods} ${GITLAB_RUNNER_VOLUME} || true
  ${KVFCTL} --address "${KVF_ADDRESS}" --namespace "${NAMESPACE}" --token "${KVF_TOKEN}" thaw ${postgres_pods} ${POSTGRES_VOLUME} || true

  echo "=> Stop forwarder..."
  kill -TERM "${KVF_PID}"
}

trap 'trapError ${LINENO} $?' ERR
trap cleanup EXIT

echo "=> Retrieve Persistant Disk Name for each Deployment from Kubernetes API Server..."
gitlab_pd_name=$(get_deployment_pd_name "${NAMESPACE}" "${GITLAB_DEPLOYMENT_NAME}" "${GITLAB_VOLUME}")
gitlab_runner_pd_name=$(get_deployment_pd_name "${NAMESPACE}" "${GITLAB_RUNNER_DEPLOYMENT_NAME}" "${GITLAB_RUNNER_VOLUME}")
postgres_pd_name=$(get_deployment_pd_name "${NAMESPACE}" "${POSTGRES_DEPLOYMENT_NAME}" "${POSTGRES_VOLUME}")

echo "Gitlab PD-Name: ${gitlab_pd_name}"
echo "Gitlab Runner PD-Name: ${gitlab_runner_pd_name}"
echo "Postgres PD-Name: ${postgres_pd_name}"

echo "=> Retrieve Pod names from Kubernetes API Server..."
gitlab_pods=$(get_deployment_pods "${NAMESPACE}" "${GITLAB_DEPLOYMENT_NAME}")
gitlab_runner_pods=$(get_deployment_pods "${NAMESPACE}" "${GITLAB_RUNNER_DEPLOYMENT_NAME}")
postgres_pods=$(get_deployment_pods "${NAMESPACE}" "${POSTGRES_DEPLOYMENT_NAME}")
kvf_pods=$(get_service_pods "${KVF_NAMESPACE}" "${KVF_SERVICE_NAME}")

echo "Gitlab Pods: ${gitlab_pods}"
echo "Gitlab Runner Pods: ${gitlab_runner_pods}"
echo "Postgres Pods: ${postgres_pods}"
echo "kube-volume-freezer Pods: ${kvf_pods}"

echo "=> Retrieve Zone of each Pod from Kubernetes API Server..."
gitlab_zone=$(get_pod_zone "${NAMESPACE}" "${gitlab_pods}")
gitlab_runner_zone=$(get_pod_zone "${NAMESPACE}" "${gitlab_runner_pods}")
postgres_zone=$(get_pod_zone "${NAMESPACE}" "${postgres_pods}")

echo "Gitlab Pods Zone: ${gitlab_zone}"
echo "Gitlab Runner Pods Zone: ${gitlab_runner_zone}"
echo "Postgres Pods Zone: ${postgres_zone}"

echo "=> Open local port to kube-volume-freezer..."
${KUBECTL} --namespace "${KVF_NAMESPACE}" port-forward "${kvf_pods}" "${KVF_LOCAL_PORT}":"${KVF_PORT}" &
KVF_PID=$!
wait_till_200 "${KVF_ADDRESS}/healthz" 20

echo "=> Freeze desired Volumes..."
${KVFCTL} --address "${KVF_ADDRESS}" --namespace "${NAMESPACE}" --token "${KVF_TOKEN}" freeze ${gitlab_pods} ${GITLAB_VOLUME}
${KVFCTL} --address "${KVF_ADDRESS}" --namespace "${NAMESPACE}" --token "${KVF_TOKEN}" freeze ${gitlab_runner_pods} ${GITLAB_RUNNER_VOLUME}
${KVFCTL} --address "${KVF_ADDRESS}" --namespace "${NAMESPACE}" --token "${KVF_TOKEN}" freeze ${postgres_pods} ${POSTGRES_VOLUME}

echo "=> Perform Snapshot..."
date=$(date +"%Y%m%d%H%M%S")

joblist=
${GCLOUD} compute disks snapshot "${gitlab_pd_name}" --zone "${gitlab_zone}" --snapshot-names "${gitlab_pd_name}-${date}" & joblist="${joblist} $!"
${GCLOUD} compute disks snapshot "${gitlab_runner_pd_name}" --zone "${gitlab_zone}" --snapshot-names "${gitlab_runner_pd_name}-${date}" & joblist="${joblist} $!"
${GCLOUD} compute disks snapshot "${postgres_pd_name}" --zone "${gitlab_zone}" --snapshot-names "${postgres_pd_name}-${date}" & joblist="${joblist} $!"

for job in $joblist
do
  echo "Wait for Job #${job}"
  wait ${job} || let "Job #${job} failed"
done