gk8s-wait-for.sh

gk8s-wait-for.sh

gk8s_wait_for() {
  retries="$1"; shift
  regex="$1"; shift
  kind="$*"

  set +xe
  echo ":: Wating for expression:"
  echo "::   $kind"
  echo ":://"

  callback_status=1
  while (( retries )); do
    gk8s ":${K8S_CLUSTER}" get "$@"  \
    | grep -qsEie "$regex"
    gk8s_wait_for_callback "$?"
    callback_status="$?"
    if [[ "$callback_status" -eq 0 ]]; then
      break
    fi
    echo -n "[$retries]"
    sleep 1s
    (( retries -- ))
  done || true
  echo
  set -xe
  gk8s ":${K8S_CLUSTER}" get "$@"
  return $callback_status
}

deploy_local_wait() {
  # First, wait for all pods are ready, or error
  gk8s_wait_for_callback() { [[ $1 -ge 1 ]]; }
  gk8s_wait_for 900 \
    "(pending)|(creating)|(init)|(ImagePullBackOff)|(ErrImagePull)|(CrashLoopBackOff)" \
    pods -n "local" -l mineHashId="$(git_image_tag)"

  # Now, wait for the jobs, expected to finish in 30 minutes
  gk8s_wait_for_callback() { [[ $1 -eq 0 ]]; }
  gk8s_wait_for 1800 \
    "(Failed)|(Complete)" \
    jobs -n "local" -l mineHashId="$(git_image_tag)" \
    -o jsonpath='{range .items[*]}{.metadata.name};{range .status.conditions[*]}{.type};{.reason}{"\n"}{end}{end}'

  # Look for any job that success
  gk8s_wait_for_callback() { [[ $1 -ge 1 ]]; }
  gk8s_wait_for 1 \
    "Failed" \
    jobs -n "local" -l mineHashId="$(git_image_tag)" \
    -o jsonpath='{range .items[*]}{.metadata.name};{range .status.conditions[*]}{.type};{.reason}{"\n"}{end}{end}' \
  || {
    set +x
    echo "================================================================"
    echo "Some jobs have failed."
    echo "Container logs below:"
    echo "================================================================"
    gk8s ":${K8S_CLUSTER}" get pods -n local -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \
    | while read -r pod; do
        echo "================================================================"
        echo "Pod: $pod"
        echo "================================================================"
        gk8s ":${K8S_CLUSTER}" logs --all-containers=true --tail=100000 --timestamps=true -n local "$pod"
      done
    echo "================================================================"
    return 1
  }

  set +x
  # Test logs
  echo "================================================================"
  echo "Container logs below:"
  echo "================================================================"

  gk8s ":${K8S_CLUSTER}" logs --all-containers=true --tail=100000 --timestamps=true -n local -l job="${RESOURCE_PREFIX_}end2end-tester" || true

  echo "================================================================"
  echo "All tests passed."
  echo "================================================================"

  set -x
}