Apigee Hybrid Gather All #sh

apigee-hybrid-gather-all.sh

#!/bin/bash
# -*- mode:shell-script; coding:utf-8; sh-shell:bash -*-

# Copyright © 2025 Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

set -e
#set -x
SVERSION="20251121-1417"

check_required_commands() {
  local missing
  missing=()
  for cmd in "$@"; do
    #printf "checking %s\n" "$cmd"
    if ! command -v "$cmd" &>/dev/null; then
      missing+=("$cmd")
    fi
  done
  if [[ -n "$missing" ]]; then
    printf -v joined '%s,' "${missing[@]}"
    printf "\nThese commands are missing; they must be available on path: %s\nExiting.\n\n" "${joined%,}"
    exit 1
  fi
}

check_shell_variables() {
  local MISSING_ENV_VARS
  MISSING_ENV_VARS=()
  for var_name in "$@"; do
    if [[ -z "${!var_name}" ]]; then
      MISSING_ENV_VARS+=("$var_name")
    fi
  done

  [[ ${#MISSING_ENV_VARS[@]} -ne 0 ]] && {
    printf -v joined '%s,' "${MISSING_ENV_VARS[@]}"
    printf "\nMissing these required environment variables: %s\nExiting.\n\n" "${joined%,}"
    exit 1
  }

  printf "Settings in use:\n"
  for var_name in "$@"; do
    if [[ "$var_name" == *_APIKEY || "$var_name" == *_SECRET || "$var_name" == *_CLIENT_ID ]]; then
      local value="${!var_name}"
      local dots
      dots=$(printf '%*s' "${#value}" '' | tr ' ' '.')
      printf "  %s=%s\n" "$var_name" "${value:0:4}${dots}"
    else
      printf "  %s=%s\n" "$var_name" "${!var_name}"
    fi
  done
  printf "\n"
}

format_elapsed_time() {
  # Calculate total elapsed time in seconds
  local total_seconds end_seconds start_seconds minutes remainder_seconds formatted_seconds
  start_seconds=$1
  end_seconds=$2
  total_seconds=$((end_seconds - start_seconds))
  minutes=$((total_seconds / 60))
  remainder_seconds=$((total_seconds % 60))
  formatted_seconds=$(printf "%02d" "$remainder_seconds")
  echo "${minutes}m${formatted_seconds}s ${total_seconds}"
}

log_message() {
  echo "$@" >>"$MANIFEST_FILE"
}

# ====================================================================

script_name_short="${0##*/}"
printf "\n\nHello from %s\n\n" "$script_name_short"
printf "This script gathers information about the Apigee hybrid resources\n"
printf "in your Kubernetes cluster, and bundles it into a tgz archive.\n\n"

if [[ -z "$APIGEE_NS" ]]; then
  read -p "Enter the k8s namespace used for Apigee: " APIGEE_NS
  if [[ -z "$APIGEE_NS" ]]; then
    printf "\nNo namespace was provided. Exiting.\n\n"
    exit 1
  fi
fi

check_shell_variables APIGEE_NS
check_required_commands kubectl tee date

printf "Gathering Apigee hybrid information into %s ...\n" "$tmpdir"
START_SECONDS=$(date +%s)
START_TIME_HUMAN=$(date +"%Y-%m-%d %H:%M:%S %Z")
SCRIPT_NAME="${0##*/}"
CURRENT_DIR="$(pwd)"
tmpdir=$(mktemp -d)
MANIFEST_FILE="$tmpdir/Manifest.txt"

log_message "########################################"
log_message "Scripted https://docs.cloud.google.com/apigee/docs/support/best-practices#apigee-hybrid-must-gather"
log_message "Script version:    $SVERSION"
log_message "Script name:       $script_name_short"
log_message "Start Time:        $START_SECONDS"
log_message "Start Time:        $START_TIME_HUMAN"
log_message "Script Name:       $SCRIPT_NAME"
log_message "Current Directory: $CURRENT_DIR"
log_message "Temp dir:          $tmpdir"
log_message "---"

log_message "Gathering information..."

# Gather information

###--- "kubectl config" commands to get the config details of the whole Apigee Hybrid cluster ---####

kubectl config get-clusters 2>&1 | tee $tmpdir/config_get_clusters_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl config get-contexts 2>&1 | tee $tmpdir/config_get_contexts_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl config get-users 2>&1 | tee $tmpdir/config_get_users_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl config view 2>&1 | tee $tmpdir/config_view_$(date +%Y.%m.%d_%H.%M.%S).txt

### --- Collect all details of all nodes in the Kubernetes cluster.---###

kubectl describe node 2>&1 | tee $tmpdir/describe_node_$(date +%Y.%m.%d_%H.%M.%S).txt

###--- "kubectl get -A " commands to get CRD details for the whole Apigee Hybrid setup ---####

kubectl get clusterissuers -A -o wide 2>&1 | tee $tmpdir/get_clusterissuers_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get certificate -A -o wide 2>&1 | tee $tmpdir/get_certificate_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get certificaterequest -A -o wide 2>&1 | tee $tmpdir/get_certificaterequest_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get crd -A -o yaml 2>&1 | tee $tmpdir/get_crd_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get ConfigMap -A 2>&1 | tee $tmpdir/get_ConfigMap_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get ClusterRole -A -o wide 2>&1 | tee $tmpdir/get_clusterrole_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get ClusterRoleBinding -A -o wide 2>&1 | tee $tmpdir/get_clusterrole_binding_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get Deployments -A -o wide >&1 | tee $tmpdir/get_deployments_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get events -A -o wide 2>&1 | tee $tmpdir/get_events_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get endpoints -A 2>&1 | tee $tmpdir/get_endpoints_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get issuers -A -o wide 2>&1 | tee $tmpdir/get_issuers_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get mutatingwebhookconfigurations 2>&1 | tee $tmpdir/get_mutatingwebhookconfigurations_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get nodes -o wide --show-labels 2>&1 | tee $tmpdir/get_nodes_labels_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get ns 2>&1 | tee $tmpdir/get_namespace_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get PriorityClass -A -o wide 2>&1 | tee $tmpdir/get_PriorityClass_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get pv -A -o wide 2>&1 | tee $tmpdir/get_pv_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get pvc -A -o wide 2>&1 | tee $tmpdir/get_pvc_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get Role -A -o wide 2>&1 | tee $tmpdir/get_role_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get RoleBinding -A -o wide 2>&1 | tee $tmpdir/get_Role_Binding_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get replicaset -A -o wide 2>&1 | tee $tmpdir/get_replicaset_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get sa -A -o wide 2>&1 | tee $tmpdir/get_service_accounts_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get services -A -o wide 2>&1 | tee $tmpdir/get_services_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get svc -A 2>&1 | tee $tmpdir/get_svc_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get secrets -A 2>&1 | tee $tmpdir/get_secrets_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get validatingwebhookconfigurations -A 2>&1 | tee $tmpdir/get_validatingwebhookconfigurations_all_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get validatingwebhookconfigurations apigee-validating-webhook-configuration 2>&1 |
  tee $tmpdir/get_apigee-validating-webhook-configuration_$(date +%Y.%m.%d_%H.%M.%S).txt

### --- List top resource consuming nodes and pods ---####

kubectl top nodes 2>&1 | tee $tmpdir/top_nodes_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl top pod -A --containers 2>&1 | tee $tmpdir/top_pod_all_containers_$(date +%Y.%m.%d_%H.%M.%S).txt

###----- "kubectl get" commands to fetch list of all CRD for "apigee" namespace ----- #####

kubectl get all -n $APIGEE_NS -o wide 2>&1 | tee $tmpdir/get_all_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get ad -n $APIGEE_NS 2>&1 | tee $tmpdir/get_ad_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get apigeeorganization -n $APIGEE_NS 2>&1 | tee $tmpdir/get_apigeeorganization_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get apigeeenv -n $APIGEE_NS 2>&1 | tee $tmpdir/get_apigeeenv_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get apigeeds -n $APIGEE_NS 2>&1 | tee $tmpdir/get_apigeeds_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get apigeedatastore -n $APIGEE_NS 2>&1 | tee $tmpdir/get_apigeedatastore_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get ApigeeDeployment -n $APIGEE_NS 2>&1 | tee $tmpdir/get_apigeedeployment_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get ApigeeRedis -n $APIGEE_NS 2>&1 | tee $tmpdir/get_ApigeeRedis_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get ApigeeRoute -n $APIGEE_NS 2>&1 | tee $tmpdir/get_ApigeeRoute_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get ApigeeRouteConfig -n $APIGEE_NS 2>&1 | tee $tmpdir/get_ApigeeRoutesconfig_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get arc -n $APIGEE_NS -o yaml 2>&1 | tee $tmpdir/get_arc_yaml_$(date +%Y.%m.%d_%H.%M.%S).yml
kubectl get Apigeetelemetry -n $APIGEE_NS 2>&1 | tee $tmpdir/get_Apigeetelemetry_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get apigeeissues -n $APIGEE_NS 2>&1 | tee $tmpdir/get_apigeeissues_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get ControllerRevision -n $APIGEE_NS -o wide 2>&1 | tee $tmpdir/get_ControllerRevision_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get cronjob -n $APIGEE_NS -o wide 2>&1 | tee $tmpdir/get_cronjob_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get gateway -n $APIGEE_NS 2>&1 | tee $tmpdir/get_gateway_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get PodDisruptionBudget -n $APIGEE_NS -o wide 2>&1 | tee $tmpdir/get_PodDisruptionBudget_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get sc -n $APIGEE_NS -o wide 2>&1 | tee $tmpdir/get_storageclass_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get secrets -n $APIGEE_NS -o wide 2>&1 | tee $tmpdir/get_secrets_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get sr -n $APIGEE_NS -o wide 2>&1 | tee $tmpdir/get_sr_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get sts -n $APIGEE_NS 2>&1 | tee $tmpdir/get_sts_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get volumesnapshot -n $APIGEE_NS -o wide 2>&1 | tee $tmpdir/get_volumesnapshot_n_apigee_$(date +%Y.%m.%d_%H.%M.%S).txt

###----- "kubectl describe" commands to fetch details of all CRD for "apigee" namespace ----- #####

for p in $(kubectl -n $APIGEE_NS get apigeeorganization --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe apigeeorganization ${p} -n $APIGEE_NS 2>&1 | tee $tmpdir/desc_apigeeorganization_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get apigeeenv --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe apigeeenv ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_apigeeenv_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get apigeeds --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe apigeeds ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_apigeeds_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get apigeedatastore --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe apigeedatastore ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_apigeedatastore_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get ApigeeDeployment --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe ApigeeDeployment ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_ApigeeDeployment_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get ApigeeRedis --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe ApigeeRedis ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_ApigeeRedis_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get ApigeeRoute --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe ApigeeRoute ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_ApigeeRoute_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get ApigeeRouteConfig --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe ApigeeRouteConfig ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_ApigeeRouteConfig_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get Apigeetelemetry --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe Apigeetelemetry ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_Apigeetelemetry_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get apigeeissues --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe apigeeissues ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_apigeeissues_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get ControllerRevision --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n $APIGEE_NS describe ControllerRevision ${p} 2>&1 |
    tee $tmpdir/desc_ControllerRevision_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get certificate --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe certificate ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_certificate_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get cronjob --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe cronjob ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_cronjob_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get daemonset --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n $APIGEE_NS describe daemonset ${p} 2>&1 |
    tee $tmpdir/desc_daemonset_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get deployments --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n $APIGEE_NS describe deployments ${p} 2>&1 |
    tee $tmpdir/desc_deployment_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get hpa --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe hpa ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_hpa_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get jobs --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe jobs ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_jobs_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get po --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n $APIGEE_NS get po ${p} -o yaml 2>&1 |
    tee $tmpdir/get_pod_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
  kubectl -n $APIGEE_NS describe po ${p} 2>&1 |
    tee $tmpdir/desc_pod_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get po --no-headers -o custom-columns=":metadata.name" -l app=apigee-ingressgateway); do
  kubectl -n $APIGEE_NS logs ${p} --all-containers 2>&1 | tee $tmpdir/logs_apigee_ingress_gateways_${p}_$(date +%Y.%m.%d_%H.%M.%S).log
  kubectl -n $APIGEE_NS exec ${p} -- /usr/local/bin/pilot-agent request GET config_dump 2>&1 | tee $tmpdir/apigee_ingress_config_dump_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n apigee get po --no-headers -o custom-columns=":metadata.name" -l app=apigee-ingressgateway-manager); do
  kubectl -n $APIGEE_NS logs ${p} -c discovery 2>&1 | tee $tmpdir/logs_apigee_ingress_gateway_manager_discovery_${p}_$(date +%Y.%m.%d_%H.%M.%S).log
  kubectl -n $APIGEE_NS logs ${p} -c manager 2>&1 | tee $tmpdir/logs_apigee_ingress_gateway_manager_manager_${p}_$(date +%Y.%m.%d_%H.%M.%S).log
  kubectl -n $APIGEE_NS logs ${p} -c kube-rbac-proxy 2>&1 | tee $tmpdir/logs_apigee_ingress_gateway_manager_kube-rbac-proxy_${p}_$(date +%Y.%m.%d_%H.%M.%S).log
  kubectl -n $APIGEE_NS exec -it ${p} -c discovery -- pilot-discovery request GET debug/configz 2>&1 | tee $tmpdir/apigee_ingress_gateway_manager_configz_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get PodDisruptionBudget --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n $APIGEE_NS describe PodDisruptionBudget ${p} 2>&1 |
    tee $tmpdir/desc_PodDisruptionBudget_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get pv --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n $APIGEE_NS describe pv ${p} 2>&1 |
    tee $tmpdir/desc_pv_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get pvc --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n $APIGEE_NS describe pvc ${p} 2>&1 |
    tee $tmpdir/desc_pvc_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get rs --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe rs ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_replicaset_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get sc --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n $APIGEE_NS describe sc ${p} 2>&1 |
    tee $tmpdir/desc_storageclass_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get sts --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe sts ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_sts_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get secrets --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n $APIGEE_NS describe secrets ${p} 2>&1 |
    tee $tmpdir/desc_secrets_n_apigee${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get services --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe service ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_services_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get sa --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe sa ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_service_account_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n $APIGEE_NS get svc --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe svc ${p} -n $APIGEE_NS 2>&1 |
    tee $tmpdir/desc_svc_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done

###----- "kubectl logs" command to fetch logs of all containers in the "apigee" namespace ----- #####

for p in $(kubectl -n $APIGEE_NS get po --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n $APIGEE_NS logs ${p} --all-containers 2>&1 |
    tee $tmpdir/logs_n_apigee_${p}_$(date +%Y.%m.%d_%H.%M.%S).log
done

###----- "kubectl get" commands for "apigee-system" namespace ----- #####

kubectl get all -n apigee-system -o wide 2>&1 |
  tee $tmpdir/get_all_n_apigee_system_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get jobs -o wide -n apigee-system 2>&1 |
  tee $tmpdir/get_jobs_n_apigee_system_$(date +%Y.%m.%d_%H.%M.%S).txt

###----- "kubectl describe" commands for "apigee-system" namespace ----- #####

for p in $(kubectl -n apigee-system get certificate --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe certificate ${p} -n apigee-system 2>&1 |
    tee $tmpdir/desc_certificate_n_apigee_system_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n apigee-system get deployment --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe deployment ${p} -n apigee-system 2>&1 |
    tee $tmpdir/desc_deployment_n_apigee_system_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n apigee-system get jobs --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe jobs ${p} -n apigee-system 2>&1 |
    tee $tmpdir/desc_jobs_n_apigee_system_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n apigee-system get po --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n apigee-system describe po ${p} 2>&1 |
    tee $tmpdir/desc_pod_n_apigee_system_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n apigee-system get rs --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe rs ${p} -n apigee-system 2>&1 |
    tee $tmpdir/desc_replicaset_n_apigee_system_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n apigee-system get rolebinding --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe rolebinding ${p} -n apigee-system 2>&1 |
    tee $tmpdir/desc_rolebinding_n_apigee_system_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n apigee-system get services --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe service ${p} -n apigee-system 2>&1 |
    tee $tmpdir/desc_services_n_apigee_system_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n apigee-system get sa --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe sa ${p} -n apigee-system 2>&1 |
    tee $tmpdir/desc_serviceaccount_n_apigee_system_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n apigee-system get secrets --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe secrets ${p} -n apigee-system 2>&1 |
    tee $tmpdir/desc_secrets_n_apigee_system_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done

###----- "kubectl logs" command for "apigee-system" namespace ----- #####

for p in $(kubectl -n apigee-system get po --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n apigee-system logs ${p} --all-containers 2>&1 |
    tee $tmpdir/logs_n_apigee_system_${p}_$(date +%Y.%m.%d_%H.%M.%S).log
done

###----- "kubectl get" command for "cert-manager" namespace ----- #####

kubectl get all -n cert-manager -o wide 2>&1 |
  tee $tmpdir/get_all_n_cert_manager_$(date +%Y.%m.%d_%H.%M.%S).txt
kubectl get crd -n cert-manager 2>&1 |
  tee $tmpdir/get_crd_n_cert_manager_$(date +%Y.%m.%d_%H.%M.%S).txt

###----- "kubectl describe" command for "cert-manager" namespace ----- #####

for p in $(kubectl -n cert-manager get deployment --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n cert-manager describe deployment ${p} 2>&1 |
    tee $tmpdir/desc_deployment_n_cert_manager_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n cert-manager get endpoints --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe endpoints ${p} -n cert-manager 2>&1 |
    tee $tmpdir/desc_endpoints_n_cert_manager_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n cert-manager get po --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n cert-manager describe po ${p} 2>&1 |
    tee $tmpdir/desc_po_n_cert_manager_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n cert-manager get rs --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe rs ${p} -n cert-manager 2>&1 |
    tee $tmpdir/desc_replicaset_n_cert_manager_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n cert-manager get sa --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe sa ${p} -n cert-manager 2>&1 |
    tee $tmpdir/desc_serviceaccount_n_cert_manager_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n cert-manager get secrets --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe secrets ${p} -n cert-manager 2>&1 |
    tee $tmpdir/desc_secrets_n_cert_manager_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n cert-manager get services --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe service ${p} -n cert-manager 2>&1 |
    tee $tmpdir/desc_service_n_cert_manager_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done
for p in $(kubectl -n cert-manager get svc --no-headers -o custom-columns=":metadata.name"); do
  kubectl describe svc ${p} -n cert-manager 2>&1 |
    tee $tmpdir/desc_svc_n_cert_manager_${p}_$(date +%Y.%m.%d_%H.%M.%S).txt
done

###----- "kubectl logs" command for "cert-manager" namespace ----- #####

for p in $(kubectl -n cert-manager get po --no-headers -o custom-columns=":metadata.name"); do
  kubectl -n cert-manager logs ${p} --all-containers 2>&1 |
    tee $tmpdir/logs_n_cert_manager_${p}_$(date +%Y.%m.%d_%H.%M.%S).log
done

### ====================================================================

END_SECONDS=$(date +%s)
END_TIME_HUMAN=$(date +"%Y-%m-%d %H:%M:%S %Z")

read RUNTIME ELAPSED_SECONDS < <(format_elapsed_time "$START_SECONDS" "$END_SECONDS")

log_message "finished."
log_message "End Time:          $END_SECONDS"
log_message "End Time:          $END_TIME_HUMAN"
log_message "Elapsed Time:      $ELAPSED_SECONDS seconds"
log_message "Elapsed Time:      $RUNTIME"
log_message "########################################"

tgzfile=/tmp/apigee-gathered-data__$(date +%Y.%m.%d_%H.%M.%S).tar.gz

(
  cd $tmpdir
  tar -cvzf $tgzfile .
)

printf "\n\n\n============================================\nTGZ file:\n   %s\n\n" "$tgzfile"
printf "\n\nTo remove the temporary files:\n  rm -fr $tmpdir\n\n"