AWS Airgap Prep Script with k0rdent Pre-requisites installed on all the airgapped nodes

updated-airgap-prepare-with-k0rdent-prereq.sh

#!/bin/bash

# ############################################################################################################
#
# This script prepares airgapped nodes deployed with TF stack located within this folder.
#
# See README.md for more details.
#
# Usage: ./airgap-prepare.sh
#
# ############################################################################################################

set -e

#comment out if on linux
#if ! command -v ggrep 2>&1 >/dev/null
#then
#    echo "ggrep could not be found; install it with 'brew install grep'"
#    echo "if you're on linux, not macos, find and replace 'ggrep' with 'grep', sorry about that"
#    exit 1
#fi

: "${TMPDIR:=/tmp}"
: "${WITH_MKE3:=false}"
: "${BUNDLE_NAME:="mke_bundle_dev.tar.gz"}"

timestamp() {
  echo "[INFO] $(date -u "+%Y-%m-%dT%H:%M:%SZ") \"$1\""
}

with_retry () {
    local max_retries=3
    local count=0
    local sleep_time=5

    until "$@"; do
        count=$((count + 1))
        if [ "$count" -ge "$max_retries" ]; then
            echo "Command failed after $max_retries attempts: $*"
            return 1
        fi
        sleep $((sleep_time * count))
    done
}

remote_command() {
  with_retry ssh -F ./.ssh_config "$1" "${@:2}"
}

join() {
  local IFS=" "
  shift
  echo "$*"
}

timestamp "Set variables"

files_folder=airgap-files
binary_name=mkectl-linux
ssh_key_path=$(terraform output -raw ssh_key_path)
msr_version=v4.0.2
bundle_name=$BUNDLE_NAME
remote_home_dir=$(remote_command airgapped-manager-0 "pwd")

mke3_version="3.8.4"
mke3_bundle_url="https://packages.mirantis.com/caas/ucp_images_$mke3_version.tar.gz"
mke3_bundle_name="ucp_images_$mke3_version.tar.gz"
mke3_kubectl_version='v1.32.3'

skopeo_version="v1.18.0"
skopeo_image="quay.io/skopeo/stable:$skopeo_version"
skopeo_tar="skopeo_$skopeo_version.tar.gz"
run_skopeo="docker run --rm -v $remote_home_dir/.docker/config.json:/config.json -v ./bundle:/bundle $skopeo_image"

# following substituion does not work
# run_skopeo="/usr/local/bin/skopeo"

first_manager_ip=$(yq '.spec.hosts[0].ssh.address' mke4.yaml)
ingress_https_port=$(yq '.spec.ingressController.nodePorts.https' mke4.yaml)
msr4_port=8080
admin_password=orcaorcaorca
registry_hostname="registry.mke4dev.mirantis.net"
registry_address="$registry_hostname:$msr4_port"
zone_filename="db.$registry_hostname"

cosign_version="v2.4.1"
helm_version="v3.16.3"
kubectl_version="v1.30.0"


timestamp "Verifying presence of the expected files"

required_files=(
  "$files_folder/$bundle_name"
  "$files_folder/harbor/tls.crt"
  "$files_folder/harbor/tls.key"
)

for file in "${required_files[@]}"; do
  if [ ! -f "$file" ]; then
    echo "$(basename "$file") must be present at $file"
    echo "See README.md for details"
    exit 1
  fi
done

timestamp "Compile mkectl for linux"

BINARY_NAME=$binary_name make --directory ../.. build-amd64

timestamp "Prepare files for transferring"

mkdir -p "./$files_folder"
remote_command bastion "mkdir -p ~/$files_folder/docker && mkdir -p ~/$files_folder/skopeo"

# rename mkectl-linux to mkectl
mv "../../bin/$binary_name" "./$files_folder/mkectl"

# rename ssh key
cp "$ssh_key_path" "./$files_folder/ssh_key.pem"

# replace ssh path in mke4.yaml with the correct path on the remote machine
sed "s|$ssh_key_path|$remote_home_dir/ssh_key.pem|g" mke4.yaml > "$files_folder/mke4.yaml"
sed "s|$ssh_key_path|$remote_home_dir/ssh_key.pem|g" nodes.yaml > "$files_folder/nodes.yaml"

if [[ "$WITH_MKE3" == true ]]; then
  timestamp "Download kubectl and bundle for MKE 3"
  remote_command bastion "stat ~/$files_folder/$mke3_bundle_name >/dev/null || wget $mke3_bundle_url -O ~/$files_folder/$mke3_bundle_name"
  remote_command bastion "curl -L https://dl.k8s.io/release/$mke3_kubectl_version/bin/linux/amd64/kubectl -o ~/$files_folder/kubectl && chmod +x ~/$files_folder/kubectl"
fi

timestamp "Download required binaries: kubectl, helm, cosign"

# Download kubectl
remote_command bastion "curl -L https://dl.k8s.io/release/$kubectl_version/bin/linux/amd64/kubectl -o ~/$files_folder/kubectl && chmod +x ~/$files_folder/kubectl"

# Download helm
remote_command bastion "curl -L https://get.helm.sh/helm-$helm_version-linux-amd64.tar.gz -o ~/$files_folder/helm.tar.gz && \
  tar -xzvf ~/$files_folder/helm.tar.gz -C ~/$files_folder && \
  mv ~/$files_folder/linux-amd64/helm ~/$files_folder/helm && \
  chmod +x ~/$files_folder/helm"

# Download cosign
remote_command bastion "curl -L https://github.com/sigstore/cosign/releases/download/$cosign_version/cosign-linux-amd64 -o ~/$files_folder/cosign && chmod +x ~/$files_folder/cosign"


timestamp "Installing Go on bastion"

GO_VERSION="1.24.4"
GO_TARBALL="go${GO_VERSION}.linux-amd64.tar.gz"
GO_URL="https://go.dev/dl/${GO_TARBALL}"

remote_command bastion "
  set -e
  echo '[INFO] Downloading Go \$GO_VERSION from ${GO_URL}...'
  wget -q '${GO_URL}' -O /tmp/${GO_TARBALL}
  echo '[INFO] Removing previous Go installation if any...'
  sudo rm -rf /usr/local/go
  echo '[INFO] Extracting Go...'
  sudo tar -C /usr/local -xzf /tmp/${GO_TARBALL}
  echo '[INFO] Updating PATH...'
  echo 'export PATH=\$PATH:/usr/local/go/bin' >> ~/.bashrc
  export PATH=\$PATH:/usr/local/go/bin
  echo '[INFO] Verifying Go installation...'
  go version
"

timestamp "Installing skopeo build dependencies (including make) on bastion"

remote_command bastion "
  sudo apt-get update && \
  sudo apt-get install -y build-essential pkg-config \
    libgpgme-dev libassuan-dev libbtrfs-dev \
    libdevmapper-dev libglib2.0-dev libseccomp-dev \
    libostree-dev libselinux1-dev uidmap btrfs-progs libgpg-error-dev
"

timestamp "Compiling required binaries: skopeo"

remote_command bastion "
  export PATH=\$PATH:/usr/local/go/bin && \
  cd ~ && \
  rm -rf skopeo && \
  git clone --depth 1 --branch $skopeo_version https://github.com/containers/skopeo.git && \
  cd skopeo && \
  make bin/skopeo
"

remote_command bastion "cp ~/skopeo/bin/skopeo ~/$files_folder/skopeo-bin && chmod +x ~/$files_folder/skopeo-bin"


timestamp "Install skopeo on bastion"
remote_command bastion "sudo cp ~/$files_folder/skopeo-bin /usr/local/bin/skopeo && sudo chmod +x /usr/local/bin/skopeo"


timestamp "Download docker"

# TODO: this is a temporary solution until we need to get AMI with everything pre-installed

# URL of the HTML page with the list of Docker files
HTML_URL="https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/"
# Fetch the list of available packages
PACKAGE_LIST=$(curl -s "$HTML_URL" | grep -oP '(?<=href=")[^"]+(?=")' | grep -oP '.*\.deb' | sort -Vr)
# Function to download a package with a progress bar
download_package() {
  FILENAME=$1
  URL="${HTML_URL}${FILENAME}"
  timestamp "Downloading $FILENAME..."
  remote_command bastion "wget \"$URL\" -x -O \"$files_folder/docker/$FILENAME\""
}
# Download the latest package for each type
TYPES=("containerd.io" "docker-ce" "docker-ce-cli" "docker-buildx-plugin" "docker-compose-plugin")
PACKAGES=()

for TYPE in "${TYPES[@]}"; do
  PACKAGE=$(echo "$PACKAGE_LIST" | grep "$TYPE" | head -n 1)
  if [ -n "$PACKAGE" ]; then
    download_package "$PACKAGE"
    PACKAGES+=("$PACKAGE")
  fi
done

timestamp "Docker packages downloaded successfully!"

timestamp "Install Docker on Bastion"

remote_command bastion "docker || sudo dpkg -i $(printf " ~/$files_folder/docker/%s" "${PACKAGES[@]}") && sudo usermod -aG docker ubuntu"

timestamp "Download skopeo via docker"

remote_command bastion "stat ~/$files_folder/$skopeo_tar >/dev/null || docker pull $skopeo_image && docker save $skopeo_image > ~/$files_folder/$skopeo_tar"


timestamp "Prepping for additional packages"

remote_command bastion "sudo NEEDRESTART_MODE=a apt-get -y update"

ADD_PACKAGES=("dpkg-dev" "jq" "zip" "bind9" "bind9utils" "bind9-doc" "net-tools" "passwd")
for PKG in "${ADD_PACKAGES[@]}"; do
  remote_command bastion "sudo NEEDRESTART_MODE=a apt-get -y install $PKG"
done

remote_command bastion "sudo mkdir -p ~/$files_folder/add_pkg_repo && sudo cp -a /var/cache/apt/archives/*.deb ~/$files_folder/add_pkg_repo && cd ~/$files_folder/add_pkg_repo && sudo dpkg-scanpackages ./ /dev/null | sudo tee ./Packages >/dev/null"

timestamp "Download MSR 4"

if [ ! -f $files_folder/harbor/harbor.yml ]; then
  curl -L -o $files_folder/msr4.tar.gz https://s3-us-east-2.amazonaws.com/packages-mirantis.com/msr/harbor-offline-installer-$msr_version.tgz
  tar -xzf $files_folder/msr4.tar.gz -C $files_folder
  rm $files_folder/msr4.tar.gz
fi

timestamp "Prepare MSR 4"

if [ ! -f "$files_folder/harbor/harbor.yml" ]; then
  mv $files_folder/harbor/harbor.yml.tmpl $files_folder/harbor/harbor.yml
fi

yq -i ".hostname = \"$registry_hostname\"" $files_folder/harbor/harbor.yml
yq -i ".data_volume = \"$remote_home_dir/harbor/data\"" $files_folder/harbor/harbor.yml
yq -i ".http.port = \"8079\"" $files_folder/harbor/harbor.yml
yq -i ".https.port = \"$msr4_port\"" $files_folder/harbor/harbor.yml
yq -i ".https.certificate = \"$remote_home_dir/harbor/tls.crt\"" $files_folder/harbor/harbor.yml
yq -i ".https.private_key = \"$remote_home_dir/harbor/tls.key\"" $files_folder/harbor/harbor.yml
yq -i ".harbor_admin_password = \"$admin_password\"" $files_folder/harbor/harbor.yml

yq -i ".spec.registries.chartRegistry.url = \"oci://$registry_address/mke\"" $files_folder/mke4.yaml
yq -i ".spec.registries.imageRegistry.url = \"$registry_address/mke\"" $files_folder/mke4.yaml
# FIXME: AWS LB is very slow; we need to fix it before we can use it here
# until then, replace it with the first manager IP
yq -i ".spec.apiServer.externalAddress = \"$first_manager_ip:$ingress_https_port\"" $files_folder/mke4.yaml

remote_command airgapped-manager-0 "mkdir -p ~/harbor/data"

timestamp "Prepare DNS zone for MSR 4"

FIRST_MANAGER_IP="$first_manager_ip" MSR_HOSTNAME="$registry_hostname" envsubst '${FIRST_MANAGER_IP} ${MSR_HOSTNAME}' < "airgap-dns/zone.tpl" > "$files_folder/$zone_filename"

timestamp "Transfer files"

# transfer all files as one archive using rsync
with_retry rsync -a -e "ssh -F ./.ssh_config" --progress "./$files_folder/" airgapped-manager-0:~/

for host in $(yq '.spec.hosts[].ssh.address' mke4.yaml); do
  with_retry rsync -a -e "ssh -F ./.ssh_config" --progress "./$files_folder/ssh_key.pem" bastion:~/
  remote_command bastion "rsync -a -e \"ssh -i ssh_key.pem -o StrictHostKeyChecking=no\" --progress \"$files_folder/\" \"ubuntu@$host:~/\""
done

timestamp "Setup nodes"

add_zone=$(cat <<EOF
zone \"$registry_hostname\" {
    type master;
    file \"/etc/bind/zones/db.$registry_hostname\";
};
EOF
)

default_dns="172.31.0.2"
named_conf_options=$(cat <<EOF
options {
    directory \"/var/cache/bind\";

    forwarders {
        $default_dns;
    };

    allow-query { any; };
    recursion yes;

    dnssec-validation no;
};
EOF
)

resolved_conf=$(cat <<EOF
DNS=$first_manager_ip
FallbackDNS=
Domains=~.
EOF
)

hosts=$(cat .ssh_config | grep "airgapped-" | awk '{print $2}')
for host in $hosts; do
  timestamp "Install packages on $host"
  remote_command "$host" "sudo sh -c \"echo 'deb [trusted=yes] file://$remote_home_dir/add_pkg_repo ./' > /etc/apt/sources.list.d/myrepo.list && rm -f /etc/apt/sources.list && apt-get update -y && apt install -y $(join "${ADD_PACKAGES[@]}") --allow-unauthenticated\""

  timestamp "Install Docker on $host"
  remote_command "$host" "docker || sudo dpkg -i $(printf " ~/docker/%s" "${PACKAGES[@]}") && sudo usermod -aG docker ubuntu"

  timestamp "Install kubectl, helm, and cosign on $host"
  remote_command "$host" "sudo mv ~/kubectl /usr/local/bin/kubectl"
  remote_command "$host" "sudo mv ~/helm    /usr/local/bin/helm"
  remote_command "$host" "sudo mv ~/cosign  /usr/local/bin/cosign"
  remote_command "$host" "sudo mv ~/skopeo-bin /usr/local/bin/skopeo"

  if [[ "$WITH_MKE3" == true ]]; then
    remote_command "$host" "docker image inspect mirantis/ucp-dsinfo:3.8.4 >/dev/null || docker load -i ~/$mke3_bundle_name"
  fi

  # try resolving registry hostname; the output should contain the first manager IP
  # if it doesn't, the DNS isn't configured
  dig_registry=$(remote_command "$host" "dig $registry_hostname @127.0.0.1")
  if [[ "$dig_registry" != *"$first_manager_ip"* ]] ; then
    # install and configure DNS server for private registry
    timestamp "Install and configure DNS server on $host"
    rsync -a -e "ssh -F ./.ssh_config" --progress "./$files_folder/$zone_filename" "$host:~/"
    remote_command "$host" "sudo grep \"$registry_hostname\" /etc/bind/named.conf.local || sudo sh -c \"echo '$add_zone' > /etc/bind/named.conf.local\""
    remote_command "$host" "stat '/etc/bind/zones/$zone_filename' >/dev/null || sudo mkdir -p /etc/bind/zones && sudo cp '$remote_home_dir/$zone_filename' '/etc/bind/zones/$zone_filename'"
    remote_command "$host" "sudo grep \"$default_dns\" /etc/bind/named.conf.options || sudo sh -c \"echo '$named_conf_options' > /etc/bind/named.conf.options\""
    remote_command "$host" "sudo systemctl restart bind9"
    remote_command "$host" "sudo grep \"$first_manager_ip\" /etc/systemd/resolved.conf || sudo sh -c \"echo '$resolved_conf' >> /etc/systemd/resolved.conf\""
    remote_command "$host" "sudo systemctl restart systemd-resolved"
  fi

  if [[ "$host" == "airgapped-manager-0" ]]; then
    timestamp "Install skopeo on $host"
    remote_command "$host" "docker image inspect $skopeo_image >/dev/null || docker load -i ~/$skopeo_tar"
  fi
done

# move binaries to /usr/local/bin
remote_command airgapped-manager-0 "sudo mv ~/mkectl /usr/local/bin/mkectl"

timestamp "Setup MSR 4 and create 'mke' project"

msr_exists=$(remote_command airgapped-manager-0 "[[ \$(docker ps -f name=harbor-core -q) ]] && echo 'exists' || echo 'please install'")
if [[ "$msr_exists" == "exists" ]]; then
  timestamp "MSR 4 is already installed. Skipping installation."
else
  remote_command airgapped-manager-0 "cd ~/harbor && sudo ./install.sh"
  # let MSR 4 start
  sleep 10
fi

remote_command airgapped-manager-0 "curl -s -X POST -H \"Content-Type: application/json\" \
  -d '{\"project_name\": \"mke\",\"public\": true}' \
  https://admin:$admin_password@$registry_address/api/v2.0/projects || true"

timestamp "Unpack MKE 4 charts and images"

#removed --multi-arch all flag
remote_command airgapped-manager-0 "docker login $registry_address -u admin -p $admin_password && \
  tar -xzf $bundle_name -C ./ && \
  for archive in \$(find ./bundle -print | grep \".tar\"); do \
    img=\$(basename \"\$archive\" | sed 's~\.tar~~' | tr '&' '/' | tr '@' ':'); \
    echo \"Uploading \$img\"; \
    $run_skopeo copy --authfile=/config.json -q --retry-times 3 --multi-arch all oci-archive:\${archive#\".\"} docker://$registry_address/mke/\$img; \
  done;"

echo ''
echo '------------------------------- SUCCESS!!! -------------------------------'
echo ''
echo "Here are some handy commands for interacting with your airgapped environment:"
echo "- 'ssh -F ./.ssh_config airgapped-manager-0' to connect to the manager node"
echo "- 'ssh -F ./.ssh_config -L \"8080:localhost:$msr4_port\" ubuntu@airgapped-manager-0' to port-forward MSR 4 and access it at https://localhost:8080"
echo "- Airgap Installation: SSH into airgapped-manager-0 and run 'mkectl apply'"
echo "- Airgap Upgrade: Follow the instructions in the README.md file to install MKE 3 and upgrade to MKE 4"
echo "- When MKE 4 is running, use 'ssh -F ./.ssh_config -L \"3000:localhost:$ingress_https_port\" ubuntu@airgapped-manager-0' to port-forward MKE 4 Dashboard and access it at https://localhost:3000"
echo ''