Kubevirt development automation for Kubevirt updates and backward compatabilty scenerios

iterate.sh

# This script enable faster development iterations in Kubevirt
# when working on a change that might affect backward compatibility,
# upgrades or legacy VMs (a VM that existed before an upgrade) disruption.
#
# Use case:
# 1. Changing virt-launcher pods secondary networks interfaces name scheme.
#    Different network interfaces name-schemes breaks migration for legacy VMs
#    when Kubevirt is updated.
#
#    In order to test this, its necessary to have both versions images,
#    the initial version (latest stable/nightly) and a version with the new
#    name-scheme bits.
#    And introduce an e2e test that creates a VM, performs kubevirt upgrade and
#    verify that the VM is functional and can be migrated successfully.
#
#
# Definitions:
# previous-version - Kubervirt version upgrading from.
# target-version   - Kubervirt version upgrading to.
# Example:
# Kubevirt upgrade from v0.58.0 to v0.59.0
# previous-version would be v0.58.0 and target-version is v0.59.0
#
# Environment variables:
#   PREV_VER_BRANCH:      Controls the previous-version branch name where previous version Kubevirt is built from (default: release-0.58).
#   PREV_VER_TAG:         Controls the previous-version Kubevirt images tag (default: v0.58.0)
#   PREV_VER_REGISTRY:    Controls the previous-version Kubevirt images registry (default: quay.io/kubevirt).
#   TARGET_VER_BRANCH:    Controls the target-version branch name where target version Kubevirt is built to (default: main).
#   TARGET_VER_TAG:       Controls the target-version Kubevirt images tag (default: latest).
#   TARGET_VER_REGISTRY:  Controls the target-version Kubevirt images registry (default: registry:5000/kubevirt).
#
# Flags:
#   --build-previous-version:   Checkout previous-version branch Kubevirt upgrading from, build and push the images to the previous-version registry.
#                               Note: Changes that are committed are stashed just before checkout the previous-version branch, the stash entry identifier is "iterate".
#
#   --build-target-version:     Checkout the target-version branch Kubevirt upgrading to, build and push the images to the target-version registry.
#                               Note: Changes that were not committed are popd form the "iterate" stash entry.
#
#   --functest:                 Run kubevirt e2e tests.
#                               Note: All necessary flags are passed to ensure the correct images are used in tests.
#                               Export KUBEVIRT_E2E_FOCUS env var to run specific tests.
#
#   --cluster-up:               Spin up ephemeral cluster using github.com/kubevirt/kubevirtci providers.
#
#   --deploy-previous-version:  Deploy Kubevirt that was build from the previous version branch.
#                               Note: The Kubevirt operator is deployed from the target version branch in order to have the latest Kubevirt CRDs.
#
#   --deploy-target-version:    Deploy Kubevirt that was build from the target version branch.
#
# How to use:
# Test Kubevirt upgrade between two versions that were built from local branches:
# export TARGET_VER_BRANCH=new-pod-net-iface-namescheme
# export TARGET_VER_TAG=latest
# export TARGET_VER_REGISTRY=registry:5000/kubevirt
# export PREV_VER_BRANCH=main
# export PREV_VER_TAG=devel
# export PREV_VER_REGISTRY=registry:5000/kubevirt
#
# export KUBEVIRT_E2E_FOCUS="<upgrade test key>"
# ./dev-iterate.sh --build-previous-version --build-target-version --deploy-target-version --functest
#
# In case the test expects the previous version to be deployed:
# ./dev-iterate.sh --build-previous-version --build-target-version --deploy-previous-version --functest
#
#
#
#!/bin/bash

set -e

CLUSTER_UP=false
BUILD_PREVIOUS_VERSION_KUBEVIRT=false
BUILD_TARGET_VERSION_KUBEVIRT=false
DEPLOY_PREVIOUS_VERSION_KUBEVIRT=false
DEPLOY_TARGET_VERSION_KUBEVIRT=false
FUNCTEST=false

options=$(getopt --options "" \
    --long cluster-up,build-previous-version,build-target-version,deploy-previous-version,deploy-target-version,functest,help\
    -- "${@}")
eval set -- "$options"
while true; do
    case "$1" in
     --cluster-up)
        CLUSTER_UP=true
        ;;
    --build-previous-version)
        BUILD_PREVIOUS_VERSION_KUBEVIRT=true
        ;;
    --build-target-version)
        BUILD_TARGET_VERSION_KUBEVIRT=true
        ;;
    --deploy-previous-version)
        DEPLOY_PREVIOUS_VERSION_KUBEVIRT=true
        ;;
    --deploy-target-version)
        DEPLOY_TARGET_VERSION_KUBEVIRT=true
        ;;
    --functest)
        FUNCTEST=true
        ;;
    --help)
        set +x
        echo "$0 [--cluster-up] [--build-previous-version] [--build-target-version] [--deploy-previous-version] [--deploy-target-version] [--functest]"
        exit
        ;;
    --)
        shift
        break
        ;;
    esac
    shift
done

set -x

if ${CLUSTER_UP}; then
  export KUBEVIRT_PROVIDER="k8s-1.25"
  export KUBEVIRT_NUM_NODES="2"
  export KUBEVIRT_NUM_SECONDARY_NICS="1"
  export KUBEVIRT_WITH_CNAO="true"
  export KUBEVIRT_DEPLOY_ISTIO="true"
  export KUBEVIRT_DEPLOY_CDI="false"
  export KUBECONFIG=$(./cluster-up/kubeconfig.sh)

  make cluster-up
fi

if ! kubectl cluster-info; then
  echo "failed to interact with the cluster API"
  exit 1
fi

PREV_VER_BRANCH="${PREV_VER_BRANCH:-upstream/main}"
PREV_VER_TAG="${PREV_VER_TAG:-main}"
PREV_VER_REGISTRY="${PREV_VER_REGISTRY:-registry:5000/kubevirt}"

TARGET_VER_BRANCH="${TARGET_VER_BRANCH:-pod-network-name-scheme}"
TARGET_VER_TAG="${TARGET_VER_TAG:-latest}"
TARGET_VER_REGISTRY="${TARGET_VER_REGISTRY:-registry:5000/kubevirt}"

readonly PREV_KV_MANIFESTS="./prev-kv-manifests"
readonly GIT_STASH_ENTRY="iterate"

patch_kubevirt_cr() {
    local -r path=$1
    local -r tag=$2
    local -r registry=$3

    # set Kubevirt workload update strategy to LiveMigrate, VMs will migrate as soon as an upgrade completes
    # sed -i "s?workloadUpdateStrategy: {}?workloadUpdateStrategy:\n    workloadUpdateMethods:\n    - LiveMigrate?g" ${path}
    # specify previous version explicitly to enable upgrading Kubevirt vi the CR.
    sed -i "s?spec:?spec:\n  imageTag: ${tag}\n  imageRegistry: ${registry}?g" ${path}
    # ensure the latest images digest is used by setting Kbevirt deployments imagePullPolicy to always
    sed -i "s?imagePullPolicy: IfNotPresent?imagePullPolicy: Always?g" ${path}
    # raise Kubevirt components log verbosity
    sed -i "s?developerConfiguration:?developerConfiguration:\n      logVerbosity:\n        virtHandler: 4\n        virtLauncher: 4\n        virtController: 4?g" ${path}
    # enable hotplug NICs feature
    sed -i "s?featureGates: \[\]?featureGates: \[\"HotplugNICs\",\"Macvtap\"\]?g" ${path}
}

if ${BUILD_PREVIOUS_VERSION_KUBEVIRT}; then
  if [ "$(git status | grep -i -Po "On branch \K.*")" != "${PREV_VER_BRANCH}" ]; then
    # stash current changes
    if git status -s | grep M; then
      git stash save ${GIT_STASH_ENTRY}
    fi
    # switch to Kubevirt previous version branch
    git checkout ${PREV_VER_BRANCH}
  fi

  # build Kubevirt from previous version branch and push to local registry
  DOCKER_TAG=${PREV_VER_TAG} DOCKER_TAG_ALT=${PREV_VER_TAG} make cluster-build manifests

  # keep previous-version manifests from deployment
  mkdir -p ${PREV_KV_MANIFESTS}
  cp -rf "_out"/* "${PREV_KV_MANIFESTS}"
fi

if ${BUILD_TARGET_VERSION_KUBEVIRT}; then
  if [ "$(git status | grep -i -Po "On branch \K.*")" != "${TARGET_VER_BRANCH}" ]; then
    # switch to development branch
    git checkout ${TARGET_VER_BRANCH}
    # pop development branch changes
    if git stash list | grep "stash@{0}" | grep "${GIT_STASH_ENTRY}"; then
      git stash pop stash@{0}
    fi
  fi

  # build Kubevirt from development branch and push to local registry
  DOCKER_TAG=${TARGET_VER_TAG} DOCKER_TAG_ALT=${TARGET_VER_TAG} make cluster-build manifests
fi

if ${DEPLOY_PREVIOUS_VERSION_KUBEVIRT}; then
  # patch Kubevirt CR
  PATCHED_KUBEVIRT_CR="${PREV_KV_MANIFESTS}/manifests/release/kubevirt-cr-patched.yaml"
  cp ${PREV_KV_MANIFESTS}/manifests/release/kubevirt-cr.yaml ${PATCHED_KUBEVIRT_CR}
  patch_kubevirt_cr ${PATCHED_KUBEVIRT_CR} ${PREV_VER_TAG} ${PREV_VER_REGISTRY}

  # deploy Kubevirt operator from development version (if previous version operator is deployed we wont get the latest CRDs)
  kubectl apply -f _out/manifests/release/kubevirt-operator.yaml

  # deploy Kubevirt CR previous version
  kubectl apply -f ${PATCHED_KUBEVIRT_CR}

  # wait for Kubevirt to be ready
  kubectl wait kubevirt -n kubevirt kubevirt --for condition=Available=True --timeout 5m
fi

if ${DEPLOY_TARGET_VERSION_KUBEVIRT}; then
  # patch Kubevirt CR
  PATCHED_KUBEVIRT_CR="./kubevirt-cr-patched.yaml"
  cp _out/manifests/release/kubevirt-cr.yaml ${PATCHED_KUBEVIRT_CR}
  patch_kubevirt_cr ${PATCHED_KUBEVIRT_CR} ${TARGET_VER_TAG} ${TARGET_VER_REGISTRY}

  # deploy Kubevirt operator from development version (if previous version operator is deployed we wont get the latest CRDs)
  kubectl apply -f _out/manifests/release/kubevirt-operator.yaml

  # deploy Kubevirt CR previous version
  kubectl apply -f ${PATCHED_KUBEVIRT_CR}

  # wait for Kubevirt to be ready
  kubectl wait kubevirt -n kubevirt kubevirt --for condition=Available=True --timeout 5m
fi

if ${FUNCTEST}; then
  # run operator test:
  # 1. Create a VM with secondary NIC
  # 2. Upgrade kubevirt to development branch version.
  # 3. Assert that VM been migrated successfully following the upgrade
  # 4. Migrate the VM again
  # 5. Assert that VM been migrated successfully
  args=()
  args+=("-container-prefix=${TARGET_VER_REGISTRY}")
  args+=("-container-tag=${TARGET_VER_TAG}")
  args+=("-utility-container-prefix=${TARGET_VER_REGISTRY}")
  args+=("-utility-container-tag=${TARGET_VER_TAG}")

  args+=("-previous-release-registry=${PREV_VER_REGISTRY}")
  args+=("-previous-release-tag=${PREV_VER_TAG}")
  args+=("-previous-utility-container-registry=${PREV_VER_REGISTRY}")
  args+=("-previous-utility-container-tag=${PREV_VER_TAG}")

  args+=("-skip-shasums-check=true")

  args+=("-deploy-testing-infra=false")
  args+=("-apply-default-e2e-configuration=false")

  args+=("-ginkgo.label-filter=PostUpgrade")

  args+=("--ginkgo.v --ginkgo.progress")
  args="${args[*]}"

  KUBEVIRT_FUNC_TEST_SUITE_ARGS="${args}" make functest
fi