Test Helm and Kustomize app label update

test-helm.sh

#!/usr/bin/env bash

# Test script for Kueue installation and upgrade using Helm
# This script validates that:
# 1. Kueue v0.14.2 has 'app.kubernetes.io/component' label on ONE Service (metrics)
# 2. After upgrading from local repo, ALL Services have 'app.kubernetes.io/component' label

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Configuration
CLUSTER_NAME="kueue-test-helm"
KUEUE_VERSION="0.14.2"  # Helm chart version (without 'v' prefix)
CERT_MANAGER_VERSION="v1.16.2"
KUEUE_NAMESPACE="kueue-system"
HELM_RELEASE_NAME="kueue"

# Helper functions
log_info() {
    echo -e "${GREEN}[INFO]${NC} $*"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $*"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $*"
}

cleanup() {
    if [[ "${CLEANUP:-true}" == "true" ]]; then
        log_info "Cleaning up Kind cluster..."
        kind delete cluster --name "${CLUSTER_NAME}" 2>/dev/null || true
    else
        log_warn "Skipping cleanup (CLEANUP=false)"
    fi
}

# Set trap to cleanup on exit
trap cleanup EXIT

# Check if helm is installed
if ! command -v helm &> /dev/null; then
    log_error "helm is not installed. Please install helm first."
    exit 1
fi

# Step 1: Create Kind cluster
log_info "Creating Kind cluster: ${CLUSTER_NAME}"
if kind get clusters | grep -q "^${CLUSTER_NAME}$"; then
    log_warn "Cluster ${CLUSTER_NAME} already exists, deleting it first..."
    kind delete cluster --name "${CLUSTER_NAME}"
fi
kind create cluster --name "${CLUSTER_NAME}"

# Step 2: Install cert-manager
log_info "Installing cert-manager ${CERT_MANAGER_VERSION}..."
kubectl apply -f "https://github.com/cert-manager/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml"

log_info "Waiting for cert-manager to be ready..."
kubectl wait --for=condition=available --timeout=300s \
    deployment/cert-manager \
    deployment/cert-manager-cainjector \
    deployment/cert-manager-webhook \
    -n cert-manager

# Step 3: Install Kueue v0.14.2 from upstream OCI registry
log_info "Installing Kueue ${KUEUE_VERSION} from OCI registry..."
helm install "${HELM_RELEASE_NAME}" oci://registry.k8s.io/kueue/charts/kueue \
    --version="${KUEUE_VERSION}" \
    --namespace "${KUEUE_NAMESPACE}" \
    --create-namespace \
    --wait \
    --timeout 300s

# Give it a moment to fully stabilize
sleep 5

# Step 4: Verify only ONE Service has 'app.kubernetes.io/component' label (pre-upgrade state)
log_info "Verifying only ONE Service has 'app.kubernetes.io/component' label (baseline check for v${KUEUE_VERSION})..."
SERVICES_WITH_APP_LABEL=$(kubectl get svc -n "${KUEUE_NAMESPACE}" -l app.kubernetes.io/component -o name 2>/dev/null | wc -l)
TOTAL_SERVICES=$(kubectl get svc -n "${KUEUE_NAMESPACE}" -o name 2>/dev/null | wc -l)

log_info "Found ${SERVICES_WITH_APP_LABEL} Service(s) with 'app.kubernetes.io/component' label out of ${TOTAL_SERVICES} total Service(s)"

if [[ ${SERVICES_WITH_APP_LABEL} -ne 1 ]]; then
    log_error "FAILED: Expected exactly 1 Service with 'app.kubernetes.io/component' label in v${KUEUE_VERSION}"
    log_error "Found: ${SERVICES_WITH_APP_LABEL} Services"
    kubectl get svc -n "${KUEUE_NAMESPACE}" -o custom-columns=NAME:.metadata.name,LABELS:.metadata.labels
    exit 1
fi

log_info "SUCCESS: Exactly 1 Service has 'app.kubernetes.io/component' label (expected for v${KUEUE_VERSION})"
kubectl get svc -n "${KUEUE_NAMESPACE}" -o custom-columns=NAME:.metadata.name,LABELS:.metadata.labels

# Step 5: Upgrade Kueue from local repository Helm chart
log_info "Upgrading Kueue from local repository Helm chart..."
helm upgrade "${HELM_RELEASE_NAME}" ./charts/kueue \
    --namespace "${KUEUE_NAMESPACE}" \
    --values ./charts/kueue/values.yaml \
    --wait \
    --timeout 300s

# Give the upgrade time to settle
sleep 5

# Step 6: Post-upgrade verification - ALL Services MUST have 'app.kubernetes.io/component' label
log_info "Verifying ALL Services have 'app.kubernetes.io/component' label (post-upgrade check)..."
SERVICES_WITH_APP_LABEL=$(kubectl get svc -n "${KUEUE_NAMESPACE}" -l app.kubernetes.io/component -o name 2>/dev/null | wc -l)
TOTAL_SERVICES=$(kubectl get svc -n "${KUEUE_NAMESPACE}" -o name 2>/dev/null | wc -l)

log_info "Found ${SERVICES_WITH_APP_LABEL} Service(s) with 'app.kubernetes.io/component' label out of ${TOTAL_SERVICES} total Service(s)"

if [[ ${SERVICES_WITH_APP_LABEL} -eq 0 ]]; then
    log_error "FAILED: No Services have 'app.kubernetes.io/component' label after upgrade"
    log_error "Expected ALL Services to have 'app.kubernetes.io/component' label after upgrading from local repository"
    kubectl get svc -n "${KUEUE_NAMESPACE}" -o custom-columns=NAME:.metadata.name,LABELS:.metadata.labels
    exit 1
fi

if [[ ${SERVICES_WITH_APP_LABEL} -ne ${TOTAL_SERVICES} ]]; then
    log_error "FAILED: Not all Services have 'app.kubernetes.io/component' label after upgrade"
    log_error "Expected: ${TOTAL_SERVICES} Services with 'app.kubernetes.io/component' label"
    log_error "Found: ${SERVICES_WITH_APP_LABEL} Services with 'app.kubernetes.io/component' label"
    log_error "Missing 'app.kubernetes.io/component' label on $((TOTAL_SERVICES - SERVICES_WITH_APP_LABEL)) Service(s)"
    echo ""
    log_error "Current Services and their labels:"
    kubectl get svc -n "${KUEUE_NAMESPACE}" -o custom-columns=NAME:.metadata.name,LABELS:.metadata.labels
    exit 1
fi

log_info "SUCCESS: ALL ${SERVICES_WITH_APP_LABEL}/${TOTAL_SERVICES} Service(s) have 'app.kubernetes.io/component' label after upgrade"
kubectl get svc -n "${KUEUE_NAMESPACE}" -o custom-columns=NAME:.metadata.name,LABELS:.metadata.labels

# Step 7: Basic functionality check
log_info "Verifying Kueue is still functioning correctly..."
if kubectl get crd | grep -q "kueue.x-k8s.io"; then
    log_info "SUCCESS: Kueue CRDs are present"
else
    log_error "FAILED: Kueue CRDs not found"
    exit 1
fi

# Verify Helm release status
log_info "Verifying Helm release status..."
HELM_STATUS=$(helm status "${HELM_RELEASE_NAME}" -n "${KUEUE_NAMESPACE}" -o json | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
if [[ "${HELM_STATUS}" == "deployed" ]]; then
    log_info "SUCCESS: Helm release status is 'deployed'"
else
    log_error "FAILED: Helm release status is '${HELM_STATUS}', expected 'deployed'"
    exit 1
fi

# Final success message
log_info "======================================"
log_info "ALL TESTS PASSED!"
log_info "======================================"
log_info "Summary:"
log_info "  - Kueue v${KUEUE_VERSION} (Helm): 1 Service with 'app.kubernetes.io/component' label ✓"
log_info "  - After upgrade: ALL Services (${TOTAL_SERVICES}/${TOTAL_SERVICES}) have 'app.kubernetes.io/component' label ✓"
log_info "  - Kueue functionality: OK ✓"
log_info "  - Helm release status: deployed ✓"

test-kustomize.sh

#!/usr/bin/env bash

# Test script for Kueue installation and upgrade using Kustomize
# This script validates that:
# 1. Kueue v0.14.2 has 'app.kubernetes.io/component:controller' label on Services
# 2. After upgrading from local repo, Services have new specific component labels

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Configuration
CLUSTER_NAME="kueue-test-kustomize"
KUEUE_VERSION="v0.14.2"
CERT_MANAGER_VERSION="v1.16.2"
KUEUE_NAMESPACE="kueue-system"

# Helper functions
log_info() {
	echo -e "${GREEN}[INFO]${NC} $*"
}

log_warn() {
	echo -e "${YELLOW}[WARN]${NC} $*"
}

log_error() {
	echo -e "${RED}[ERROR]${NC} $*"
}

cleanup() {
	if [[ "${CLEANUP:-true}" == "true" ]]; then
		log_info "Cleaning up Kind cluster..."
		kind delete cluster --name "${CLUSTER_NAME}" 2>/dev/null || true
	else
		log_warn "Skipping cleanup (CLEANUP=false)"
	fi
}

# Set trap to cleanup on exit
trap cleanup EXIT

# Step 1: Create Kind cluster
log_info "Creating Kind cluster: ${CLUSTER_NAME}"
if kind get clusters | grep -q "^${CLUSTER_NAME}$"; then
	log_warn "Cluster ${CLUSTER_NAME} already exists, deleting it first..."
	kind delete cluster --name "${CLUSTER_NAME}"
fi
kind create cluster --name "${CLUSTER_NAME}"

# Step 2: Install cert-manager
log_info "Installing cert-manager ${CERT_MANAGER_VERSION}..."
kubectl apply -f "https://github.com/cert-manager/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml"

log_info "Waiting for cert-manager to be ready..."
kubectl wait --for=condition=available --timeout=300s \
	deployment/cert-manager \
	deployment/cert-manager-cainjector \
	deployment/cert-manager-webhook \
	-n cert-manager

# Step 3: Install Kueue v0.14.2 from upstream
log_info "Installing Kueue ${KUEUE_VERSION} from upstream..."
kubectl apply --server-side -f "https://github.com/kubernetes-sigs/kueue/releases/download/${KUEUE_VERSION}/manifests.yaml"

log_info "Waiting for Kueue to be ready..."
kubectl wait --for=condition=available --timeout=300s \
	deployment/kueue-controller-manager \
	-n "${KUEUE_NAMESPACE}"

# Give it a moment to fully stabilize
sleep 5

# Step 4: Verify Services have 'app.kubernetes.io/component:controller' label (pre-upgrade state)
log_info "Verifying Services have 'app.kubernetes.io/component:controller' label (baseline check for ${KUEUE_VERSION})..."
SERVICES_WITH_CONTROLLER_LABEL=$(kubectl get svc -n "${KUEUE_NAMESPACE}" -l app.kubernetes.io/component=controller -o name 2>/dev/null | wc -l)
TOTAL_SERVICES=$(kubectl get svc -n "${KUEUE_NAMESPACE}" -o name 2>/dev/null | wc -l)

log_info "Found ${SERVICES_WITH_CONTROLLER_LABEL} Service(s) with 'app.kubernetes.io/component:controller' label out of ${TOTAL_SERVICES} total Service(s)"

if [[ ${SERVICES_WITH_CONTROLLER_LABEL} -eq 0 ]]; then
	log_error "FAILED: No Services have 'app.kubernetes.io/component:controller' label in ${KUEUE_VERSION}"
	log_error "Expected ALL Services to have 'app.kubernetes.io/component:controller' label"
	kubectl get svc -n "${KUEUE_NAMESPACE}" -o custom-columns=NAME:.metadata.name,LABELS:.metadata.labels
	exit 1
fi

if [[ ${SERVICES_WITH_CONTROLLER_LABEL} -ne ${TOTAL_SERVICES} ]]; then
	log_error "FAILED: Not all Services have 'app.kubernetes.io/component:controller' label in ${KUEUE_VERSION}"
	log_error "Expected: ${TOTAL_SERVICES} Services with 'app.kubernetes.io/component:controller' label"
	log_error "Found: ${SERVICES_WITH_CONTROLLER_LABEL} Services"
	kubectl get svc -n "${KUEUE_NAMESPACE}" -o custom-columns=NAME:.metadata.name,LABELS:.metadata.labels
	exit 1
fi

log_info "SUCCESS: ALL ${SERVICES_WITH_CONTROLLER_LABEL}/${TOTAL_SERVICES} Services have 'app.kubernetes.io/component:controller' label in ${KUEUE_VERSION}"
kubectl get svc -n "${KUEUE_NAMESPACE}" -o custom-columns=NAME:.metadata.name,LABELS:.metadata.labels

# Step 5: Upgrade Kueue from local repository
log_info "Upgrading Kueue from local repository using 'make deploy'..."
make deploy

log_info "Waiting for upgrade to complete..."
kubectl wait --for=condition=available --timeout=300s \
	deployment/kueue-controller-manager \
	-n "${KUEUE_NAMESPACE}"

# Give the upgrade time to settle
sleep 5

# Step 6: Post-upgrade verification - Services should have NEW specific component labels
log_info "Verifying Services have new specific 'app.kubernetes.io/component' labels (post-upgrade check)..."
TOTAL_SERVICES=$(kubectl get svc -n "${KUEUE_NAMESPACE}" -o name 2>/dev/null | wc -l)

# Check for specific component labels (not 'controller' anymore)
WEBHOOK_SERVICE=$(kubectl get svc -n "${KUEUE_NAMESPACE}" -l app.kubernetes.io/component=webhook-service -o name 2>/dev/null | wc -l)
VISIBILITY_SERVICE=$(kubectl get svc -n "${KUEUE_NAMESPACE}" -l app.kubernetes.io/component=visibility-service -o name 2>/dev/null | wc -l)
METRICS_SERVICE=$(kubectl get svc -n "${KUEUE_NAMESPACE}" -l app.kubernetes.io/component=controller-manager-metrics-service -o name 2>/dev/null | wc -l)

# Check that old 'controller' label is gone
CONTROLLER_LABEL=$(kubectl get svc -n "${KUEUE_NAMESPACE}" -l app.kubernetes.io/component=controller -o name 2>/dev/null | wc -l)

log_info "Service label counts after upgrade:"
log_info "  - webhook-service: ${WEBHOOK_SERVICE}"
log_info "  - visibility-service: ${VISIBILITY_SERVICE}"
log_info "  - controller-manager-metrics-service: ${METRICS_SERVICE}"
log_info "  - controller (old label): ${CONTROLLER_LABEL}"

if [[ ${CONTROLLER_LABEL} -gt 0 ]]; then
	log_error "FAILED: Found ${CONTROLLER_LABEL} Service(s) still using old 'app.kubernetes.io/component:controller' label"
	kubectl get svc -n "${KUEUE_NAMESPACE}" -o custom-columns=NAME:.metadata.name,LABELS:.metadata.labels
	exit 1
fi

if [[ ${WEBHOOK_SERVICE} -ne 1 ]] || [[ ${VISIBILITY_SERVICE} -ne 1 ]] || [[ ${METRICS_SERVICE} -ne 1 ]]; then
	log_error "FAILED: Not all Services have the correct new component labels"
	log_error "Expected: 1 webhook-service, 1 visibility-service, 1 controller-manager-metrics-service"
	log_error "Found: ${WEBHOOK_SERVICE} webhook-service, ${VISIBILITY_SERVICE} visibility-service, ${METRICS_SERVICE} controller-manager-metrics-service"
	kubectl get svc -n "${KUEUE_NAMESPACE}" -o custom-columns=NAME:.metadata.name,LABELS:.metadata.labels
	exit 1
fi

log_info "SUCCESS: ALL ${TOTAL_SERVICES} Service(s) have new specific component labels (no more 'controller' label)"
kubectl get svc -n "${KUEUE_NAMESPACE}" -o custom-columns=NAME:.metadata.name,LABELS:.metadata.labels

# Step 7: Basic functionality check
log_info "Verifying Kueue is still functioning correctly..."
if kubectl get crd | grep -q "kueue.x-k8s.io"; then
	log_info "SUCCESS: Kueue CRDs are present"
else
	log_error "FAILED: Kueue CRDs not found"
	exit 1
fi

# Final success message
log_info "======================================"
log_info "ALL TESTS PASSED!"
log_info "======================================"
log_info "Summary:"
log_info "  - Kueue ${KUEUE_VERSION}: ALL Services have 'app.kubernetes.io/component:controller' label ✓"
log_info "  - After upgrade: ALL Services have NEW specific component labels (webhook-service, visibility-service, controller-manager-metrics-service) ✓"
log_info "  - Kueue functionality: OK ✓"