csr-e2e-test-all-in-one.sh

    #!/bin/bash

# CSR Testing Script - All CREATE and UPDATE operations with output logging
# This script tests CSR operations on /, /status, and /approval endpoints
# Uses kubectl with proper raw API access for updates
# Also captures validation mismatch metrics after each operation

set -e

# Configuration
OUTPUT_DIR="./csr-test-outputs"
KUBECONFIG="/var/run/kubernetes/admin.kubeconfig"

# CSR name can be provided as first argument, otherwise use default
CSR_NAME="${1:-testuser-csr}"

# Extract username from CSR name (remove -csr suffix if present)
USERNAME="${CSR_NAME%-csr}"

# Create output directory with CSR name
OUTPUT_DIR="${OUTPUT_DIR}/${CSR_NAME}"
mkdir -p "$OUTPUT_DIR"

# Function to log output
log_output() {
    local filename=$1
    local content=$2
    echo "$content" | tee "$OUTPUT_DIR/$filename"
    echo "Output saved to: $OUTPUT_DIR/$filename"
    echo "----------------------------------------"
}

# Function to capture mismatch metrics
capture_metrics() {
    local test_name=$1
    local metrics_filename="${test_name}-metrics.txt"
    
    echo "Capturing validation mismatch metrics..."
    kubectl --kubeconfig=$KUBECONFIG get --raw /metrics | grep mismatch > "$OUTPUT_DIR/$metrics_filename" 2>/dev/null || echo "No mismatch metrics found" > "$OUTPUT_DIR/$metrics_filename"
    
    echo "Metrics saved to: $OUTPUT_DIR/$metrics_filename"
    echo "Current mismatch count:"
    grep "apiserver_validation_declarative_validation_mismatch_total" "$OUTPUT_DIR/$metrics_filename" | grep -v "^#" || echo "No mismatch total found"
    echo
}

# Function to execute kubectl raw request and save output
execute_kubectl_raw() {
    local description=$1
    local filename=$2
    local method=$3
    local endpoint=$4
    local data=$5
    
    echo "=== Executing: $description ==="
    echo "Timestamp: $(date)"
    
    # Save data to temporary file for kubectl
    local temp_file=$(mktemp)
    echo "$data" > "$temp_file"
    
    # Execute kubectl raw request
    if [ "$method" == "PUT" ]; then
        # Use replace --raw for PUT requests
        output=$(kubectl --kubeconfig=$KUBECONFIG replace --raw "$endpoint" -f "$temp_file" 2>&1 || true)
    elif [ "$method" == "POST" ]; then
        # Use create --raw for POST requests
        output=$(kubectl --kubeconfig=$KUBECONFIG create --raw "$endpoint" -f "$temp_file" 2>&1 || true)
    else
        # For GET/DELETE requests
        output=$(kubectl --kubeconfig=$KUBECONFIG get --raw "$endpoint" 2>&1 || true)
    fi
    
    # Clean up temp file
    rm -f "$temp_file"
    
    # Pretty print JSON if possible
    if echo "$output" | jq . 2>/dev/null; then
        pretty_output=$(echo "$output" | jq .)
        log_output "$filename" "$pretty_output"
    else
        log_output "$filename" "$output"
    fi
    
    # Capture metrics after each operation
    sleep 1  # Brief pause to ensure metrics are updated
    capture_metrics "${filename%.json}"
}

echo "Starting CSR test script..."
echo "CSR Name: $CSR_NAME"
echo "Username: $USERNAME"
echo "Output directory: $OUTPUT_DIR"
echo

# Capture initial metrics
echo "=== Capturing initial metrics ==="
capture_metrics "00-initial"

# Check if CSR already exists
if kubectl --kubeconfig=$KUBECONFIG get csr $CSR_NAME &>/dev/null; then
    echo "WARNING: CSR '$CSR_NAME' already exists. Delete it? (y/n)"
    read -r response
    if [[ "$response" == "y" ]]; then
        kubectl --kubeconfig=$KUBECONFIG delete csr $CSR_NAME
        echo "Existing CSR deleted"
    else
        echo "Exiting - CSR already exists"
        exit 1
    fi
fi

# Generate private key and CSR
echo "=== Generating private key and CSR ==="
USER_KEY_FILE="${OUTPUT_DIR}/${USERNAME}.key"
USER_CSR_FILE="${OUTPUT_DIR}/${USERNAME}.csr"

openssl genrsa -out "$USER_KEY_FILE" 2048
openssl req -new -key "$USER_KEY_FILE" -out "$USER_CSR_FILE" -subj "/CN=${USERNAME}/O=testgroup"

# Base64 encode the CSR
CSR_BASE64=$(cat "$USER_CSR_FILE" | base64 | tr -d '\n')
echo "CSR generated and base64 encoded"
echo "Key saved to: $USER_KEY_FILE"
echo "CSR saved to: $USER_CSR_FILE"
echo

# 1. CREATE CSR Request (/) using kubectl
echo "=== Executing: CREATE CSR Request (/) ==="
echo "Timestamp: $(date)"

# Create CSR using kubectl with manifest
cat > "$OUTPUT_DIR/csr-manifest.yaml" <<EOF
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
  name: $CSR_NAME
spec:
  request: $CSR_BASE64
  signerName: kubernetes.io/kube-apiserver-client
  usages:
  - client auth
EOF

output=$(kubectl --kubeconfig=$KUBECONFIG create -f "$OUTPUT_DIR/csr-manifest.yaml" -o json 2>&1 || true)
if echo "$output" | jq . 2>/dev/null; then
    pretty_output=$(echo "$output" | jq .)
    log_output "01-create-csr.json" "$pretty_output"
else
    log_output "01-create-csr.json" "$output"
fi
capture_metrics "01-create-csr"

# Verify creation and save status
echo "=== Verifying CSR creation ==="
kubectl --kubeconfig=$KUBECONFIG get csr $CSR_NAME -o json 2>&1 | jq . > "$OUTPUT_DIR/02-csr-created-status.json" || echo "{}" > "$OUTPUT_DIR/02-csr-created-status.json"
echo "CSR status saved to: $OUTPUT_DIR/02-csr-created-status.json"
capture_metrics "02-csr-created-status"
echo

# 2. Valid UPDATE on /status (Approved)
RESOURCE_VERSION=$(kubectl --kubeconfig=$KUBECONFIG get csr $CSR_NAME -o jsonpath='{.metadata.resourceVersion}' 2>/dev/null || echo "")
echo "Current resource version: $RESOURCE_VERSION"

if [ -n "$RESOURCE_VERSION" ]; then
    STATUS_APPROVED_DATA='{
      "apiVersion": "certificates.k8s.io/v1",
      "kind": "CertificateSigningRequest",
      "metadata": {
        "name": "'$CSR_NAME'",
        "resourceVersion": "'$RESOURCE_VERSION'"
      },
      "status": {
        "conditions": [{
          "type": "Approved",
          "status": "True",
          "reason": "ApprovedByAdmin",
          "message": "This CSR was approved by admin",
          "lastUpdateTime": "'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"
        }]
      }
    }'
    
    execute_kubectl_raw \
        "Valid UPDATE on /status (Approved)" \
        "03-update-status-approved-valid.json" \
        "PUT" \
        "/apis/certificates.k8s.io/v1/certificatesigningrequests/$CSR_NAME/status" \
        "$STATUS_APPROVED_DATA"
else
    echo "Skipping status update - CSR not found"
    echo "{\"error\": \"CSR not found\"}" > "$OUTPUT_DIR/03-update-status-approved-valid.json"
fi

# 3. Invalid UPDATE on /status (Approved + Denied)
RESOURCE_VERSION=$(kubectl --kubeconfig=$KUBECONFIG get csr $CSR_NAME -o jsonpath='{.metadata.resourceVersion}' 2>/dev/null || echo "")
echo "Updated resource version: $RESOURCE_VERSION"

if [ -n "$RESOURCE_VERSION" ]; then
    STATUS_INVALID_DATA='{
      "apiVersion": "certificates.k8s.io/v1",
      "kind": "CertificateSigningRequest",
      "metadata": {
        "name": "'$CSR_NAME'",
        "resourceVersion": "'$RESOURCE_VERSION'"
      },
      "status": {
        "conditions": [{
          "type": "Approved",
          "status": "True",
          "reason": "ApprovedByAdmin",
          "message": "This CSR was approved by admin",
          "lastUpdateTime": "'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"
        },
        {
          "type": "Denied",
          "status": "True",
          "reason": "DeniedByAdmin",
          "message": "This CSR was denied by admin",
          "lastUpdateTime": "'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"
        }]
      }
    }'
    
    execute_kubectl_raw \
        "Invalid UPDATE on /status (Approved + Denied)" \
        "04-update-status-approved-denied-invalid.json" \
        "PUT" \
        "/apis/certificates.k8s.io/v1/certificatesigningrequests/$CSR_NAME/status" \
        "$STATUS_INVALID_DATA"
else
    echo "Skipping status update - CSR not found"
    echo "{\"error\": \"CSR not found\"}" > "$OUTPUT_DIR/04-update-status-approved-denied-invalid.json"
fi

# Reset CSR for /approval tests
echo "=== Resetting CSR for /approval tests ==="
kubectl --kubeconfig=$KUBECONFIG delete csr $CSR_NAME 2>/dev/null || echo "CSR already deleted"
sleep 2

# Recreate CSR
echo "=== Executing: Recreate CSR for /approval tests ==="
echo "Timestamp: $(date)"
output=$(kubectl --kubeconfig=$KUBECONFIG create -f "$OUTPUT_DIR/csr-manifest.yaml" -o json 2>&1 || true)
if echo "$output" | jq . 2>/dev/null; then
    pretty_output=$(echo "$output" | jq .)
    log_output "05-recreate-csr.json" "$pretty_output"
else
    log_output "05-recreate-csr.json" "$output"
fi
capture_metrics "05-recreate-csr"

# 4. Valid UPDATE on /approval (Approved)
CSR_UID=$(kubectl --kubeconfig=$KUBECONFIG get csr $CSR_NAME -o jsonpath='{.metadata.uid}' 2>/dev/null || echo "")
RESOURCE_VERSION=$(kubectl --kubeconfig=$KUBECONFIG get csr $CSR_NAME -o jsonpath='{.metadata.resourceVersion}' 2>/dev/null || echo "")
echo "CSR UID: $CSR_UID"
echo "Resource Version: $RESOURCE_VERSION"

if [ -n "$CSR_UID" ] && [ -n "$RESOURCE_VERSION" ]; then
    APPROVAL_APPROVED_DATA='{
      "apiVersion": "certificates.k8s.io/v1",
      "kind": "CertificateSigningRequest",
      "metadata": {
        "name": "'$CSR_NAME'",
        "uid": "'$CSR_UID'",
        "resourceVersion": "'$RESOURCE_VERSION'"
      },
      "status": {
        "conditions": [{
          "type": "Approved",
          "status": "True",
          "reason": "ApprovedByAdmin",
          "message": "This CSR was approved by admin",
          "lastUpdateTime": "'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"
        }]
      }
    }'
    
    execute_kubectl_raw \
        "Valid UPDATE on /approval (Approved)" \
        "06-update-approval-approved-valid.json" \
        "PUT" \
        "/apis/certificates.k8s.io/v1/certificatesigningrequests/$CSR_NAME/approval" \
        "$APPROVAL_APPROVED_DATA"
else
    echo "Skipping approval update - CSR not found"
    echo "{\"error\": \"CSR not found\"}" > "$OUTPUT_DIR/06-update-approval-approved-valid.json"
fi

# 5. Invalid UPDATE on /approval (Approved + Denied)
CSR_UID=$(kubectl --kubeconfig=$KUBECONFIG get csr $CSR_NAME -o jsonpath='{.metadata.uid}' 2>/dev/null || echo "")
RESOURCE_VERSION=$(kubectl --kubeconfig=$KUBECONFIG get csr $CSR_NAME -o jsonpath='{.metadata.resourceVersion}' 2>/dev/null || echo "")
echo "Updated CSR UID: $CSR_UID"
echo "Updated Resource Version: $RESOURCE_VERSION"

if [ -n "$CSR_UID" ] && [ -n "$RESOURCE_VERSION" ]; then
    APPROVAL_INVALID_DATA='{
      "apiVersion": "certificates.k8s.io/v1",
      "kind": "CertificateSigningRequest",
      "metadata": {
        "name": "'$CSR_NAME'",
        "uid": "'$CSR_UID'",
        "resourceVersion": "'$RESOURCE_VERSION'"
      },
      "status": {
        "conditions": [{
          "type": "Approved",
          "status": "True",
          "reason": "ApprovedByAdmin",
          "message": "This CSR was approved by admin",
          "lastUpdateTime": "'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"
        },
        {
          "type": "Denied",
          "status": "True",
          "reason": "DeniedByAdmin",
          "message": "This CSR was denied by admin",
          "lastUpdateTime": "'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"
        }]
      }
    }'
    
    execute_kubectl_raw \
        "Invalid UPDATE on /approval (Approved + Denied)" \
        "07-update-approval-approved-denied-invalid.json" \
        "PUT" \
        "/apis/certificates.k8s.io/v1/certificatesigningrequests/$CSR_NAME/approval" \
        "$APPROVAL_INVALID_DATA"
else
    echo "Skipping approval update - CSR not found"
    echo "{\"error\": \"CSR not found\"}" > "$OUTPUT_DIR/07-update-approval-approved-denied-invalid.json"
fi

# Final CSR status
echo "=== Final CSR Status ==="
kubectl --kubeconfig=$KUBECONFIG get csr $CSR_NAME -o json 2>&1 | jq . > "$OUTPUT_DIR/08-final-csr-status.json" || echo "{}" > "$OUTPUT_DIR/08-final-csr-status.json"
echo "Final CSR status saved to: $OUTPUT_DIR/08-final-csr-status.json"
capture_metrics "08-final-csr-status"

# Generate metrics summary
echo "=== Generating Metrics Summary ==="
{
    echo "Validation Mismatch Metrics Summary"
    echo "==================================="
    echo
    for metrics_file in "$OUTPUT_DIR"/*-metrics.txt; do
        if [[ -f "$metrics_file" ]]; then
            basename=$(basename "$metrics_file" -metrics.txt)
            echo "After step: $basename"
            grep "apiserver_validation_declarative_validation_mismatch_total" "$metrics_file" | grep -v "^#" || echo "No mismatch total found"
            echo
        fi
    done
} > "$OUTPUT_DIR/metrics-summary.txt"

cat "$OUTPUT_DIR/metrics-summary.txt"

# Cleanup
echo
echo "=== Cleanup ==="

# Remove temporary files
rm -f "$OUTPUT_DIR/csr-manifest.yaml"

# Ask before deleting CSR
echo "Delete CSR '$CSR_NAME'? (y/n)"
read -r response
if [[ "$response" == "y" ]]; then
    kubectl --kubeconfig=$KUBECONFIG delete csr $CSR_NAME 2>/dev/null || echo "CSR already deleted"
    echo "CSR deleted"
else
    echo "CSR kept - you can delete it later with:"
    echo "kubectl --kubeconfig=$KUBECONFIG delete csr $CSR_NAME"
fi

# Summary
echo
echo "=== Test Complete ==="
echo "CSR Name: $CSR_NAME"
echo "All outputs saved in: $OUTPUT_DIR"
echo
echo "Output files:"
ls -la "$OUTPUT_DIR"/*.json 2>/dev/null || echo "No JSON files found"
echo
echo "Metrics files:"
ls -la "$OUTPUT_DIR"/*-metrics.txt 2>/dev/null || echo "No metrics files found"
echo
echo "Key and CSR files:"
ls -la "$OUTPUT_DIR"/*.key "$OUTPUT_DIR"/*.csr 2>/dev/null || echo "No key/csr files found"
echo
echo "To view any output, use: cat $OUTPUT_DIR/<filename>"
echo "To pretty-print JSON outputs, use: jq . $OUTPUT_DIR/<filename>"
echo "To see metrics summary: cat $OUTPUT_DIR/metrics-summary.txt"