Foremanctl Backup Restore Script - Complete with all fixes from May 2026 testing

test-restore-final.sh

#!/bin/bash
#
# test-restore-final.sh - Same-box restore testing for foremanctl backup
#
# Usage: ./test-restore-final.sh /path/to/backup/foreman-backup-TIMESTAMP
#
# This script performs a DESTRUCTIVE restore on the same box:
# - Drops and recreates all databases
# - Restores database dumps
# - Fixes database ownership and permissions
# - Transfers table/sequence ownership (required for migrations)
# - Restores Pulp content and encryption keys
# - Runs foremanctl deploy to complete migrations
#
# WARNING: This will OVERWRITE current data!
#
# FIXES APPLIED (from testing on May 12-13, 2026):
# 1. Secrets restored BEFORE PostgreSQL starts (prevents connection failures)
# 2. Database ownership and permissions fixed after pg_restore
# 3. Table and sequence ownership transferred (required for migrations)
# 4. JQ syntax corrected for secrets parsing (.[] | @json)
# 5. Post-restore migrations via foremanctl deploy (fixes schema mismatches)
#

set -e  # Exit on error

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Helper functions
log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Check arguments
if [ $# -ne 1 ]; then
    echo "Usage: $0 /path/to/backup/foreman-backup-TIMESTAMP"
    exit 1
fi

BACKUP_PATH="$1"

if [ ! -d "$BACKUP_PATH" ]; then
    log_error "Backup directory does not exist: $BACKUP_PATH"
    exit 1
fi

log_info "Starting restore from: $BACKUP_PATH"

# Verify backup contains required files
log_info "Verifying backup contents..."
REQUIRED_FILES=(
    "metadata.yml"
    "foreman.dump"
    "candlepin.dump"
    "pulp.dump"
    "config_files.tar.gz"
)

for file in "${REQUIRED_FILES[@]}"; do
    if [ ! -f "$BACKUP_PATH/$file" ]; then
        log_error "Required file missing: $file"
        exit 1
    fi
done
log_info "✓ Backup verification passed"

# Confirmation prompt
log_warn "════════════════════════════════════════════════════════════════"
log_warn "WARNING: This will DESTROY current data and restore from backup!"
log_warn "Backup path: $BACKUP_PATH"
log_warn "════════════════════════════════════════════════════════════════"
read -p "Are you sure you want to continue? (type 'yes' to confirm): " confirm
if [ "$confirm" != "yes" ]; then
    log_info "Restore cancelled"
    exit 0
fi

# ============================================================================
# PHASE 1: Stop Services
# ============================================================================
log_info "Phase 1: Stopping Foreman services..."
systemctl stop foreman.target
sleep 5
log_info "✓ Services stopped"

# ============================================================================
# PHASE 2: Restore Podman Secrets (BEFORE starting PostgreSQL!)
# ============================================================================
log_info "Phase 2: Restoring podman secrets..."
log_warn "CRITICAL: Secrets MUST be restored before PostgreSQL starts!"

# Extract secrets from backup
TEMP_DIR=$(mktemp -d)
cd "$TEMP_DIR"
tar -xzf "$BACKUP_PATH/config_files.tar.gz" || {
    log_error "Failed to extract config_files.tar.gz"
    exit 1
}

if [ ! -f "secrets.json" ]; then
    log_error "secrets.json not found in config_files.tar.gz"
    exit 1
fi

# Count secrets in backup
backup_secret_count=$(jq 'length' secrets.json)
log_info "Found $backup_secret_count secrets in backup"

# List current secrets
current_secret_count=$(podman secret ls --format '{{.Name}}' | wc -l)
log_info "Currently have $current_secret_count secrets"

# Remove all existing secrets (WARNING: DESTRUCTIVE!)
log_warn "Removing all existing podman secrets..."
podman secret ls --format '{{.ID}}' | while read secret_id; do
    podman secret rm "$secret_id" 2>/dev/null || log_warn "Failed to remove secret: $secret_id"
done

# Recreate secrets from backup
log_info "Recreating secrets from backup..."
secret_count=0
# FIX: Changed from '.[] | .[0] | @json' to '.[] | @json'
jq -r '.[] | @json' secrets.json | while read -r secret_json; do
    name=$(echo "$secret_json" | jq -r '.Spec.Name')
    data=$(echo "$secret_json" | jq -r '.SecretData')

    echo "$data" | podman secret create "$name" - > /dev/null 2>&1 || {
        log_warn "Failed to create secret: $name"
        continue
    }
    secret_count=$((secret_count + 1))
done

# Verify secret count
restored_secret_count=$(podman secret ls --format '{{.Name}}' | wc -l)
log_info "  ✓ Restored $restored_secret_count podman secrets"

if [ "$restored_secret_count" -ne "$backup_secret_count" ]; then
    log_warn "Secret count mismatch! Expected: $backup_secret_count, Got: $restored_secret_count"
fi

# Cleanup temp directory
rm -rf "$TEMP_DIR"

# ============================================================================
# PHASE 3: Start PostgreSQL (now that secrets exist)
# ============================================================================
log_info "Phase 3: Starting PostgreSQL (secrets are now available)..."
systemctl start postgresql.service

log_info "Waiting for PostgreSQL to be ready..."
for i in {1..30}; do
    if systemctl is-active --quiet postgresql.service; then
        log_info "✓ PostgreSQL is ready"
        break
    fi
    echo -n "."
    sleep 2
done
echo ""

# Get PostgreSQL password (now that secrets are restored)
POSTGRES_PASSWORD=$(podman secret inspect postgresql-admin-password --format '{{ .SecretData }}' | base64 -d)

# ============================================================================
# PHASE 4: Restore Databases
# ============================================================================
log_info "Phase 4: Restoring databases..."

# Detect which databases to restore
DATABASES=()
for dump in "$BACKUP_PATH"/*.dump; do
    [ -f "$dump" ] || continue
    dbname=$(basename "$dump" .dump)
    # Handle IOP database naming (iop_advisor.dump -> advisor_db)
    if [[ "$dbname" == iop_* ]]; then
        dbname="${dbname#iop_}_db"
    fi
    DATABASES+=("$dbname")
done

log_info "Found ${#DATABASES[@]} databases to restore: ${DATABASES[*]}"

# Drop and recreate each database
for dbname in "${DATABASES[@]}"; do
    log_info "Dropping and recreating database: $dbname"
    podman exec -e PGPASSWORD="$POSTGRES_PASSWORD" postgresql \
        psql -U postgres -c "DROP DATABASE IF EXISTS \"$dbname\";" 2>&1 | grep -v "NOTICE" || true

    podman exec -e PGPASSWORD="$POSTGRES_PASSWORD" postgresql \
        psql -U postgres -c "CREATE DATABASE \"$dbname\" OWNER postgres;" || {
        log_error "Failed to create database: $dbname"
        exit 1
    }
done

# Restore each database dump
for dbname in "${DATABASES[@]}"; do
    # Find the dump file (handle IOP naming)
    dump_file=""
    if [[ "$dbname" == *_db ]]; then
        # Try iop_ prefix version
        iop_name="iop_${dbname%_db}"
        if [ -f "$BACKUP_PATH/${iop_name}.dump" ]; then
            dump_file="$BACKUP_PATH/${iop_name}.dump"
        fi
    fi
    if [ -z "$dump_file" ]; then
        dump_file="$BACKUP_PATH/${dbname}.dump"
    fi

    if [ ! -f "$dump_file" ]; then
        log_warn "Dump file not found for database: $dbname (skipping)"
        continue
    fi

    log_info "Restoring database: $dbname from $(basename $dump_file)"

    podman exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" postgresql \
        pg_restore -U postgres -d "$dbname" -v --no-owner --no-acl < "$dump_file" \
        > "/tmp/restore-${dbname}.log" 2>&1 || {
        log_warn "pg_restore reported errors (this may be normal - check /tmp/restore-${dbname}.log)"
    }

    # Verify restore
    table_count=$(podman exec -e PGPASSWORD="$POSTGRES_PASSWORD" postgresql \
        psql -U postgres -d "$dbname" -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public';" | tr -d ' ')
    log_info "  ✓ Database $dbname restored ($table_count tables)"
done

log_info "✓ All databases restored"

# ============================================================================
# PHASE 5: Fix Database Ownership and Permissions
# ============================================================================
log_info "Phase 5: Fixing database ownership and permissions..."
log_warn "This is required because pg_restore used --no-owner --no-acl flags"

# Define database ownership mappings
declare -A DB_OWNERS=(
    ["foreman"]="foreman"
    ["candlepin"]="candlepin"
    ["pulp"]="pulp"
    ["advisor_db"]="advisor_user"
    ["inventory_db"]="inventory_admin"
    ["remediations_db"]="remediations_user"
    ["vmaas_db"]="vmaas_admin"
    ["vulnerability_db"]="vulnerability_admin"
)

# Fix ownership for each database
for dbname in "${!DB_OWNERS[@]}"; do
    owner="${DB_OWNERS[$dbname]}"

    # Check if database exists
    if ! podman exec -e PGPASSWORD="$POSTGRES_PASSWORD" postgresql \
        psql -U postgres -lqt | cut -d \| -f 1 | grep -qw "$dbname"; then
        continue
    fi

    log_info "Setting ownership for $dbname to $owner..."

    # Change database owner
    podman exec -e PGPASSWORD="$POSTGRES_PASSWORD" postgresql \
        psql -U postgres -c "ALTER DATABASE \"$dbname\" OWNER TO \"$owner\";" 2>&1 | grep -v "NOTICE" || true

    # Grant privileges on all tables and sequences
    podman exec -e PGPASSWORD="$POSTGRES_PASSWORD" postgresql \
        psql -U postgres -d "$dbname" -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO \"$owner\";" 2>&1 | grep -v "NOTICE" || true

    podman exec -e PGPASSWORD="$POSTGRES_PASSWORD" postgresql \
        psql -U postgres -d "$dbname" -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO \"$owner\";" 2>&1 | grep -v "NOTICE" || true
done

log_info "✓ Database ownership and permissions fixed"

# ============================================================================
# PHASE 6: Transfer Table and Sequence Ownership (Required for Migrations)
# ============================================================================
log_info "Phase 6: Transferring table and sequence ownership..."
log_warn "This is CRITICAL for migrations to work properly"

# Only transfer ownership for databases that need migrations
for dbname in foreman candlepin pulp; do
    owner="${DB_OWNERS[$dbname]}"

    # Check if database exists
    if ! podman exec -e PGPASSWORD="$POSTGRES_PASSWORD" postgresql \
        psql -U postgres -lqt | cut -d \| -f 1 | grep -qw "$dbname"; then
        continue
    fi

    log_info "Transferring table ownership in $dbname to $owner..."

    # Transfer table ownership
    podman exec -e PGPASSWORD="$POSTGRES_PASSWORD" postgresql psql -U postgres -d "$dbname" -t -c "
    SELECT 'ALTER TABLE ' || schemaname || '.' || tablename || ' OWNER TO $owner;'
    FROM pg_tables WHERE schemaname = 'public';" 2>/dev/null | grep "ALTER TABLE" | \
    podman exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" postgresql psql -U postgres -d "$dbname" > /dev/null 2>&1 || true

    # Transfer sequence ownership
    podman exec -e PGPASSWORD="$POSTGRES_PASSWORD" postgresql psql -U postgres -d "$dbname" -t -c "
    SELECT 'ALTER SEQUENCE ' || schemaname || '.' || sequencename || ' OWNER TO $owner;'
    FROM pg_sequences WHERE schemaname = 'public';" 2>/dev/null | grep "ALTER SEQUENCE" | \
    podman exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" postgresql psql -U postgres -d "$dbname" > /dev/null 2>&1 || true

    log_info "  ✓ Ownership transferred for $dbname"
done

log_info "✓ Table and sequence ownership transferred"

# ============================================================================
# PHASE 7: Restore Pulp Content and Encryption Keys
# ============================================================================
log_info "Phase 7: Restoring Pulp content and encryption keys..."

if [ -f "$BACKUP_PATH/pulp-content.tar.gz" ]; then
    log_info "Backing up current Pulp media directory..."
    if [ -d /var/lib/pulp/media ]; then
        mv /var/lib/pulp/media /var/lib/pulp/media.backup-$(date +%s)
    fi

    log_info "Extracting Pulp content to /var/lib/pulp..."
    cd /var/lib/pulp
    tar -xzf "$BACKUP_PATH/pulp-content.tar.gz" || {
        log_error "Failed to extract Pulp content"
        exit 1
    }

    # Verify encryption keys were restored
    if [ -f /var/lib/pulp/database_fields.symmetric.key ]; then
        log_info "  ✓ Pulp encryption key restored"
    else
        log_error "Pulp encryption key NOT found after restore!"
        exit 1
    fi

    if [ -f /var/lib/pulp/django_secret_key ]; then
        log_info "  ✓ Django secret key restored"
    else
        log_error "Django secret key NOT found after restore!"
        exit 1
    fi

    # Count restored files
    file_count=$(find /var/lib/pulp/media -type f 2>/dev/null | wc -l)
    log_info "  ✓ Pulp content restored ($file_count files in media directory)"
else
    log_warn "No pulp-content.tar.gz found in backup (may have been skipped)"
fi

# ============================================================================
# PHASE 8: Restore Foremanctl State
# ============================================================================
log_info "Phase 8: Restoring foremanctl state..."

if [ -f "$BACKUP_PATH/foremanctl-state.tar.gz" ]; then
    log_info "Backing up current foremanctl state..."
    if [ -d /var/lib/foremanctl ]; then
        # Backup current state (excluding certs)
        tar -czf /tmp/foremanctl-state-backup-$(date +%s).tar.gz \
            --exclude=/var/lib/foremanctl/certs \
            /var/lib/foremanctl 2>/dev/null || true
    fi

    log_info "Extracting foremanctl state to /var/lib/foremanctl..."
    cd /var/lib
    tar -xzf "$BACKUP_PATH/foremanctl-state.tar.gz" || {
        log_error "Failed to extract foremanctl state"
        exit 1
    }

    log_info "  ✓ Foremanctl state restored"
else
    log_warn "No foremanctl-state.tar.gz found in backup"
fi

# ============================================================================
# PHASE 9: Run Migrations via foremanctl deploy
# ============================================================================
log_info "Phase 9: Running migrations via foremanctl deploy..."
log_warn "This is CRITICAL to fix schema mismatches and complete the restore"

# Check if foremanctl is available
if [ ! -f "/root/foremanctl/foremanctl" ]; then
    log_error "foremanctl not found at /root/foremanctl/foremanctl"
    log_warn "You MUST run 'foremanctl deploy' manually to complete migrations!"
    SKIP_MIGRATIONS=true
else
    SKIP_MIGRATIONS=false
fi

if [ "$SKIP_MIGRATIONS" = false ]; then
    log_info "Running: cd /root/foremanctl && source .venv/bin/activate && ./foremanctl deploy"

    cd /root/foremanctl
    if [ -f .venv/bin/activate ]; then
        source .venv/bin/activate
    fi
    export OBSAH_STATE=/var/lib/foremanctl

    ./foremanctl deploy 2>&1 | tee /tmp/post-restore-deploy.log

    if [ ${PIPESTATUS[0]} -eq 0 ]; then
        log_info "  ✓ Migrations completed successfully"
    else
        log_error "  ✗ Migrations failed! Check /tmp/post-restore-deploy.log"
        log_warn "Services may not start correctly until migrations complete"
    fi
else
    log_warn "Skipping migrations - run manually: cd /root/foremanctl && ./foremanctl deploy"
fi

# ============================================================================
# PHASE 10: Verification
# ============================================================================
log_info "Phase 10: Verifying restore..."

log_info "Waiting for services to stabilize (60 seconds)..."
sleep 60

# Check service status
log_info "Checking service status..."
if systemctl is-active --quiet foreman.target; then
    log_info "  ✓ foreman.target is active"
else
    log_warn "  ✗ foreman.target is NOT active"
fi

# Check containers
log_info "Checking running containers..."
container_count=$(podman ps --format '{{.Names}}' | wc -l)
log_info "  Running containers: $container_count"

# Check key services
for service in postgresql foreman pulp-api candlepin; do
    if podman ps --format '{{.Names}}' | grep -q "^${service}$"; then
        log_info "  ✓ $service container running"
    else
        log_warn "  ✗ $service container NOT running"
    fi
done

# Test database connectivity
log_info "Testing database connectivity..."
if timeout 10 podman exec foreman rails runner 'puts "Hosts: #{Host.count}"' 2>/dev/null | grep -q "Hosts:"; then
    log_info "  ✓ Foreman database connection working"
else
    log_warn "  ✗ Foreman database connection failed (may need more time to start)"
fi

if timeout 10 podman exec pulp-api pulpcore-manager shell -c "from pulpcore.app.models import Repository; print(f'Repos: {Repository.objects.count()}')" 2>/dev/null | grep -q "Repos:"; then
    log_info "  ✓ Pulp database connection working"
else
    log_warn "  ✗ Pulp database connection failed (may need more time to start)"
fi

# ============================================================================
# COMPLETION
# ============================================================================
echo ""
log_info "════════════════════════════════════════════════════════════════"
log_info "Restore completed!"
log_info "════════════════════════════════════════════════════════════════"
log_info "Next steps:"
log_info "1. Access Foreman UI: https://$(hostname)"
log_info "2. Login with credentials from: /var/lib/foremanctl/foreman-admin-init-passwd"
log_info "3. Verify your data is restored to backup state"
log_info "4. Check for any errors: journalctl -u foreman.target --since '10 minutes ago'"
log_info ""
log_info "Database restore logs: /tmp/restore-*.log"
log_info "Migration logs: /tmp/post-restore-deploy.log"
log_info "════════════════════════════════════════════════════════════════"
log_info ""
log_info "FIXES APPLIED IN THIS VERSION:"
log_info "1. ✓ Secrets restored BEFORE PostgreSQL starts (Phase 2)"
log_info "2. ✓ Database ownership and permissions fixed (Phase 5)"
log_info "3. ✓ Table/sequence ownership transferred for migrations (Phase 6)"
log_info "4. ✓ JQ syntax corrected (.[] | @json instead of .[] | .[0] | @json)"
log_info "5. ✓ Migrations run via foremanctl deploy (Phase 9)"
log_info "════════════════════════════════════════════════════════════════"