CompleteFailoverCycle.ps1

CompleteFailoverCycle.ps1

# completeFailoverCycle.ps1
# This script implements a complete failover cycle by:
# 1. Performing initial failover (A -> B)
# 2. Restarting the original server
# 3. Performing second failover (B -> A)
# All while using the existing dnsFailover_v2.ps1 script

[CmdletBinding()]
param(
    [Parameter(Mandatory = $false)]
    [ValidateSet("Dev", "Prod")]
    [string]$Env = "Dev",
    
    [Parameter(Mandatory = $false)]
    [string]$dnsServer,
    
    [Parameter(Mandatory = $false)]
    [string]$lookupZone,
    
    [Parameter(Mandatory = $false)]
    [string]$dnsFailoverScriptPath = "$PSScriptRoot\dnsFailover_v2.ps1",
    
    [Parameter(Mandatory = $false)]
    [string]$logFilePath = "$PSScriptRoot\completeFailoverCycle.log",
    
    [Parameter(Mandatory = $false)]
    [int]$serverRestartTimeout = 600,
    
    [Parameter(Mandatory = $false)]
    [int]$serverAvailabilityCheckInterval = 15,
    
    [Parameter(Mandatory = $false)]
    [int]$maxFailoverWaitTime = 1800,
    
    [Parameter(Mandatory = $false)]
    [switch]$SkipServerRestart,
    
    [Parameter(Mandatory = $false)]
    [switch]$SkipSecondFailover,
    
    [Parameter(Mandatory = $false)]
    [switch]$Help,
    
    [Parameter(Mandatory = $false)]
    [bool]$IgnoreFailoverExitCodes = $true
)

# Function to show help information
function Show-Help {
    $helpText = @"
Complete Failover Cycle Script Help
==================================

Description:
    This script implements a complete failover cycle by:
    1. Performing initial failover (A -> B)
    2. Restarting the original server
    3. Performing second failover (B -> A)
    All while using the existing dnsFailover_v2.ps1 script

Syntax:
    .\completeFailoverCycle.ps1 [-Env <String>] [-dnsServer <String>] [-lookupZone <String>] 
                              [-dnsFailoverScriptPath <String>] [-logFilePath <String>]
                              [-serverRestartTimeout <Int>] [-serverAvailabilityCheckInterval <Int>]
                              [-maxFailoverWaitTime <Int>] [-SkipServerRestart] [-SkipSecondFailover]
                              [-IgnoreFailoverExitCodes <Bool>] [-Help]

Parameters:
    -Env <String>
        Specifies the environment to operate in.
        Valid values: Dev, Prod
        Default: Prod

    -dnsServer <String>
        Specifies the DNS server to use for operations.
        
    -lookupZone <String>
        Specifies the DNS lookup zone.
        
    -dnsFailoverScriptPath <String>
        Path to the dnsFailover_v2.ps1 script.
        Default: .\dnsFailover_v2.ps1
        
    -logFilePath <String>
        Path to the log file.
        Default: .\CompleteFailoverCycle.log
        
    -serverRestartTimeout <Int>
        Maximum time in seconds to wait for server restart.
        Default: 600 (10 minutes)
        
    -serverAvailabilityCheckInterval <Int>
        Interval in seconds between server availability checks.
        Default: 15
        
    -maxFailoverWaitTime <Int>
        Maximum time in seconds to wait for failover completion.
        Default: 1800 (30 minutes)
        
    -SkipServerRestart [Switch]
        If specified, skips the server restart phase.
        
    -SkipSecondFailover [Switch]
        If specified, skips the second failover (B -> A) phase.
        
    -IgnoreFailoverExitCodes <Bool>
        Whether to ignore non-zero exit codes from the failover script and
        determine success based on the output content.
        Default: True

    -Help [Switch]
        Shows this help message.

Examples:
    # Show help
    .\completeFailoverCycle.ps1 -Help

    # Run complete failover cycle in Dev environment
    .\completeFailoverCycle.ps1 -Env Dev

    # Run complete failover cycle in Prod environment with custom DNS settings
    .\completeFailoverCycle.ps1 -Env Prod -dnsServer "dns1.company.com" -lookupZone "company.local"

    # Run only the first failover and server restart, but skip second failover
    .\completeFailoverCycle.ps1 -SkipSecondFailover

    # Run first and second failover without server restart
    .\completeFailoverCycle.ps1 -SkipServerRestart
    
    # Consider exit codes from the failover script (default is to ignore)
    .\completeFailoverCycle.ps1 -IgnoreFailoverExitCodes $false

Notes:
    - This script requires the dnsFailover_v2.ps1 script to be accessible
    - Appropriate permissions are required for DNS, IIS, and remote server operations
    - All operations are logged to the specified log file
"@

    Write-Host $helpText
    exit 0
}

# Show help if requested
if ($Help) {
    Show-Help
}

# Enable strict mode for better error handling
Set-StrictMode -Version Latest

# Set error action preference to stop script execution on error
$ErrorActionPreference = 'Stop'

# Function to write log messages
function Write-Log {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [string]$Message,
        
        [Parameter(Mandatory = $false)]
        [ValidateSet("INFO", "WARNING", "ERROR", "SUCCESS", "DEBUG")]
        [string]$Level = "INFO"
    )
    
    $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
    $logMessage = "[$timestamp] [$Level] $Message"
    
    # Write to console with appropriate color
    switch ($Level) {
        "INFO" { Write-Host $logMessage -ForegroundColor Cyan }
        "WARNING" { Write-Host $logMessage -ForegroundColor Yellow }
        "ERROR" { Write-Host $logMessage -ForegroundColor Red }
        "SUCCESS" { Write-Host $logMessage -ForegroundColor Green }
        "DEBUG" { 
            # Only show debug messages in verbose mode
            if ($VerbosePreference -eq 'Continue') {
                Write-Host $logMessage -ForegroundColor Gray 
            }
        }
    }
    
    # Append to log file
    try {
        # Create the log directory if it doesn't exist
        $logDir = Split-Path -Path $logFilePath -Parent
        if (-not (Test-Path -Path $logDir -PathType Container)) {
            New-Item -Path $logDir -ItemType Directory -Force | Out-Null
        }
        
        Add-Content -Path $logFilePath -Value $logMessage -ErrorAction Stop
    }
    catch {
        Write-Warning "Failed to write to log file: $_"
    }
}

# Function to validate the dnsFailover script exists
function Test-DNSFailoverScript {
    if (-not (Test-Path -Path $dnsFailoverScriptPath)) {
        Write-Log "DNS Failover script not found at: $dnsFailoverScriptPath" -Level "ERROR"
        Write-Log "Please provide the correct path using -dnsFailoverScriptPath parameter" -Level "ERROR"
        return $false
    }
    return $true
}

# Function to extract hostname from FQDN
function Get-HostnameFromFQDN {
    param (
        [Parameter(Mandatory = $true)]
        [string]$FQDN
    )
    
    # Extract hostname part (remove domain if present)
    return $FQDN -replace '\..*$', ''
}

# Function to parse outputs from the dnsFailover_v2.ps1 script
function Parse-ScriptOutput {
    param (
        [Parameter(Mandatory = $true)]
        [string]$Output,
        
        [Parameter(Mandatory = $true)]
        [string]$Pattern,
        
        [Parameter(Mandatory = $false)]
        [string]$DefaultValue = $null
    )
    
    if ($Output -match $Pattern) {
        return $Matches[1]
    }
    
    return $DefaultValue
}

# Function to execute the dns failover script
function Invoke-DNSFailover {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [string]$Operation,
        
        [Parameter(Mandatory = $false)]
        [string]$Description = "DNS Failover"
    )
    
    Write-Log "Starting $Description..." -Level "INFO"
    
    # Create the command line
    $cmdArgs = "-Env `"$Env`" -Ops `"$Operation`""
    
    # Add optional parameters if provided
    if ($dnsServer) {
        $cmdArgs += " -dnsServer `"$dnsServer`""
    }
    
    if ($lookupZone) {
        $cmdArgs += " -lookupZone `"$lookupZone`""
    }
    
    Write-Log "Executing DNS failover script with arguments: $cmdArgs" -Level "INFO"
    
    try {
        # Execute the script using Invoke-Expression
        $tempFile = [System.IO.Path]::GetTempFileName()
        $scriptCmd = "& '$dnsFailoverScriptPath' $cmdArgs *>&1 | Tee-Object -FilePath '$tempFile'"
        
        Write-Log "Running command: $scriptCmd" -Level "DEBUG"
        $result = Invoke-Expression $scriptCmd
        $exitCode = $LASTEXITCODE
        
        # Read the captured output
        $output = Get-Content -Path $tempFile -Raw
        Remove-Item -Path $tempFile -Force
        
        # Log the output for debugging
        Write-Log "Script output: $output" -Level "DEBUG"
        
        # Check output for success indicators regardless of exit code
        $dnsPropagationSuccess = $false
        $iisManagementSuccess = $false
        
        # Check for DNS propagation success patterns in the output
        if ($output -match "DNS failover succeeded" -or 
            $output -match "Failover successful" -or 
            $output -match "New active host:" -or
            $output -match "new host is" -or
            $output -match "DNS failover succeeded, new host is") {
            $dnsPropagationSuccess = $true
            Write-Log "DNS propagation appears successful based on output" -Level "INFO"
        }
        
        # Check for IIS management success using JSON results pattern
        # First check if the JSON results exist in the output
        $jsonResults = $false
        $allAppPoolsSuccessful = $true
        
        if ($output -match '"Results"\s*:\s*\[.*\]') {
            $jsonResults = $true
            Write-Log "Found IIS management JSON results in output" -Level "INFO"
            
            # Extract all AppPoolName, RestartStatus, and RecycleStatus pairs using regex
            $appPoolMatches = [regex]::Matches($output, '"AppPoolName"\s*:\s*"([^"]+)"[^}]+"RestartStatus"\s*:\s*"([^"]+)"[^}]+"RecycleStatus"\s*:\s*"([^"]+)"')
            
            if ($appPoolMatches.Count -gt 0) {
                Write-Log "Found $($appPoolMatches.Count) application pools in the results" -Level "INFO"
                
                foreach ($match in $appPoolMatches) {
                    $appPoolName = $match.Groups[1].Value
                    $restartStatus = $match.Groups[2].Value
                    $recycleStatus = $match.Groups[3].Value
                    
                    Write-Log "App Pool: $appPoolName, RestartStatus: $restartStatus, RecycleStatus: $recycleStatus" -Level "DEBUG"
                    
                    if ($restartStatus -ne "Success" -or $recycleStatus -ne "Success") {
                        $allAppPoolsSuccessful = $false
                        Write-Log "App Pool $appPoolName does not have both restart and recycle success" -Level "WARNING"
                    }
                }
                
                if ($allAppPoolsSuccessful) {
                    $iisManagementSuccess = $true
                    Write-Log "All IIS application pools were successfully restarted and recycled" -Level "SUCCESS"
                }
                else {
                    Write-Log "Not all IIS application pools were successfully restarted and recycled" -Level "WARNING"
                }
            }
            else {
                Write-Log "No application pool restart/recycle status found in the output" -Level "WARNING"
            }
        }
        else {
            # Fallback to text-based pattern matching for IIS success if JSON isn't found
            $appPoolRestartMatches = [regex]::Matches($output, "Application Pool restarted successfully: ([^\r\n]+)")
            $appPoolRecycleMatches = [regex]::Matches($output, "Application Pool recycled successfully: ([^\r\n]+)")
            
            #if ($appPoolRestartMatches.Count -gt 0 && $appPoolRecycleMatches.Count -gt 0) {
            if ($appPoolRestartMatches.Count -gt 0 -and $appPoolRecycleMatches.Count -gt 0) {
                # Create sets of app pool names from both matches
                $restartedPools = @($appPoolRestartMatches | ForEach-Object { $_.Groups[1].Value })
                $recycledPools = @($appPoolRecycleMatches | ForEach-Object { $_.Groups[1].Value })
                
                Write-Log "Found $($restartedPools.Count) restarted pools and $($recycledPools.Count) recycled pools" -Level "DEBUG"
                
                # Check if all restarted pools were also recycled
                $allPoolsHandled = $true
                foreach ($pool in $restartedPools) {
                    if ($recycledPools -notcontains $pool) {
                        $allPoolsHandled = $false
                        Write-Log "App Pool $pool was restarted but not recycled" -Level "WARNING"
                    }
                }
                
                if ($allPoolsHandled) {
                    $iisManagementSuccess = $true
                    Write-Log "All IIS application pools were successfully restarted and recycled (text pattern)" -Level "SUCCESS"
                }
            }
        }
        
        # Determine overall success based on DNS and IIS operations
        $actualSuccess = $dnsPropagationSuccess -and ($iisManagementSuccess -or (-not $jsonResults))
        
        # Always treat as success if IgnoreFailoverExitCodes is true
        if ($IgnoreFailoverExitCodes -eq $true) {
            $actualSuccess = $true
        }
        
        if ($exitCode -ne 0) {
            if ($actualSuccess) {
                Write-Log "$Description reported exit code: $exitCode, but successful operations detected" -Level "WARNING"
                
                return @{
                    Success       = $true
                    Output        = $output
                    ExitCode      = $exitCode
                    ActualSuccess = $true
                    DNSSuccess    = $dnsPropagationSuccess
                    IISSuccess    = $iisManagementSuccess
                }
            }
            else {
                Write-Log "$Description failed with exit code: $exitCode and operations were not successful" -Level "ERROR"
                return @{
                    Success       = $false
                    Output        = $output
                    ExitCode      = $exitCode
                    ActualSuccess = $false
                    DNSSuccess    = $dnsPropagationSuccess
                    IISSuccess    = $iisManagementSuccess
                }
            }
        }
        
        Write-Log "$Description completed successfully" -Level "SUCCESS"
        return @{
            Success       = $true
            Output        = $output
            ExitCode      = $exitCode
            ActualSuccess = $true
            DNSSuccess    = $dnsPropagationSuccess
            IISSuccess    = $iisManagementSuccess
        }
    }
    catch {
        Write-Log "$Description execution error: $_" -Level "ERROR"
        return @{
            Success       = $false
            Output        = $_.ToString()
            ExitCode      = -1
            ActualSuccess = $false
            DNSSuccess    = $false
            IISSuccess    = $false
        }
    }
}

# Function to get active host
function Get-ActiveHost {
    [CmdletBinding()]
    param()
    
    Write-Log "Attempting to determine current active host..." -Level "INFO"
    
    # Create the command line
    $cmdArgs = "-Env `"$Env`" -Ops `"check`""
    
    # Add optional parameters if provided
    if ($dnsServer) {
        $cmdArgs += " -dnsServer `"$dnsServer`""
    }
    
    if ($lookupZone) {
        $cmdArgs += " -lookupZone `"$lookupZone`""
    }
    
    try {
        # Execute the script using Invoke-Expression
        $tempFile = [System.IO.Path]::GetTempFileName()
        $scriptCmd = "& '$dnsFailoverScriptPath' $cmdArgs *>&1 | Tee-Object -FilePath '$tempFile'"
        
        Write-Log "Running command: $scriptCmd" -Level "DEBUG"
        $result = Invoke-Expression $scriptCmd
        $exitCode = $LASTEXITCODE
        
        # Read the captured output
        $output = Get-Content -Path $tempFile -Raw
        Remove-Item -Path $tempFile -Force
        
        # Log the output for debugging
        Write-Log "Script output: $output" -Level "DEBUG"
        
        # Parse the output to find the active host - trying multiple patterns
        $activeHost = $null
        
        # Pattern 1: Direct "Active host check completed: X" message
        if ($output -match "Active host check completed: ([^\s]+)") {
            $activeHost = $Matches[1]
        }
        # Pattern 2: Look for "Active host is reachable: X" message
        elseif ($output -match "Active host is reachable: ([^\s]+)") {
            $activeHost = $Matches[1]
        }
        # Pattern 3: Look for "Current active host: X" message
        elseif ($output -match "Current active host(?: before failover)?: ([^\s]+)") {
            $activeHost = $Matches[1]
        }
        # Additional fallback patterns
        elseif ($output -match "host(?:name)?\s+obtained: ([^\s]+)") {
            $activeHost = $Matches[1]
        }
        elseif ($output -match "Pinging\s+([^\s]+)\s+") {
            $activeHost = $Matches[1]
        }
        elseif ($output -match "Active host is: ([^\s]+)") {
            $activeHost = $Matches[1]
        }
        
        if ($activeHost) {
            Write-Log "Current active host: $activeHost" -Level "INFO"
            return $activeHost
        }
        else {
            Write-Log "Could not determine active host from script output" -Level "ERROR"
            Write-Log "Script output: $output" -Level "DEBUG"
            return $null
        }
    }
    catch {
        Write-Log "Error getting active host: $_" -Level "ERROR"
        return $null
    }
}

# Function to check server availability
function Test-ServerAvailability {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [string]$ServerName,
        
        [Parameter(Mandatory = $false)]
        [int]$Timeout = 60,
        
        [Parameter(Mandatory = $false)]
        [int]$CheckInterval = 5
    )
    
    Write-Log "Checking availability of server: $ServerName" -Level "INFO"
    $startTime = Get-Date
    $serverAvailable = $false
    
    do {
        try {
            # Try pinging the server
            $ping = Test-Connection -ComputerName $ServerName -Count 1 -Quiet -ErrorAction SilentlyContinue
            
            if ($ping) {
                # Try WMI connection to verify the server is ready
                try {
                    $wmiResult = Get-WmiObject -Class Win32_OperatingSystem -ComputerName $ServerName -ErrorAction SilentlyContinue
                    
                    if ($wmiResult) {
                        $serverAvailable = $true
                        try {
                            # Convert WMI datetime strings to DateTime objects first
                            $lastBootTime = [Management.ManagementDateTimeConverter]::ToDateTime($wmiResult.LastBootUpTime)
                            $localDateTime = [Management.ManagementDateTimeConverter]::ToDateTime($wmiResult.LocalDateTime)
                            $uptimeMinutes = [math]::Round(($localDateTime - $lastBootTime).TotalMinutes, 2)
                            Write-Log "Server $ServerName is available (Uptime: $uptimeMinutes minutes)" -Level "SUCCESS"
                        }
                        catch {
                            # Some systems may have issues calculating uptime
                            Write-Log "Server $ServerName is available (Uptime calculation error: $_)" -Level "SUCCESS"
                        }
                        break
                    }
                    else {
                        Write-Log "Server $ServerName is pingable but WMI query returned null" -Level "INFO"
                    }
                }
                catch {
                    Write-Log "Server $ServerName is pingable but WMI query failed: $_" -Level "INFO"
                    
                    # Alternative check - try basic PowerShell remoting
                    try {
                        $psSession = New-PSSession -ComputerName $ServerName -ErrorAction SilentlyContinue
                        if ($psSession) {
                            Remove-PSSession $psSession -ErrorAction SilentlyContinue
                            $serverAvailable = $true
                            Write-Log "Server $ServerName is available (verified via PowerShell remoting)" -Level "SUCCESS"
                            break
                        }
                    }
                    catch {
                        Write-Log "Server $ServerName PowerShell remoting check failed: $_" -Level "DEBUG"
                    }
                }
            }
            else {
                Write-Log "Server $ServerName is not responding to ping" -Level "INFO"
            }
        }
        catch {
            Write-Log "Error checking server availability: $_" -Level "DEBUG"
        }
        
        # Calculate elapsed time
        $elapsedTime = (Get-Date) - $startTime
        $elapsedSeconds = $elapsedTime.TotalSeconds
        $remainingSeconds = $Timeout - $elapsedSeconds
        
        if ($remainingSeconds -le 0) {
            Write-Log "Timeout reached while waiting for server $ServerName to be available" -Level "WARNING"
            break
        }
        
        Write-Log "Waiting for server $ServerName... ($([math]::Round($elapsedSeconds, 0))s elapsed, $([math]::Round($remainingSeconds, 0))s remaining)" -Level "INFO"
        Start-Sleep -Seconds $CheckInterval
        
    } while (-not $serverAvailable -and $elapsedTime.TotalSeconds -lt $Timeout)
    
    return $serverAvailable
}

# Function to safely restart server
function Restart-RemoteServer {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [string]$ServerName,
        
        [Parameter(Mandatory = $false)]
        [int]$Timeout = 600,
        
        [Parameter(Mandatory = $false)]
        [int]$CheckInterval = 15,
        
        [Parameter(Mandatory = $false)]
        [int]$MaxRetryAttempts = 3
    )
    
    Write-Log "Initiating restart of server: $ServerName" -Level "INFO"
    
    # Try multiple methods to restart the server
    $restartSuccess = $false
    $attemptCount = 0
    
    while (-not $restartSuccess -and $attemptCount -lt $MaxRetryAttempts) {
        $attemptCount++
        Write-Log "Restart attempt $attemptCount of $MaxRetryAttempts" -Level "INFO"
        
        try {
            # First check if the server is accessible
            if (-not (Test-Connection -ComputerName $ServerName -Count 1 -Quiet -ErrorAction SilentlyContinue)) {
                Write-Log "Server $ServerName is not accessible for restart" -Level "WARNING"
                
                if ($attemptCount -lt $MaxRetryAttempts) {
                    Write-Log "Waiting 30 seconds before next attempt..." -Level "INFO"
                    Start-Sleep -Seconds 30
                    continue
                }
                else {
                    Write-Log "Maximum retry attempts reached. Cannot restart server." -Level "ERROR"
                    return $false
                }
            }
            
            # Try method 1: PowerShell Invoke-Command with Restart-Computer
            Write-Log "Attempting restart using PowerShell Invoke-Command..." -Level "INFO"
            try {
                Invoke-Command -ComputerName $ServerName -ScriptBlock {
                    Restart-Computer -Force
                } -ErrorAction Stop
                
                $restartSuccess = $true
                Write-Log "Restart command sent successfully via Invoke-Command" -Level "SUCCESS"
            }
            catch {
                Write-Log "Failed to restart using Invoke-Command: $_" -Level "WARNING"
                
                # Try method 2: WMI
                Write-Log "Attempting restart using WMI..." -Level "INFO"
                try {
                    $os = Get-WmiObject -Class Win32_OperatingSystem -ComputerName $ServerName -ErrorAction Stop
                    $result = $os.Win32Shutdown(6) # 6 = Reboot
                    
                    if ($result.ReturnValue -eq 0) {
                        $restartSuccess = $true
                        Write-Log "Restart command sent successfully via WMI" -Level "SUCCESS"
                    }
                    else {
                        Write-Log "WMI restart command returned non-zero value: $($result.ReturnValue)" -Level "WARNING"
                    }
                }
                catch {
                    Write-Log "Failed to restart using WMI: $_" -Level "WARNING"
                    
                    # Try method 3: shutdown.exe
                    Write-Log "Attempting restart using shutdown.exe..." -Level "INFO"
                    try {
                        $process = Start-Process -FilePath "shutdown.exe" -ArgumentList "/r /f /m \\$ServerName /t 0" -Wait -PassThru -NoNewWindow
                        
                        if ($process.ExitCode -eq 0) {
                            $restartSuccess = $true
                            Write-Log "Restart command sent successfully via shutdown.exe" -Level "SUCCESS"
                        }
                        else {
                            Write-Log "shutdown.exe returned non-zero exit code: $($process.ExitCode)" -Level "WARNING"
                        }
                    }
                    catch {
                        Write-Log "Failed to restart using shutdown.exe: $_" -Level "WARNING"
                    }
                }
            }
        }
        catch {
            Write-Log "Error during restart attempt $attemptCount : $_" -Level "ERROR"
        }
        
        if (-not $restartSuccess -and $attemptCount -lt $MaxRetryAttempts) {
            Write-Log "Waiting 30 seconds before next attempt..." -Level "INFO"
            Start-Sleep -Seconds 30
        }
    }
    
    if (-not $restartSuccess) {
        Write-Log "All restart methods failed after $MaxRetryAttempts attempts" -Level "ERROR"
        return $false
    }
    
    # Give the server a moment to start shutting down
    Start-Sleep -Seconds 10
    
    # Wait for the server to become unavailable (confirm it's restarting)
    $startTime = Get-Date
    $serverUnavailable = $false
    $offlineTimeout = 120  # 2 minutes to go offline
    
    Write-Log "Waiting for server $ServerName to go offline..." -Level "INFO"
    do {
        if (-not (Test-Connection -ComputerName $ServerName -Count 1 -Quiet -ErrorAction SilentlyContinue)) {
            $serverUnavailable = $true
            Write-Log "Server $ServerName is now offline, waiting for it to come back online" -Level "INFO"
            break
        }
        
        Start-Sleep -Seconds 5
        $elapsedTime = (Get-Date) - $startTime
        
        if ($elapsedTime.TotalSeconds -gt $offlineTimeout) {
            Write-Log "Timeout reached while waiting for server to go offline. Continuing anyway..." -Level "WARNING"
            $serverUnavailable = $true
            break
        }
    } while (-not $serverUnavailable)
    
    # Now wait for the server to come back online
    $serverAvailable = Test-ServerAvailability -ServerName $ServerName -Timeout $Timeout -CheckInterval $CheckInterval
    
    if ($serverAvailable) {
        Write-Log "Server $ServerName has been successfully restarted" -Level "SUCCESS"
        return $true
    }
    else {
        Write-Log "Server $ServerName failed to come back online within the timeout period" -Level "ERROR"
        return $false
    }
}

# Function to wait for DNS propagation and IIS services
function Wait-ForDNSPropagation {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [string]$ExpectedHost,
        
        [Parameter(Mandatory = $false)]
        [int]$Timeout = 300,
        
        [Parameter(Mandatory = $false)]
        [int]$CheckInterval = 15
    )
    
    Write-Log "Waiting for DNS propagation to $ExpectedHost..." -Level "INFO"
    $startTime = Get-Date
    $propagationComplete = $false
    
    do {
        # Get current active host
        $currentHost = Get-ActiveHost
        
        if ($currentHost) {
            # Compare hostnames (ignoring case and domain)
            $shortCurrentHost = Get-HostnameFromFQDN -FQDN $currentHost
            $shortExpectedHost = Get-HostnameFromFQDN -FQDN $ExpectedHost
            
            if ($shortCurrentHost -ieq $shortExpectedHost) {
                $propagationComplete = $true
                Write-Log "DNS propagation complete. Active host is now $currentHost" -Level "SUCCESS"
                
                # Test if the server is fully available
                Write-Log "Testing if $currentHost is fully available..." -Level "INFO"
                $serverAvailable = Test-ServerAvailability -ServerName $shortCurrentHost -Timeout 60 -CheckInterval 5
                
                if ($serverAvailable) {
                    Write-Log "Server $currentHost is fully available" -Level "SUCCESS"
                }
                else {
                    Write-Log "Server $currentHost is reachable but may not be fully operational" -Level "WARNING"
                }
                
                break
            }
            
            Write-Log "DNS not yet propagated. Current: $currentHost, Expected: $ExpectedHost" -Level "INFO"
        }
        else {
            Write-Log "Unable to determine current active host" -Level "WARNING"
        }
        
        # Calculate elapsed time
        $elapsedTime = (Get-Date) - $startTime
        $elapsedSeconds = $elapsedTime.TotalSeconds
        $remainingSeconds = $Timeout - $elapsedSeconds
        
        if ($remainingSeconds -le 0) {
            Write-Log "Timeout reached while waiting for DNS propagation" -Level "WARNING"
            break
        }
        
        Write-Log "Waiting for DNS propagation... ($([math]::Round($elapsedSeconds, 0))s elapsed, $([math]::Round($remainingSeconds, 0))s remaining)" -Level "INFO"
        Start-Sleep -Seconds $CheckInterval
        
    } while (-not $propagationComplete -and $elapsedTime.TotalSeconds -lt $Timeout)
    
    return $propagationComplete
}

# Main function to execute complete failover cycle
function Invoke-CompleteFailoverCycle {
    [CmdletBinding()]
    param()
    
    Write-Log "Starting complete failover cycle in $Env environment" -Level "INFO"
    
    # Validate the DNS Failover script exists
    if (-not (Test-DNSFailoverScript)) {
        return $false
    }
    
    # STEP 1: FIRST FAILOVER (A → B)
    Write-Log "PHASE 1: INITIAL FAILOVER (A → B)" -Level "INFO"
    
    # Get initial active host before failover
    $initialHost = Get-ActiveHost
    if (-not $initialHost) {
        Write-Log "Cannot proceed without determining initial active host" -Level "ERROR"
        return $false
    }
    
    # Clean up host names
    $initialShortHost = Get-HostnameFromFQDN -FQDN $initialHost
    Write-Log "Initial active host: $initialHost (short name: $initialShortHost)" -Level "INFO"
    
    # Execute first failover
    $failoverResult = Invoke-DNSFailover -Operation "failover" -Description "Initial Failover (A → B)"
    if (-not $failoverResult.Success -and -not $failoverResult.ActualSuccess) {
        Write-Log "First failover operation failed, aborting cycle" -Level "ERROR"
        return $false
    }
    
    # If we're here, either Success was true or ActualSuccess was true - continue processing
    if ($failoverResult.ActualSuccess -and -not $failoverResult.Success) {
        Write-Log "First failover reported non-zero exit code but operation appears successful - continuing" -Level "WARNING"
    }
    
    # Get new active host after first failover
    $newActiveHost = Get-ActiveHost
    if (-not $newActiveHost) {
        Write-Log "Cannot determine new active host after first failover" -Level "ERROR"
        return $false
    }
    
    $newShortHost = Get-HostnameFromFQDN -FQDN $newActiveHost
    
    # Compare short hostnames to check if failover was successful
    if ($initialShortHost -ieq $newShortHost) {
        Write-Log "First failover appears to have failed - active host did not change" -Level "ERROR"
        Write-Log "Initial host: $initialHost, Current host: $newActiveHost" -Level "ERROR"
        return $false
    }
    
    Write-Log "First failover successful: $initialHost → $newActiveHost" -Level "SUCCESS"
    
    # Wait for DNS propagation and services to be ready
    $propagationSuccess = Wait-ForDNSPropagation -ExpectedHost $newActiveHost -Timeout 300 -CheckInterval 15
    if (-not $propagationSuccess) {
        Write-Log "DNS propagation check timed out, but continuing with the process" -Level "WARNING"
    }
    
    # STEP 2: RESTART ORIGINAL SERVER
    if (-not $SkipServerRestart) {
        Write-Log "PHASE 2: RESTARTING ORIGINAL SERVER" -Level "INFO"
        
        Write-Log "Restarting original server: $initialShortHost" -Level "INFO"
        $restartSuccess = Restart-RemoteServer -ServerName $initialShortHost -Timeout $serverRestartTimeout -CheckInterval $serverAvailabilityCheckInterval
        
        if (-not $restartSuccess) {
            Write-Log "Failed to restart original server $initialShortHost, aborting cycle" -Level "ERROR"
            return $false
        }
        
        Write-Log "Original server $initialShortHost has been successfully restarted" -Level "SUCCESS"
        
        # Additional wait time to ensure the server is stabilized
        Write-Log "Allowing additional time for services to stabilize..." -Level "INFO"
        Start-Sleep -Seconds 60
    }
    else {
        Write-Log "PHASE 2: SKIPPING SERVER RESTART (as requested)" -Level "INFO"
    }
    
    # STEP 3: SECOND FAILOVER (B → A)
    if (-not $SkipSecondFailover) {
        Write-Log "PHASE 3: SECOND FAILOVER (B → A)" -Level "INFO"
        
        # Execute second failover
        $failoverResult = Invoke-DNSFailover -Operation "failover" -Description "Second Failover (B → A)"
        if (-not $failoverResult.Success -and -not $failoverResult.ActualSuccess) {
            Write-Log "Second failover operation failed" -Level "ERROR"
            return $false
        }
        
        # If we're here, either Success was true or ActualSuccess was true - continue processing
        if ($failoverResult.ActualSuccess -and -not $failoverResult.Success) {
            Write-Log "Second failover reported non-zero exit code but operation appears successful - continuing" -Level "WARNING"
        }
        
        # Get final active host after second failover
        $finalActiveHost = Get-ActiveHost
        if (-not $finalActiveHost) {
            Write-Log "Cannot determine final active host after second failover" -Level "ERROR"
            return $false
        }
        
        $finalShortHost = Get-HostnameFromFQDN -FQDN $finalActiveHost
        
        # Compare short hostnames to check if second failover was successful
        if ($newShortHost -ieq $finalShortHost) {
            Write-Log "Second failover appears to have failed - active host did not change" -Level "ERROR"
            Write-Log "Host after first failover: $newActiveHost, Current host: $finalActiveHost" -Level "ERROR"
            return $false
        }
        
        # Check if we're back to the original host
        if ($initialShortHost -ine $finalShortHost) {
            Write-Log "Second failover did not return to original host ($initialShortHost → $finalShortHost)" -Level "WARNING"
        }
        else {
            Write-Log "Second failover successful: $newActiveHost → $finalActiveHost" -Level "SUCCESS"
        }
        
        # Wait for DNS propagation and services to be ready
        $propagationSuccess = Wait-ForDNSPropagation -ExpectedHost $finalActiveHost -Timeout 300 -CheckInterval 15
        if (-not $propagationSuccess) {
            Write-Log "DNS propagation check timed out, but completing the process" -Level "WARNING"
        }
    }
    else {
        Write-Log "PHASE 3: SKIPPING SECOND FAILOVER (as requested)" -Level "INFO"
    }
    
    # Summary
    Write-Log "Complete failover cycle execution summary:" -Level "INFO"
    Write-Log "- Initial active host: $initialHost" -Level "INFO"
    Write-Log "- After first failover: $newActiveHost" -Level "INFO"
    
    if (-not $SkipSecondFailover) {
        Write-Log "- After second failover: $finalActiveHost" -Level "INFO"
    }
    
    # Report on IIS management
    if ($failoverResult.IISSuccess) {
        Write-Log "- IIS application pools were successfully restarted and recycled" -Level "SUCCESS"
    }
    else {
        Write-Log "- IIS application pool management showed mixed or incomplete results" -Level "WARNING"
    }
    
    Write-Log "Complete failover cycle finished" -Level "SUCCESS"
    return $true
}

# Main execution
try {
    Write-Log "=======================================================" -Level "INFO"
    Write-Log "Starting Complete Failover Cycle Script" -Level "INFO"
    Write-Log "Environment: $Env, DNS Server: $dnsServer, Lookup Zone: $lookupZone" -Level "INFO"
    Write-Log "=======================================================" -Level "INFO"
    
    $success = Invoke-CompleteFailoverCycle
    
    if ($success) {
        Write-Log "Script completed successfully" -Level "SUCCESS"
        exit 0
    }
    else {
        Write-Log "Script completed with errors" -Level "ERROR"
        exit 1
    }
}
catch {
    Write-Log "An unhandled exception occurred: $_" -Level "ERROR"
    Write-Log $_.ScriptStackTrace -Level "ERROR"
    exit 1
}
finally {
    Write-Log "Script execution finished" -Level "INFO"
}