autoFailoverMonitor.ps1

autoFailoverMonitor.ps1

# autoFailoverMonitor.ps1
# This script monitors the active server for COMException errors in EventLog
# and automatically triggers a complete failover cycle when threshold is met.

[CmdletBinding()]
param(
    [Parameter(Mandatory = $false)]
    [ValidateSet("Dev", "Prod")]
    [string]$Env = "Dev",
    
    [Parameter(Mandatory = $false)]
    [string]$dnsServer,
    
    [Parameter(Mandatory = $false)]
    [string]$lookupZone,
    
    [Parameter(Mandatory = $false)]
    [string]$CompleteFailoverScriptPath = "$PSScriptRoot\completeFailoverCycle.ps1",
    
    [Parameter(Mandatory = $false)]
    [string]$logFilePath = "$PSScriptRoot\AutoFailoverMonitor.log",
    
    [Parameter(Mandatory = $false)]
    [string]$stateFilePath = "$PSScriptRoot\AutoFailoverState.json",
    
    [Parameter(Mandatory = $false)]
    [int]$ErrorThreshold = 3,
    
    [Parameter(Mandatory = $false)]
    [int]$TimeWindowMinutes = 10,
    
    [Parameter(Mandatory = $false)]
    [int]$CooldownPeriodMinutes = 45,

    [Parameter(Mandatory = $false)]
    [int]$DefaultTTLMinutes = 3,

    [Parameter(Mandatory = $false)]
    [int]$ReducedTTLMinutes = 1,

    [Parameter(Mandatory = $false)]
    [string]$DevDnsName = "LendingWebServerDev",

    [Parameter(Mandatory = $false)]
    [string]$ProdDnsName = "LendingWebServer",

    [Parameter(Mandatory = $false)]
    [int]$MaxFailoverAttempts = 10,

    [Parameter(Mandatory = $false)]
    [int]$DnsVerificationWaitSeconds = 10,
    
    [Parameter(Mandatory = $false)]
    [switch]$RunAsService,
    
    [Parameter(Mandatory = $false)]
    [switch]$ForceFailover,
    
    [Parameter(Mandatory = $false)]
    [switch]$Initialize,
    
    [Parameter(Mandatory = $false)]
    [switch]$TestMode,
    
    [Parameter(Mandatory = $false)]
    [switch]$SimulateError,
    
    [Parameter(Mandatory = $false)]
    [switch]$Help
)

# Function to show help information
function Show-Help {
    $helpText = @"
Automatic Failover Monitor Help
===============================

Description:
    This script monitors the active server for COMException errors in the Event Log
    and automatically triggers a complete failover cycle when threshold is met.

Syntax:
    .\autoFailoverMonitor.ps1 [-Env <String>] [-dnsServer <String>] [-lookupZone <String>] 
                              [-CompleteFailoverScriptPath <String>] [-logFilePath <String>]
                              [-stateFilePath <String>] [-ErrorThreshold <Int>] 
                              [-TimeWindowMinutes <Int>] [-CooldownPeriodMinutes <Int>]
                              [-DefaultTTLMinutes <Int>] [-ReducedTTLMinutes <Int>]
                              [-DevDnsName <String>] [-ProdDnsName <String>]
                              [-MaxFailoverAttempts <Int>] [-RetryWaitSeconds <Int>]
                              [-DnsVerificationWaitSeconds <Int>]
                              [-RunAsService] [-ForceFailover] [-Initialize] [-TestMode]
                              [-SimulateError] [-Help]

Parameters:
    -Env <String>
        Specifies the environment to operate in.
        Valid values: Dev, Prod
        Default: Dev

    -dnsServer <String>
        Specifies the DNS server to use for operations.
        
    -lookupZone <String>
        Specifies the DNS lookup zone.
        
    -CompleteFailoverScriptPath <String>
        Path to the completeFailoverCycle.ps1 script.
        Default: .\completeFailoverCycle.ps1
        
    -logFilePath <String>
        Path to the log file.
        Default: .\AutoFailoverMonitor.log
        
    -stateFilePath <String>
        Path to the state file that tracks error occurrences and cooldown period.
        Default: .\AutoFailoverState.json
        
    -ErrorThreshold <Int>
        Number of COMException errors that must occur within the time window to trigger failover.
        Default: 3
        
    -TimeWindowMinutes <Int>
        Time window in minutes within which errors are counted.
        Default: 10
        
    -CooldownPeriodMinutes <Int>
        Cooldown period in minutes after a failover during which no new failover will be triggered.
        Default: 45
        
    -DefaultTTLMinutes <Int>
        Standard DNS TTL value in minutes to use during normal operations.
        Default: 3
        
    -ReducedTTLMinutes <Int>
        Reduced DNS TTL value in minutes to use when errors are detected.
        Default: 1
        
    -DevDnsName <String>
        DNS name to use in the Dev environment.
        Default: LendingWebServerDev
        
    -ProdDnsName <String>
        DNS name to use in the Prod environment.
        Default: LendingWebServer
        
    -MaxFailoverAttempts <Int>
        Maximum number of failover attempts allowed in a 24-hour period.
        Default: 10
        
    -RetryWaitSeconds <Int>
        Number of seconds to wait before retrying a failed DNS operation.
        Default: 5
        
    -DnsVerificationWaitSeconds <Int>
        Number of seconds to wait after DNS operations for changes to propagate.
        Default: 2
        
    -RunAsService [Switch]
        If specified, the script will run as a continuous monitoring service.
        
    -ForceFailover [Switch]
        If specified, forces a failover regardless of error count or cooldown period.
        
    -Initialize [Switch]
        If specified, initializes the monitoring environment (creates state file, etc.)
        
    -TestMode [Switch]
        If specified, runs in test mode without actually triggering failover.
        
    -SimulateError [Switch]
        If specified, simulates COMException errors for testing purposes.
        
    -Help [Switch]
        Shows this help message.

Examples:
    # Show help
    .\autoFailoverMonitor.ps1 -Help

    # Initialize the monitoring environment
    .\autoFailoverMonitor.ps1 -Initialize
    
    # Run the monitor once to check for errors and trigger failover if needed
    .\autoFailoverMonitor.ps1 -Env Prod -dnsServer "dns1.company.com" -lookupZone "company.local"

    # Run with custom TTL settings
    .\autoFailoverMonitor.ps1 -Env Prod -DefaultTTLMinutes 5 -ReducedTTLMinutes 2

    # Run the monitor as a continuous service
    .\autoFailoverMonitor.ps1 -Env Prod -RunAsService -MaxFailoverAttempts 15

    # Force a failover regardless of error count or cooldown period
    .\autoFailoverMonitor.ps1 -ForceFailover

    # Test the monitoring without triggering actual failover
    .\autoFailoverMonitor.ps1 -TestMode -SimulateError

Notes:
    - This script requires the completeFailoverCycle.ps1 script to be accessible
    - Appropriate permissions are required for reading Event Logs and executing the failover script
    - All operations are logged to the specified log file
    - The script maintains state to track error occurrences and enforce cooldown periods
    - TTL management functions automatically reduce DNS cache times when errors are detected
"@

    Write-Host $helpText
    exit 0
}

# Show help if requested
if ($Help) {
    Show-Help
}

# Enable strict mode for better error handling
Set-StrictMode -Version Latest

# Set error action preference to stop script execution on error
$ErrorActionPreference = 'Stop'

#region Supporting Functions

# Function to write log messages
function Write-Log {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [string]$Message,
        
        [Parameter(Mandatory = $false)]
        [ValidateSet("INFO", "WARNING", "ERROR", "SUCCESS", "DEBUG")]
        [string]$Level = "INFO"
    )
    
    $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
    $logMessage = "[$timestamp] [$Level] $Message"
    
    # Write to console with appropriate color
    switch ($Level) {
        "INFO" { Write-Host $logMessage -ForegroundColor Cyan }
        "WARNING" { Write-Host $logMessage -ForegroundColor Yellow }
        "ERROR" { Write-Host $logMessage -ForegroundColor Red }
        "SUCCESS" { Write-Host $logMessage -ForegroundColor Green }
        "DEBUG" { 
            # Only show debug messages in verbose mode
            if ($VerbosePreference -eq 'Continue') {
                Write-Host $logMessage -ForegroundColor Gray 
            }
        }
    }
    
    # Append to log file
    try {
        # Create the log directory if it doesn't exist
        $logDir = Split-Path -Path $logFilePath -Parent
        if (-not (Test-Path -Path $logDir -PathType Container)) {
            New-Item -Path $logDir -ItemType Directory -Force | Out-Null
        }
        
        Add-Content -Path $logFilePath -Value $logMessage -ErrorAction Stop
    }
    catch {
        Write-Warning "Failed to write to log file: $_"
    }
}

# Function to rotate log files
function Rotate-LogFile {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [string]$LogPath,
        
        [Parameter(Mandatory = $false)]
        [int]$MaxSizeMB = 10,
        
        [Parameter(Mandatory = $false)]
        [int]$FilesToKeep = 5
    )
    
    # Check if log file exists and exceeds max size
    if (Test-Path $LogPath) {
        $logFile = Get-Item $LogPath
        
        if ($logFile.Length -gt ($MaxSizeMB * 1MB)) {
            Write-Log "Log file size limit reached. Rotating logs..." -Level "DEBUG"
            
            $directory = Split-Path $LogPath -Parent
            $baseName = (Split-Path $LogPath -Leaf).Split('.')[0]
            $extension = if ($logFile.Extension) { $logFile.Extension } else { ".log" }
            $timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
            $newName = Join-Path $directory "$($baseName)_$($timestamp)$extension"
            
            # Rename current log file
            try {
                Copy-Item -Path $LogPath -Destination $newName -Force
                Remove-Item -Path $LogPath -Force
                Write-Host "Log file rotated to: $newName"
                
                # Clean up old log files
                $oldLogs = Get-ChildItem -Path $directory -Filter "$baseName*$extension" | 
                Where-Object { $_.Name -ne (Split-Path $LogPath -Leaf) } |
                Sort-Object LastWriteTime -Descending | 
                Select-Object -Skip $FilesToKeep
                
                foreach ($old in $oldLogs) {
                    Remove-Item $old.FullName -Force
                    Write-Host "Removed old log file: $($old.Name)"
                }
            }
            catch {
                Write-Warning "Failed to rotate log file: $_"
            }
        }
    }
}

function Update-DnsTTL {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [string]$DnsServer,
        
        [Parameter(Mandatory = $true)]
        [string]$LookupZone,
        
        [Parameter(Mandatory = $true)]
        [string]$DnsName,
        
        [Parameter(Mandatory = $true)]
        [int]$TTLMinutes,
        
        [Parameter(Mandatory = $false)]
        [int]$RetryAttempts = 2,
        
        [Parameter(Mandatory = $false)]
        [int]$RetryWaitSeconds = 5
    )
    
    try {
        Write-Log "Attempting to update TTL for $DnsName to $TTLMinutes minutes" -Level "INFO"
    
        for ($attempt = 1; $attempt -le $RetryAttempts; $attempt++) {
            try {
                # Get current CNAME record
                $dnsRecord = Get-DnsServerResourceRecord -Name $DnsName -RRType CName -ZoneName $LookupZone -ComputerName $DnsServer -ErrorAction Stop
                if ($null -eq $dnsRecord) {
                    Write-Log "DNS record not found: $DnsName" -Level "ERROR"
                    return $false
                }
            
                $currentAlias = $dnsRecord.RecordData.HostNameAlias
                $currentTTL = $dnsRecord.TimeToLive.TotalMinutes
            
                Write-Log "Attempt $attempt : Current record - Alias: $currentAlias, TTL: $currentTTL minutes" -Level "INFO"
            
                # Only update if the TTL actually needs to change
                if ([Math]::Abs($currentTTL - $TTLMinutes) -lt 0.1) {
                    Write-Log "TTL already set to $TTLMinutes minutes, no update needed" -Level "INFO"
                    return $true
                }
            
                # Create new TTL timespan
                $newTTL = [System.TimeSpan]::FromMinutes($TTLMinutes)
            
                # Remove existing record
                Write-Log "Removing existing DNS record" -Level "INFO"
                Remove-DnsServerResourceRecord -ZoneName $LookupZone -Name $DnsName -RRType CName -ComputerName $DnsServer -Force -ErrorAction Stop
            
                # Brief pause to ensure record is fully removed
                Start-Sleep -Seconds $DnsVerificationWaitSeconds
            
                # Add record with new TTL
                Write-Log "Adding DNS record with new TTL" -Level "INFO"
                Add-DnsServerResourceRecordCName -Name $DnsName -HostNameAlias $currentAlias -ZoneName $LookupZone -ComputerName $DnsServer -TimeToLive $newTTL -ErrorAction Stop
            
                # Allow time for the change to propagate
                Start-Sleep -Seconds $DnsVerificationWaitSeconds
            
                # Verify the update
                $verifyRecord = Get-DnsServerResourceRecord -Name $DnsName -RRType CName -ZoneName $LookupZone -ComputerName $DnsServer -ErrorAction Stop
                $actualTTL = $verifyRecord.TimeToLive.TotalMinutes
            
                if ([Math]::Abs($actualTTL - $TTLMinutes) -lt 0.1) {
                    Write-Log "Successfully updated TTL from $currentTTL to $actualTTL minutes" -Level "SUCCESS"
                
                    # Force DNS server to commit changes
                    Write-Log "Forcing DNS server to commit changes" -Level "INFO"
                    return $true
                }
                else {
                    Write-Log "TTL verification failed. Current: $actualTTL, Expected: $TTLMinutes" -Level "WARNING"
                
                    if ($attempt -lt $RetryAttempts) {
                        Write-Log "Will retry TTL update (attempt $attempt of $RetryAttempts)" -Level "WARNING"
                        Start-Sleep -Seconds $RetryWaitSeconds
                    }
                    else {
                        Write-Log "Maximum retry attempts reached" -Level "ERROR"
                        return $false
                    }
                }
            }
            catch {
                Write-Log "Error updating TTL (attempt $attempt of $RetryAttempts): $_" -Level "ERROR"
            
                if ($attempt -lt $RetryAttempts) {
                    Write-Log "Retrying in $RetryWaitSeconds seconds..." -Level "WARNING"
                    Start-Sleep -Seconds $RetryWaitSeconds
                }
                else {
                    Write-Log "Failed to update TTL after all retry attempts" -Level "ERROR"
                    return $false
                }
            }
        }
    }
    catch {
        Write-Log "Failed to update DNS TTL: $_" -Level "ERROR"
        return $false
    }
}


# Function to validate the CompleteFailoverCycle script exists
function Test-CompleteFailoverScript {
    if (-not (Test-Path -Path $CompleteFailoverScriptPath)) {
        Write-Log "CompleteFailoverCycle script not found at: $CompleteFailoverScriptPath" -Level "ERROR"
        Write-Log "Please provide the correct path using -CompleteFailoverScriptPath parameter" -Level "ERROR"
        return $false
    }
    return $true
}

# Function to extract hostname from FQDN
function Get-HostnameFromFQDN {
    param (
        [Parameter(Mandatory = $true)]
        [string]$FQDN
    )
    
    # Extract hostname part (remove domain if present)
    return $FQDN -replace '\..*$', ''
}

# Function to get current active server
function Get-CurrentActiveServer {
    try {
        # Determine the path to the dnsFailover script
        $dnsFailoverScriptPath = Join-Path (Split-Path $CompleteFailoverScriptPath -Parent) "dnsFailover_v2.ps1"
        if (-not (Test-Path $dnsFailoverScriptPath)) {
            Write-Log "DNS Failover script not found at: $dnsFailoverScriptPath" -Level "ERROR"
            return $null
        }
        
        Write-Log "Attempting to determine current active host..." -Level "INFO"
        
        # Create the command line
        $cmdArgs = "-Env `"$Env`" -Ops `"check`""
        
        # Add optional parameters if provided
        if ($dnsServer) {
            $cmdArgs += " -dnsServer `"$dnsServer`""
        }
        
        if ($lookupZone) {
            $cmdArgs += " -lookupZone `"$lookupZone`""
        }
        
        # Execute the script using Invoke-Expression
        $tempFile = [System.IO.Path]::GetTempFileName()
        $scriptCmd = "& '$dnsFailoverScriptPath' $cmdArgs *>&1 | Tee-Object -FilePath '$tempFile'"
        
        Write-Log "Running command: $scriptCmd" -Level "DEBUG"
        $result = Invoke-Expression $scriptCmd
        $exitCode = $LASTEXITCODE
        
        # Read the captured output
        $output = Get-Content -Path $tempFile -Raw -ErrorAction SilentlyContinue
        Remove-Item -Path $tempFile -Force -ErrorAction SilentlyContinue
        
        # Log the output for debugging
        Write-Log "Script output: $output" -Level "DEBUG"
        
        # Parse the output to find the active host - trying multiple patterns
        $activeHost = $null
        
        # Pattern 1: Direct "Active host check completed: X" message
        if ($output -match "Active host check completed: ([^\s]+)") {
            $activeHost = $Matches[1]
        }
        # Pattern 2: Look for "Active host is reachable: X" message
        elseif ($output -match "Active host is reachable: ([^\s]+)") {
            $activeHost = $Matches[1]
        }
        # Pattern 3: Look for "Current active host: X" message
        elseif ($output -match "Current active host(?: before failover)?: ([^\s]+)") {
            $activeHost = $Matches[1]
        }
        # Additional fallback patterns
        elseif ($output -match "host(?:name)?\s+obtained: ([^\s]+)") {
            $activeHost = $Matches[1]
        }
        elseif ($output -match "Pinging\s+([^\s]+)\s+") {
            $activeHost = $Matches[1]
        }
        elseif ($output -match "Active host is: ([^\s]+)") {
            $activeHost = $Matches[1]
        }
        
        # Fallback to a default server name if we cannot determine the active server
        # This is only for simulation/test purposes
        if (-not $activeHost -and ($TestMode -or $SimulateError)) {
            # Use a generic server name based on environment
            # In the Dev environment, there are two potential servers that could be active
            if ($Env -eq "Dev") {
                # Randomly choose between SERVER1 and SERVER2 for Dev environment
                $serverNumber = Get-Random -Minimum 1 -Maximum 3  # Will return either 1 or 2
                $activeHost = "SERVER$serverNumber.$Env.example.com"
            }
            else {
                $activeHost = "SERVER1.$Env.example.com"
            }
            Write-Log "Using default test server: $activeHost" -Level "WARNING"
        }
        
        if ($activeHost) {
            Write-Log "Current active host: $activeHost" -Level "INFO"
            return $activeHost
        }
        else {
            Write-Log "Could not determine active host from script output" -Level "ERROR"
            Write-Log "Script output: $output" -Level "DEBUG"
            return $null
        }
    }
    catch {
        Write-Log "Error getting current active host: $_" -Level "ERROR"
        
        # Fallback to a default server name in case of error
        # This is only for simulation/test purposes
        if ($TestMode -or $SimulateError) {
            if ($Env -eq "Dev") {
                # Randomly choose between SERVER1 and SERVER2 for Dev environment
                $serverNumber = Get-Random -Minimum 1 -Maximum 3  # Will return either 1 or 2
                $activeHost = "SERVER$serverNumber.$Env.example.com"
            }
            else {
                $activeHost = "SERVER1.$Env.example.com"
            }
            Write-Log "Using default test server: $activeHost" -Level "WARNING"
            return $activeHost
        }
        
        return $null
    }
}

# Function to get or initialize state
function Get-State {
    $defaultState = @{
        Errors           = @()
        LastFailoverTime = $null
        FailoverCount    = 0
        TTLStatus        = "Standard"  # Added for TTL tracking (Standard/Reduced)
        LastTTLChange    = $null
    }
    
    # Check if state file exists
    if (-not (Test-Path $stateFilePath)) {
        # Create parent directory if it doesn't exist
        $stateDir = Split-Path -Path $stateFilePath -Parent
        if (-not (Test-Path -Path $stateDir -PathType Container)) {
            try {
                New-Item -Path $stateDir -ItemType Directory -Force | Out-Null
            }
            catch {
                Write-Log "Failed to create state directory: $_" -Level "ERROR"
                return $defaultState
            }
        }
        
        # Create new state file
        try {
            $defaultState | ConvertTo-Json -Depth 5 | Set-Content -Path $stateFilePath
            Write-Log "Initialized new state file at $stateFilePath" -Level "INFO"
        }
        catch {
            Write-Log "Failed to create state file: $_" -Level "WARNING"
        }
        
        return $defaultState
    }
    
    # Try to read existing state file
    try {
        $stateContent = Get-Content -Path $stateFilePath -Raw -ErrorAction Stop
        $state = $stateContent | ConvertFrom-Json -ErrorAction Stop
        
        # Validate required properties exist
        $requiredProperties = @("Errors", "LastFailoverTime", "FailoverCount", "TTLStatus", "LastTTLChange")
        $missingProperties = @()
        
        foreach ($prop in $requiredProperties) {
            if (-not (Get-Member -InputObject $state -Name $prop -MemberType Properties)) {
                $missingProperties += $prop
                Add-Member -InputObject $state -MemberType NoteProperty -Name $prop -Value $defaultState[$prop]
            }
        }
        
        if ($missingProperties.Count -gt 0) {
            Write-Log "Added missing properties to state file: $($missingProperties -join ', ')" -Level "WARNING"
            $state | ConvertTo-Json -Depth 5 | Set-Content -Path $stateFilePath
        }
        
        return $state
    }
    catch {
        Write-Log "Error reading state file: $_" -Level "ERROR"
        
        # Try to create backup of corrupted state file
        try {
            $backupPath = "$stateFilePath.bak"
            Copy-Item -Path $stateFilePath -Destination $backupPath -Force
            Write-Log "Created backup of corrupted state file at $backupPath" -Level "WARNING"
        }
        catch {
            Write-Log "Failed to backup corrupted state file: $_" -Level "ERROR"
        }
        
        # Return default state
        return $defaultState
    }
}

# Function to save state
function Save-State {
    param (
        [Parameter(Mandatory = $true)]
        [PSCustomObject]$State
    )
    
    try {
        $State | ConvertTo-Json -Depth 5 | Set-Content -Path $stateFilePath
        Write-Log "State saved successfully" -Level "DEBUG"
    }
    catch {
        Write-Log "Error saving state: $_" -Level "ERROR"
    }
}

# Function to check if within cooldown period
function Test-CooldownPeriod {
    param (
        [Parameter(Mandatory = $true)]
        [PSCustomObject]$State
    )
    
    if ($null -eq $State.LastFailoverTime) {
        return $false
    }
    
    try {
        $lastFailover = [DateTime]::Parse($State.LastFailoverTime)
        $cooldownEndTime = $lastFailover.AddMinutes($CooldownPeriodMinutes)
        $now = Get-Date
        
        if ($now -lt $cooldownEndTime) {
            $minutesRemaining = [math]::Ceiling(($cooldownEndTime - $now).TotalMinutes)
            Write-Log "Currently in cooldown period. $minutesRemaining minutes remaining before next possible failover." -Level "WARNING"
            return $true
        }
    }
    catch {
        Write-Log "Error calculating cooldown period: $_" -Level "ERROR"
        # If there's an error parsing the date, assume we're not in cooldown
        return $false
    }
    
    return $false
}

#endregion

#region Event Monitoring

# Function to simulate COMException events for testing
function New-SimulatedCOMExceptionEvents {
    param (
        [Parameter(Mandatory = $false)]
        [int]$Count = 3
    )
    
    Write-Log "Simulating $Count COMException events for testing" -Level "INFO"
    
    $events = @()
    $now = Get-Date
    
    # Create error messages that match real COMException patterns
    $comExceptionMessages = @(
        "System.Runtime.InteropServices.COMException: Exception from HRESULT: 0x80010105 (RPC_E_SERVERFAULT)",
        "System.Runtime.InteropServices.COMException: The RPC server is unavailable. (Exception from HRESULT: 0x800706BA)",
        "System.Runtime.InteropServices.COMException: The object invoked has disconnected from its clients. (Exception from HRESULT: 0x80010108)",
        "System.Runtime.InteropServices.COMException: Class not registered (Exception from HRESULT: 0x80040154)",
        "System.Runtime.InteropServices.COMException: The interface is unknown. (Exception from HRESULT: 0x80004002)"
    )
    
    for ($i = 0; $i -lt $Count; $i++) {
        $eventTime = $now.AddMinutes( - ($i * 2))
        $randomMessageIndex = Get-Random -Minimum 0 -Maximum $comExceptionMessages.Count
        
        $errorMessage = $comExceptionMessages[$randomMessageIndex]
        $stackTrace = @"
   at MyApp.ServiceProxy.ExecuteRequest()
   at MyApp.Controller.ProcessCommand()
   at MyApp.Program.Main()
"@
        
        $fullMessage = $errorMessage + "`r`n" + $stackTrace
        
        $event = @{
            TimeCreated = $eventTime
            EventID     = 1000 + $i
            Level       = "Error"
            Message     = $fullMessage
            LogName     = "Application"
        }
        
        $events += $event
        Write-Log "Simulated Event ID: $($event.EventID), Time: $($event.TimeCreated), Error: $($comExceptionMessages[$randomMessageIndex])" -Level "DEBUG"
    }
    
    return $events
}

# Function to check Event Log for COMExceptions
function Get-COMExceptionEvents {
    param (
        [Parameter(Mandatory = $true)]
        [string]$ComputerName,
        
        [Parameter(Mandatory = $true)]
        [int]$MinutesAgo
    )
    
    # If in simulation mode, return simulated events
    if ($SimulateError) {
        return New-SimulatedCOMExceptionEvents
    }
    
    try {
        $startTime = (Get-Date).AddMinutes(-$MinutesAgo)
        $events = @()
        
        Write-Log "Checking for COMException events on $ComputerName in the last $MinutesAgo minutes" -Level "INFO"
        
        # Test if the computer is reachable
        if (-not (Test-Connection -ComputerName $ComputerName -Count 1 -Quiet -ErrorAction SilentlyContinue)) {
            Write-Log "Server $ComputerName is not reachable, cannot query event log" -Level "ERROR"
            return @()
        }
        
        # Define logs to search - Application is primary but also check System
        $logsToSearch = @('Application', 'System')
        
        foreach ($logName in $logsToSearch) {
            # Create filter for the query
            $filter = @{
                LogName   = $logName
                StartTime = $startTime
                EndTime   = Get-Date
            }
            
            try {
                # First attempt - Get-WinEvent with remote computer
                $logEvents = Get-WinEvent -FilterHashtable $filter -ComputerName $ComputerName -ErrorAction Stop
                
                # Filter for COMException - case insensitive
                $comExceptionEvents = $logEvents | Where-Object { $_.Message -match "(?i)COMException" }
                
                foreach ($event in $comExceptionEvents) {
                    $eventInfo = @{
                        TimeCreated = $event.TimeCreated
                        EventID     = $event.Id
                        Level       = $event.LevelDisplayName
                        Message     = $event.Message
                        LogName     = $logName
                    }
                    
                    $events += $eventInfo
                    Write-Log "Found COMException in $logName log: Event ID: $($event.Id), Time: $($event.TimeCreated)" -Level "DEBUG"
                }
            }
            catch {
                # Handle the specific case when no events were found
                if ($_.Exception.Message -match "No events were found that match the specified selection criteria") {
                    Write-Log "No events found in $logName log matching criteria" -Level "INFO"
                    continue
                }
                
                Write-Log "Error querying $logName event log with Get-WinEvent: $_" -Level "WARNING"
                
                try {
                    # Second attempt - PowerShell session
                    $scriptBlock = {
                        param($filterStart, $filterEnd, $logName)
                        $filter = @{
                            LogName   = $logName
                            StartTime = $filterStart
                            EndTime   = $filterEnd
                        }
                        
                        try {
                            $events = Get-WinEvent -FilterHashtable $filter -ErrorAction Stop
                            return $events | Where-Object { $_.Message -match "(?i)COMException" }
                        }
                        catch {
                            if ($_.Exception.Message -match "No events were found that match the specified selection criteria") {
                                return @()
                            }
                            throw $_
                        }
                    }
                    
                    $session = New-PSSession -ComputerName $ComputerName -ErrorAction Stop
                    $sessionEvents = Invoke-Command -Session $session -ScriptBlock $scriptBlock -ArgumentList $startTime, (Get-Date), $logName
                    Remove-PSSession $session -ErrorAction SilentlyContinue
                    
                    foreach ($event in $sessionEvents) {
                        $eventInfo = @{
                            TimeCreated = $event.TimeCreated
                            EventID     = $event.Id
                            Level       = $event.LevelDisplayName
                            Message     = $event.Message
                            LogName     = $logName
                        }
                        
                        $events += $eventInfo
                        Write-Log "Found COMException in $logName log via PS Session: Event ID: $($event.Id), Time: $($event.TimeCreated)" -Level "DEBUG"
                    }
                }
                catch {
                    # Handle the specific case when no events were found
                    if ($_.Exception.Message -match "No events were found that match the specified selection criteria") {
                        Write-Log "No events found in $logName log matching criteria (via PS Session)" -Level "INFO"
                    }
                    else {
                        Write-Log "Error querying $logName event log with PowerShell session: $_" -Level "WARNING"
                    }
                }
            }
        }
        
        # Log results
        if ($events.Count -gt 0) {
            Write-Log "Found $($events.Count) COMException events across all logs" -Level "WARNING"
        }
        else {
            Write-Log "No COMException events found in the specified time period" -Level "INFO"
        }
        
        return $events
    }
    catch {
        Write-Log "Error in event monitoring: $_" -Level "ERROR"
        # Ensure we return an empty array rather than null
        return @()
    }
}

# Function to update error state
function Update-ErrorState {
    param (
        [Parameter(Mandatory = $true)]
        [PSCustomObject]$State,
        
        [Parameter(ValueFromPipeline = $true)]
        [object[]]$NewErrors = @()
    )
    
    # Defensive check - make sure NewErrors is not null and is an array
    if ($null -eq $NewErrors) {
        $NewErrors = @()
        Write-Log "No new errors to update (null converted to empty array)" -Level "WARNING"
    }
    
    # If NewErrors is an empty array or empty collection, log warning but continue
    if ($NewErrors.Count -eq 0) {
        Write-Log "No events found or events returned empty collection - using empty array" -Level "WARNING"
    }
    
    $now = Get-Date
    $cutoffTime = $now.AddMinutes(-$TimeWindowMinutes)
    
    # Remove errors older than the time window
    $updatedErrors = @()
    $invalidEntries = 0
    $oldEntries = 0
    
    # Has issue when running in Scheduled Task, but No issue when running directly
    #foreach ($error in $State.Errors) {
    #    $errorTime = $null
    #    if ([DateTime]::TryParse($error.TimeCreated, [ref]$errorTime)) {
    #        if ($errorTime -gt $cutoffTime) {
    #            $updatedErrors += $error
    #        }
    #        else {
    #            $oldEntries++
    #        }
    #    }
    #    else {
    #        $invalidEntries++
    #    }
    #}

    foreach ($error in $State.Errors) {
        try {
            $errorTime = [DateTime]::Parse($error.TimeCreated)
            if ($errorTime -gt $cutoffTime) {
                $updatedErrors += $error
            }
            else {
                $oldEntries++
            }
        }
        catch {
            $invalidEntries++
        }
    }
    
    if ($invalidEntries -gt 0) {
        Write-Log "Found $invalidEntries invalid date entries in error state" -Level "WARNING"
    }
    
    if ($oldEntries -gt 0) {
        Write-Log "Removed $oldEntries old entries from error state" -Level "DEBUG"
    }
    
    # Add new errors (loop only processes if NewErrors has elements)
    foreach ($error in $NewErrors) {
        $errorEntry = @{
            TimeCreated = $error.TimeCreated.ToString('o')
            EventID     = $error.EventID
            Level       = $error.Level
            LogName     = $error.LogName
            Message     = if ($error.Message.Length -gt 500) { $error.Message.Substring(0, 500) + "..." } else { $error.Message }
        }
        
        $updatedErrors += $errorEntry
    }
    
    # Update state
    $State.Errors = $updatedErrors
    
    # Save the updated state
    Save-State -State $State
    
    # Return count of errors in the current time window
    return $updatedErrors.Count
}

#endregion

#region Failover Management

# Function to trigger failover
function Invoke-CompleteFailoverCycle {
    param (
        [Parameter(Mandatory = $false)]
        [string]$Reason = "Automatic failover triggered by COMException threshold"
    )
    
    Write-Log "Triggering complete failover cycle: $Reason" -Level "WARNING"
    
    if (-not (Test-CompleteFailoverScript)) {
        Write-Log "Cannot proceed with failover - CompleteFailoverCycle script not found" -Level "ERROR"
        return $false
    }
    
    # If in test mode, simulate a successful failover
    if ($TestMode) {
        Write-Log "TEST MODE: Simulating failover operation (no actual failover performed)" -Level "WARNING"
        Start-Sleep -Seconds 5  # Simulate some processing time
        
        # Update state with last failover time
        $state = Get-State
        $state.LastFailoverTime = (Get-Date).ToString('o')
        $state.FailoverCount++
        $state.Errors = @()  # Clear errors after successful failover
        Save-State -State $state
        
        Write-Log "TEST MODE: Simulated failover completed successfully" -Level "SUCCESS"
        return $true
    }
    
    try {
        # Create temp file for output
        $tempFile = [System.IO.Path]::GetTempFileName()
        
        # Build powershell command with proper argument escaping
        $scriptCmd = "& '$CompleteFailoverScriptPath'"
        $scriptCmd += " -Env '$Env'"
        
        # Add optional parameters with proper escaping
        if ($dnsServer) {
            $scriptCmd += " -dnsServer '$dnsServer'"
        }
        
        if ($lookupZone) {
            $scriptCmd += " -lookupZone '$lookupZone'"
        }
        
        # Redirect output to a file
        $scriptCmd += " *>&1 | Tee-Object -FilePath '$tempFile'"
        
        # Log the command
        Write-Log "Executing failover: $scriptCmd" -Level "INFO"
        
        # Execute the command
        $result = Invoke-Expression $scriptCmd
        $exitCode = $LASTEXITCODE
        
        # Read the captured output
        $output = Get-Content -Path $tempFile -Raw -ErrorAction SilentlyContinue
        Remove-Item -Path $tempFile -Force -ErrorAction SilentlyContinue
        
        # Log output for debugging
        Write-Log "Failover script output: $output" -Level "DEBUG"
        
        # Check if failover was successful
        $success = $false
        
        if ($exitCode -eq 0) {
            $success = $true
        }
        else {
            # Check for success indicators in the output even if exit code is non-zero
            if ($output -match "Complete failover cycle finished" -or
                $output -match "Script completed successfully" -or
                $output -match "First failover successful:" -or
                $output -match "Second failover successful:") {
                $success = $true
                Write-Log "Failover script reported non-zero exit code but appears successful" -Level "WARNING"
            }
        }
        
        if ($success) {
            Write-Log "Complete failover cycle executed successfully" -Level "SUCCESS"
            
            # Update state with last failover time
            $state = Get-State
            $state.LastFailoverTime = (Get-Date).ToString('o')
            $state.FailoverCount++
            $state.Errors = @()  # Clear errors after successful failover

            $dnsName = if ($Env -eq "Dev") { $DevDnsName } else { $ProdDnsName }

            # Reset TTL back to standard value after successful failover
            if ($state.TTLStatus -eq "Reduced") {
                Write-Log "Resetting TTL back to standard value ($DefaultTTLMinutes minutes) for $dnsName" -Level "INFO"
                if (Update-DnsTTL -DnsServer $dnsServer -LookupZone $lookupZone -DnsName $dnsName -TTLMinutes $DefaultTTLMinutes) {
                    $state.TTLStatus = "Standard"
                    $state.LastTTLChange = (Get-Date).ToString('o')
                    Write-Log "TTL reset to standard value successfully" -Level "SUCCESS"
                }
            }

            Save-State -State $state
            
            return $true
        }
        else {
            Write-Log "Complete failover cycle failed with exit code: $exitCode" -Level "ERROR"
            return $false
        }
    }
    catch {
        Write-Log "Error executing complete failover cycle: $_" -Level "ERROR"
        return $false
    }
}

#endregion

#region Setup and Monitoring

# Function to initialize the monitoring environment
function Initialize-Monitoring {
    Write-Log "Initializing Auto Failover Monitoring environment" -Level "INFO"
    
    # Validate the CompleteFailoverCycle script exists
    if (-not (Test-CompleteFailoverScript)) {
        Write-Log "Initialization failed - CompleteFailoverCycle script not found" -Level "ERROR"
        return $false
    }
    
    # Initialize state file
    $state = Get-State
    Write-Log "State file initialized at: $stateFilePath" -Level "INFO"
    
    # Create scheduled task for periodic monitoring if not running as service
    if (-not $RunAsService) {
        try {
            # Check if scheduled task already exists
            $taskName = "AutoFailoverMonitor_$Env"
            $task = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
            
            if ($task) {
                Write-Log "Scheduled task '$taskName' already exists" -Level "INFO"
            }
            else {
                # Create a scheduled task to run every 5 minutes
                $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument "-NoProfile -ExecutionPolicy Bypass -File `"$PSCommandPath`" -Env `"$Env`""
                $trigger = New-ScheduledTaskTrigger -Once -At (Get-Date) -RepetitionInterval (New-TimeSpan -Minutes 5)
                $settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 10) -RestartCount 3
                
                Register-ScheduledTask -TaskName $taskName -Action $action -Trigger $trigger -Settings $settings -Description "Automatic Failover Monitor for $Env environment"
                
                Write-Log "Scheduled task '$taskName' created to run every 5 minutes" -Level "SUCCESS"
            }
        }
        catch {
            Write-Log "Error creating scheduled task: $_" -Level "WARNING"
            Write-Log "You may need to manually create a scheduled task to run this script periodically" -Level "WARNING"
        }
    }
    
    Write-Log "Initialization completed successfully" -Level "SUCCESS"
    return $true
}

# Function to run the monitor once
function Invoke-MonitoringCheck {
    Write-Log "Starting monitoring check for COMException errors" -Level "INFO"
    
    $dnsName = if ($Env -eq "Dev") { $DevDnsName } else { $ProdDnsName }
    Write-Log "Using DNS name for $Env environment: $dnsName" -Level "DEBUG"

    # Get current active server
    $activeServer = Get-CurrentActiveServer
    if (-not $activeServer) {
        Write-Log "Cannot proceed without determining active server" -Level "ERROR"
        return $false
    }
    
    # Extract server name without domain
    $serverName = Get-HostnameFromFQDN -FQDN $activeServer
    Write-Log "Active server short name: $serverName" -Level "INFO"
    
    # Get current state
    $state = Get-State
    
    # Check if we're in a cooldown period (unless force failover is specified)
    if (-not $ForceFailover -and (Test-CooldownPeriod -State $state)) {
        Write-Log "Skipping error check due to cooldown period" -Level "INFO"
        return $true
    }
    
    # Check for COMException events on the active server
    $events = Get-COMExceptionEvents -ComputerName $serverName -MinutesAgo $TimeWindowMinutes
    
    # Update error state with new events 
    $errorCount = Update-ErrorState -State $state -NewErrors $events
    
    Write-Log "Current COMException error count: $errorCount/$ErrorThreshold in last $TimeWindowMinutes minutes" -Level "INFO"

    # Add TTL Management - When first error is detected (or force failover), reduce TTL to 1 minute
    if ( ($errorCount -ge 1 -and $state.TTLStatus -eq "Standard") -or $ForceFailover) {
        Write-Log "First COM error detected (or force failover), proactively reducing TTL" -Level "WARNING"
        if (Update-DnsTTL -DnsServer $dnsServer -LookupZone $lookupZone -DnsName $dnsName -TTLMinutes $ReducedTTLMinutes) {
            $state.TTLStatus = "Reduced"
            $state.LastTTLChange = (Get-Date).ToString('o')
            Save-State -State $state
            Write-Log "TTL reduced to $ReducedTTLMinutes minute(s) successfully" -Level "SUCCESS"
        }
    }
    
    # Check if threshold is exceeded or force failover is specified
    if ($errorCount -ge $ErrorThreshold -or $ForceFailover) {
        $reason = if ($ForceFailover) {
            "Forced failover requested"
        }
        else {
            "COMException threshold reached ($errorCount events in $TimeWindowMinutes minutes)"
        }
        
        # Trigger failover
        $result = Invoke-CompleteFailoverCycle -Reason $reason
        
        if ($result) {
            Write-Log "Automatic failover successfully completed" -Level "SUCCESS"
        }
        else {
            Write-Log "Automatic failover failed" -Level "ERROR"
        }
        
        return $result
    }
    else {
        Write-Log "Error threshold not reached, no action needed" -Level "INFO"
        return $true
    }
}

# Function to run continuous monitoring as a service
function Start-MonitoringService {
    Write-Log "Starting Auto Failover Monitoring service" -Level "INFO"
    
    # Validate script dependencies before entering loop
    if (-not (Test-CompleteFailoverScript)) {
        Write-Log "Cannot start monitoring service - CompleteFailoverCycle script not found" -Level "ERROR"
        return $false
    }
    
    try {
        # Initial check to get and validate active server
        $activeServer = Get-CurrentActiveServer
        if (-not $activeServer) {
            Write-Log "Cannot start monitoring service - unable to determine active server" -Level "ERROR"
            return $false
        }
        
        # Initialize failover counter
        $failoverAttempts = 0
        Write-Log "Maximum failover attempts set to: $MaxFailoverAttempts per 24-hour period" -Level "INFO"
        $failoverCountResetTime = (Get-Date).AddHours(24)
        
        Write-Log "Monitoring service started successfully" -Level "SUCCESS"
        
        # Main monitoring loop
        while ($true) {
            try {
                # Rotate log file if needed
                Rotate-LogFile -LogPath $logFilePath
                
                # Check if we need to reset the failover counter
                $now = Get-Date
                if ($now -gt $failoverCountResetTime) {
                    $failoverAttempts = 0
                    $failoverCountResetTime = $now.AddHours(24)
                    Write-Log "Failover attempt counter reset" -Level "INFO"
                }
                
                # Run monitoring check if we haven't exceeded the maximum failover attempts
                if ($failoverAttempts -lt $maxFailoverAttempts) {
                    $result = Invoke-MonitoringCheck
                    
                    # If a failover was triggered and successful, increment the counter
                    $state = Get-State
                    if ($state.LastFailoverTime) {
                        $lastFailoverTime = [DateTime]::Parse($state.LastFailoverTime)
                        if ($lastFailoverTime -gt $now.AddMinutes(-10)) {
                            $failoverAttempts++
                            Write-Log "Failover attempt count: $failoverAttempts/$maxFailoverAttempts in current 24-hour period" -Level "WARNING"
                        }
                    }
                }
                else {
                    Write-Log "Maximum failover attempts ($maxFailoverAttempts) reached for 24-hour period. Monitoring continues but no failovers will be triggered." -Level "WARNING"
                }
                
                # Sleep for 5 minutes between checks
                Write-Log "Sleeping for 5 minutes before next check..." -Level "INFO"
                Start-Sleep -Seconds 300
            }
            catch {
                Write-Log "Error in monitoring cycle: $_" -Level "ERROR"
                Write-Log "Continuing with next check in 5 minutes..." -Level "WARNING"
                Start-Sleep -Seconds 300
            }
        }
    }
    catch {
        Write-Log "Critical error in monitoring service: $_" -Level "ERROR"
        Write-Log "Monitoring service stopped" -Level "ERROR"
        return $false
    }
}

#endregion

# Main execution block
try {
    Write-Log "=======================================================" -Level "INFO"
    Write-Log "Auto Failover Monitor v1.0" -Level "INFO"
    Write-Log "Environment: $Env, Error Threshold: $ErrorThreshold/$TimeWindowMinutes min, Cooldown: $CooldownPeriodMinutes min" -Level "INFO"
    Write-Log "=======================================================" -Level "INFO"
    
    # Handle different execution modes
    if ($Initialize) {
        Initialize-Monitoring
    }
    elseif ($RunAsService) {
        Start-MonitoringService
    }
    elseif ($ForceFailover) {
        Write-Log "Force failover mode enabled - will trigger failover regardless of error state" -Level "WARNING"
        Invoke-MonitoringCheck
    }
    elseif ($TestMode) {
        Write-Log "Test mode enabled - no actual failover will be performed" -Level "WARNING"
        Invoke-MonitoringCheck
    }
    else {
        # Run a single monitoring check
        Invoke-MonitoringCheck
    }
    
    exit 0
}
catch {
    Write-Log "Unhandled exception in Auto Failover Monitor: $_" -Level "ERROR"
    Write-Log $_.ScriptStackTrace -Level "ERROR"
    exit 1
}