Digital Ocean WP Droplet Server Auto-Recovery Script

gistfile1.txt

#!/bin/bash

# ================================================================
# WordPress Server Auto-Recovery Script
# ================================================================
# Automatically monitors and recovers Apache web server on low-memory VPS
# Includes email alerts via Resend API
# 
# Author: Miguel Garrido (https://miguel.nz)
# Usage: Run via cron every 5 minutes: */5 * * * * /usr/local/bin/service-monitor.sh
# ================================================================

# Configuration
LOG_FILE="/var/log/service-monitor.log"
ALERT_SENT_FILE="/var/log/service-monitor-alert-sent"

# Email Configuration (Update these with your details)
RESEND_API_KEY="your_resend_api_key_here"  # Get from https://resend.com
FROM_EMAIL="[email protected]"
TO_EMAIL="[email protected]"
SERVER_NAME="your-server-name"

# Monitoring Configuration
WEBSITE_URL="http://localhost"  # Change to your domain if needed
MEMORY_THRESHOLD=85  # Alert when memory usage exceeds this percentage

# ================================================================
# FUNCTIONS
# ================================================================

# Function to log messages with timestamp
log_message() {
    echo "$(date): $1" >> $LOG_FILE
}

# Function to send email alert via Resend API
send_alert() {
    local subject="$1"
    local message="$2"
    
    if [ -z "$RESEND_API_KEY" ] || [ "$RESEND_API_KEY" = "your_resend_api_key_here" ]; then
        log_message "Email alert skipped (API key not configured): $subject"
        return
    fi
    
    curl -s -X POST 'https://api.resend.com/emails' \
        -H "Authorization: Bearer $RESEND_API_KEY" \
        -H 'Content-Type: application/json' \
        -d "{
            \"from\": \"$FROM_EMAIL\",
            \"to\": \"$TO_EMAIL\",
            \"subject\": \"🚨 Server Alert: $subject\",
            \"html\": \"<h2>Server Alert</h2><p>$message</p><p><strong>Server:</strong> $SERVER_NAME</p><p><strong>Time:</strong> $(date)</p><hr><p><small>Auto-generated by WordPress Server Monitor</small></p>\"
        }" > /dev/null 2>&1
        
    log_message "Email alert sent: $subject"
}

# Function to check if we already sent an alert for this issue
alert_already_sent() {
    [ -f "$ALERT_SENT_FILE" ]
}

# Function to mark alert as sent
mark_alert_sent() {
    touch "$ALERT_SENT_FILE"
}

# Function to clear alert flag when issue is resolved
clear_alert_flag() {
    rm -f "$ALERT_SENT_FILE"
}

# ================================================================
# MONITORING CHECKS
# ================================================================

# Check if Apache is running
check_apache() {
    if ! systemctl is-active --quiet apache2; then
        log_message "ALERT: Apache is down, attempting to restart..."
        
        if ! alert_already_sent; then
            send_alert "Apache Down" "Apache web server has crashed and is being restarted automatically."
            mark_alert_sent
        fi
        
        # Reset failed state and restart
        systemctl reset-failed apache2
        systemctl start apache2
        sleep 5
        
        if systemctl is-active --quiet apache2; then
            log_message "SUCCESS: Apache restarted successfully"
            send_alert "Apache Restored" "Apache web server has been successfully restarted and is now running."
            clear_alert_flag
        else
            log_message "ERROR: Failed to restart Apache"
            send_alert "Apache Restart Failed" "URGENT: Apache failed to restart. Manual intervention required!"
        fi
    else
        # Apache is running, clear any previous alert flags
        clear_alert_flag
    fi
}

# Check if website responds to HTTP requests
check_website_response() {
    if ! curl -f -s -m 10 "$WEBSITE_URL" > /dev/null 2>&1; then
        log_message "ALERT: Website not responding, restarting Apache"
        
        if ! alert_already_sent; then
            send_alert "Website Not Responding" "Website is not responding to HTTP requests. Apache is being restarted automatically."
            mark_alert_sent
        fi
        
        systemctl restart apache2
        sleep 5
        
        if curl -f -s -m 10 "$WEBSITE_URL" > /dev/null 2>&1; then
            log_message "SUCCESS: Website restored after Apache restart"
            send_alert "Website Restored" "Website is now responding after Apache restart."
            clear_alert_flag
        else
            log_message "ERROR: Website still not responding after restart"
            send_alert "Website Still Down" "URGENT: Website still not responding after Apache restart. Manual intervention required!"
        fi
    fi
}

# Check memory usage and alert if high
check_memory_usage() {
    MEMORY_USAGE=$(free | awk 'FNR==2{printf "%.0f", $3/($3+$4)*100}')
    
    if [ "$MEMORY_USAGE" -gt $MEMORY_THRESHOLD ]; then
        log_message "WARNING: High memory usage: ${MEMORY_USAGE}%"
        
        if ! alert_already_sent; then
            send_alert "High Memory Usage" "Server memory usage is at ${MEMORY_USAGE}%. Monitor closely to prevent OOM crashes."
            mark_alert_sent
        fi
    elif [ "$MEMORY_USAGE" -lt $((MEMORY_THRESHOLD - 10)) ]; then
        # Clear memory alert flag if usage drops significantly
        clear_alert_flag
    fi
}

# ================================================================
# MAIN EXECUTION
# ================================================================

# Run all monitoring checks
check_apache
check_website_response
check_memory_usage

# Log completion with current status
MEMORY_USAGE=$(free | awk 'FNR==2{printf "%.0f", $3/($3+$4)*100}')
APACHE_STATUS=$(systemctl is-active apache2)

log_message "Monitoring check completed - Apache: $APACHE_STATUS, Memory: ${MEMORY_USAGE}%"

# Optional: Clean up old log entries (keep last 1000 lines)
if [ -f "$LOG_FILE" ] && [ $(wc -l < "$LOG_FILE") -gt 1000 ]; then
    tail -n 1000 "$LOG_FILE" > "${LOG_FILE}.tmp" && mv "${LOG_FILE}.tmp" "$LOG_FILE"
fi