lamp-diag.sh

lamp-diag.sh

#!/bin/bash

## diagnose http requests from this container
## if the response timed out, create a diagnostic report, and email it

readonly NOW=$(date +%Y.%m.%d-%H:%M:%S)
reasons="Test .."
email="[email protected]"


## The timer will run this script every ten minutes
function install {

cat > /etc/systemd/system/lamp-diag.service << EOF
[Unit]
Description=lamp-diag

[Service]
Type=oneshot
ExecStart=/root/lamp-diag.sh
WorkingDirectory=/root
EOF

cat > /etc/systemd/system/lamp-diag.timer << EOF
[Unit]
Description=lamp-diag

[Timer]
OnCalendar=*:0/10
Persistent=true

[Install]
WantedBy=timers.target
EOF

systemctl enable lamp-diag.timer
systemctl start lamp-diag.timer

}

## log the last 100 lines of each file
function log_a_file() {
    file="$1"
    name="file${file//\//-}"

    if [[ -f $file ]]
    then
        echo "cat $file | tail -n 100 2>&1 >> $logdir/$name"
        cat $file | tail -n 10000 2>&1 >> "$logdir/$name"
    fi
}

## list files in a directory, that end with log
function log_a_dir {
    dir=$1

    for file in $(ls $dir)
    do
        if [[ $file == *log ]]
        then
            echo "$file"
            log_a_file "$dir/$file"
        fi
    done
}

## systemd services
function log_service {
    service=$1
    journalctl -u "$service" --since yesterday --no-pager 2>&1 > "$logdir/journal-$service.log"
    systemctl status "$service" --no-pager -n 100 2>&1 > "$logdir"/status-$service.log

}

## if the curl command fails, send the diagnostic report
function curl_timeout {
    seconds=$1
    if curl -k --max-time "$seconds" --connect-timeout "$seconds" "https://localhost" 2>&1 >> "/tmp/curl-localhost.html"
    then
        echo "Curl with $seconds seconds limit OK"
    else
        echo "curl -k --max-time $seconds --connect-timeout $seconds https://localhost - failed with exit code $?"
        reasons="$reasons curl $HOSTNAME timed out (with connect-timeout and max-time of $seconds seconds.)"
        send_report
    fi
}


function send_report {
    echo "Generating Error report"

    logdir=/error-report-$HOSTNAME-$NOW
    mkdir -p "$logdir"
    rm -fr "$logdir"/*


    ## top is observed for ten cycles
    for i in {1..30}
    do
        echo "top $i"
        echo "===============================" >> "$logdir/top.log"
        echo "top $i @ $(date +%Y.%m.%d-%H:%M:%S)" >> "$logdir/top.log"
        echo "===============================" >> "$logdir/top.log"
        top -n 1 -w 120 2>&1 >> "$logdir/top.log"
        echo "" >> "$logdir/top.log"
        echo "" >> "$logdir/top.log"
        echo "" >> "$logdir/top.log"

        systemd-cgtop -b -m --iterations=1 2>&1 >> "$logdir/systemd-cgtop.log"
        echo "" >> "$logdir/systemd-cgtop.log"

        sleep 3

    done


    du -h /var 2>&1 >> "$logdir/disk-usage.log"
    du -hs / 2>&1 >> "$logdir/disk-usage.log"

    systemctl status --no-pager 2>&1 > "$logdir/systemctl-status.log"
    systemctl list-unit-files --no-pager 2>&1 > "$logdir/systemctl-list-unit-files.log"

    log_a_dir /var/log/php-fpm
    log_a_dir /var/log/mariadb
    log_a_dir /var/log/httpd

    log_service httpd
    log_service php-fpm
    log_service mariadb


    echo "$reasons"

    echo "Sending Error report"

    zipfile="/tmp/$HOSTNAME-$NOW.zip"
    zip -r "$zipfile" "$logdir"

    if echo "$reasons \n See attachments for details" | mail -v -s "$HOSTNAME report $NOW" -a "$zipfile" $email
    then
        echo "Sent, rebooting."
        reboot
    else
        echo "Sending failed."
    fi

    exit 1
}



## install section
if [[ $1 == "--install" ]]
then
    echo "Installing the timer and service files"
    install
    exit
fi

curl_timeout 60
curl_timeout 10
# curl_timeout 5

echo "All went well, exiting."
exit 0