kdumpconfig.sh

kdumpconfig.sh

#!/bin/bash

# ===============================================================
# The primary goal of this script is to set the right crashkernel
# parameter as per the "system RAM and arch", in the BLS
# configuration file and setup kdump.conf (as per user selection
# while generating this script) in _ONE-GO_.
# Extra debugging options too will be enabled as per
# user's choice.
#
# Though this script checks most of the things and warns about
# errors/blunders, rest of the actions are to be performed by the
# end-user (initiating this script) by observing the messages/
# warnings as they appear.
#
# This script is designed & tested to work on the following archs:
#
# - x86_64
# - ppc64le
# - s390x
# - aarch64
# ===============================================================

echo ============================================================================================
echo Kdump Helper is starting to configure kdump service in this system.[For all present kernels]
echo ============================================================================================

if ! rpm -q kexec-tools > /dev/null; then
    echo -e "\nkexec-tools not found, run the following command to install: dnf install kexec-tools"
    exit 1
fi

kdump_mask=$(systemctl is-enabled kdump)
if [ $kdump_mask == "masked" ]; then
   echo "kdump service is masked on the system."
   echo "If you still want to change kdump configuration, run the following command: systemctl unmask kdump"
   echo "Then run the script again."
   exit -1
fi

arch=$(uname -m)
echo -e "\n-System arch: "$arch"."

mem_total=$(free -g |awk 'NR==2 {print $2 }')
mem_display=$(free -m |awk 'NR==2 {print $2 }')
calc=$(awk "BEGIN {printf \"%.2f\n\", ($mem_display*2^20)/2^30}")

if [ $mem_total -lt 1 ]; then
    echo -e "\n-Total memory is less than 1G."
else
    echo -e "\n-Total memory is $calc G."
fi

# Before making any changes backup the existing grub2 BLS drop-in file of the kernel(s).
# Which remains the same across UEFI and BIOS boots and decides on the kernel cmdline.

grub_conf=/boot/loader/entries/$(cat /etc/machine-id)-*.conf
directory=/boot/loader/entries/kdumphelper_backup

function backup() {
    cp $grub_conf /boot/loader/entries/kdumphelper_backup
    
    for filename in /boot/loader/entries/kdumphelper_backup/*.conf; do
        mv "$filename" "$filename.kdumphelper.$(date +%y-%m-%d-%H_%M_%S)";
    done
    
    echo -e "\n-Backup stored under:"
    ls -lAh $directory/*
}

if [ -d "$directory" ] && ls $directory/*kdumphelper* &> /dev/null; then
    echo -e "\n-Backup directory ($directory) is already present with some existing backup file(s).\nRemoving old backup and initiating a fresh one."
    rm -rf $directory/*
    echo -e "\n-Initiate a backup of grub2 BLS configuration file(s)..."
    backup
else
    mkdir -p $directory &> /dev/null
    echo -e "\n-Initiate a backup of grub2 BLS configuration file(s)..."
    backup
fi

function grub_add_entry() {
    echo -e "\n'crashkernel=$crashkernel_para' is set in:\n`ls $grub_conf`"
    grubby --update-kernel=ALL --args=crashkernel=$crashkernel_para
}

function grub_add_entry390() {
    echo -e "\n'crashkernel=$crashkernel_para' is set in:\n`ls $grub_conf`"
    grubby --update-kernel=ALL --args=crashkernel=$crashkernel_para
    echo ======================================================================
    echo "zipl:updating the recently modified file(s) to reflect upon next boot"
    echo ======================================================================
    zipl
}

#add entry for crashkernel for all the kernels present in the system
if [ $arch == x86_64 ]; then
    crashkernel_para="auto"
    grub_add_entry
elif [ $arch == s390x ]; then
    crashkernel_para="auto"
    grub_add_entry390
else
    crashkernel_para="auto"
    grub_add_entry
fi

# Check for presence of kdump.conf and back it up
kdump_conf=/etc/kdump.conf
kdump_conf_kdumphelper=/etc/kdump.conf.kdumphelper.$(date +%y-%m-%d-%H:%M:%S)

if [ -s $kdump_conf ]; then
    echo -e "\n-Back up $kdump_conf to $kdump_conf_kdumphelper."
    cp $kdump_conf $kdump_conf_kdumphelper
else
    echo -e "\e[1;31m\n-'/etc/kdump.conf' is either empty or does not exist.\nPlease re-install kexec-tools.\nExiting.\e[0m"
    exit 1;
fi

dump_path=/var/crash
dump_level=23

echo core_collector makedumpfile -c --message-level 7 -d $dump_level > $kdump_conf
echo 'default reboot' >>  $kdump_conf

# Check if the dump directory is mounted
dump_dev_name=$(df --output=source "$dump_path" | tail -1)
dump_dev_uuid=$(blkid "$dump_dev_name" | awk '{print $2}' | tr -d '"')
dump_fs_type=$(df --output=fstype "$dump_path" | tail -1)
mount_point=$(df --output=target "$dump_path" | tail -1)
if [ "$mount_point" = "/" ]; then
    echo "==== The dump directory is not mounted to a separate device. Your vmcore will be saved in the root filesystem ===="
    echo "path $dump_path" >> "$kdump_conf"
elif [ "$mount_point" = "$dump_path" ]; then
    echo "path /" >> "$kdump_conf"
    echo $dump_fs_type $dump_dev_uuid >> "$kdump_conf"
    cat /etc/fstab | awk '{print $1}' | grep -E "^$dump_dev_name|^$dump_dev_uuid" >> /dev/null
    if [ $? -ne 0 ]; then
        echo "==== You need to add an entry in the /etc/fstab to make sure the dump directory is auto-mounted after system reboot. ===="
        echo "==== See: https://access.redhat.com/solutions/1197493 ===="
    fi
else
    echo "==== The dump directory is mounted to a separate device with nested path, Your vmcore will be dumped to that path  ===="
    _path=${dump_path#$mount_point} 
    echo "path $_path" >> "$kdump_conf"
    echo $dump_fs_type $dump_dev_uuid >> "$kdump_conf"
    cat /etc/fstab | awk '{print $1}' | grep -E "^$dump_dev_name|^$dump_dev_uuid" >> /dev/null
    if [ $? -ne 0 ]; then
        echo "==== You need to add an entry in the /etc/fstab to make sure the dump directory is auto-mounted after system reboot. ===="
        echo "==== See: https://access.redhat.com/solutions/1197493 ===="
    fi
fi

# Enable kdump service and warn according to the detected error
echo -e "\n-Enabling the kdump service..."
systemctl enable kdump.service
systemctl -a | grep kdump

echo -e "\n-Restarting the kdump service..."
kdumpctl restart
if [ $? != 0 ]; then
    mem_reserved=$(cat /sys/kernel/kexec_crash_size);
    if [ $mem_reserved -eq 0 ]; then
        echo -e "\e[1;31m\nNOTE:A System reboot is mandatory to reserve the memory for crashkernel.\nKindly check the service status post reboot.\e[0m"
    else
        echo -e "\e[1;31m\nWARNING: Service not active. Kindly refer the error message and perform a sanity check.\e[0m"
    fi
else
    :
fi

# Kernel parameter change
echo -e "\n-Starting to Configure extra diagnostic options..."
sysctl_conf=/etc/sysctl.d/99-kdump-config-helper.conf
mkdir -p /etc/sysctl.d
touch $sysctl_conf

#echo -e "\n-Backed up $sysctl_conf to $sysctl_conf_kdumphelper."
#sysctl_conf_kdumphelper=/etc/sysctl.conf.kdumphelper.$(date +%y-%m-%d-%H:%M:%S)
#cp -a $sysctl_conf $sysctl_conf_kdumphelper

# Server hang
sed -i '/^kernel.sysrq/ s/kernel/#kernel/g ' $sysctl_conf 
echo >> $sysctl_conf
echo '# Panic on sysrq and nmi button, magic button alt+printscreen+c or nmi button could be pressed to collect a vmcore' >> $sysctl_conf
echo '# See: ' >> $sysctl_conf 
echo '#   https://access.redhat.com/solutions/2023' >> $sysctl_conf
echo '#   https://access.redhat.com/solutions/125103' >> $sysctl_conf
echo 'kernel.sysrq=1' >> $sysctl_conf
echo 'kernel.unknown_nmi_panic=1' >> $sysctl_conf
echo 'kernel.panic_on_io_nmi=1' >> $sysctl_conf
echo 'kernel.panic_on_unrecovered_nmi=1' >> $sysctl_conf

# Softlockup
sed -i '/^kernel.softlockup_panic/ s/kernel/#kernel/g ' $sysctl_conf 
echo >> $sysctl_conf
echo '# Panic on soft lockups.' >> $sysctl_conf
echo '# See: https://access.redhat.com/solutions/19541' >> $sysctl_conf
echo 'kernel.softlockup_panic=1' >> $sysctl_conf

sysctl --system &> /dev/null