Linux System Performance Monitoring

system_performance_monitoring.sh

# Poll for various system stats for monitoring system
# Just copy and paste the commands into a bash prompt
# Use Ctrl-C to exit the polling


# Simple Example (CPU/Load/Memory)

# Settings
INTERVAL_DURATION="5" # seconds to average across
TIME_PAD='                      '
PADDING='        '

# Header info
CUR_TIME='timestamp'
CPU_USAGE="cpu%"
CPU_60S_LOAD="load"
MEM_USAGE="mem%"

# Initialize Stats
CUR_PROC_STAT=$(grep 'cpu ' /proc/stat)

ECHO_HEADER=1
while true; do
  [ ${ECHO_HEADER} -eq 1 ] && ECHO_HEADER=0 && printf "%s %s %s %s %s %s %s %s\n" ${CUR_TIME} "${TIME_PAD:${#CUR_TIME}}" ${CPU_USAGE} "${PADDING:${#CPU_USAGE}}" ${CPU_60S_LOAD} "${PADDING:${#CPU_60S_LOAD}}" ${MEM_USAGE} "${PADDING:${#MEM_USAGE}}"
  CUR_TIME=$(date '+%Y-%m-%d %H:%M:%S')
  LAST_PROC_STAT=${CUR_PROC_STAT}
  CUR_PROC_STAT=$(grep 'cpu ' /proc/stat)
  CPU_USAGE=$(cat <(echo "${LAST_PROC_STAT}") <(echo "${CUR_PROC_STAT}") | awk -v RS="" '{printf "%.2f%%", ($13-$2+$15-$4)*100/($13-$2+$15-$4+$16-$5) }')
  CPU_60S_LOAD=$(cat /proc/loadavg | cut -d" " -f1)
  MEM_USAGE=$(free -m | awk 'NR==2{printf "%.2f%%", $3*100/$2 }')
  printf "%s %s %s %s %s %s %s %s" ${CUR_TIME} "${TIME_PAD:${#CUR_TIME}}" ${CPU_USAGE} "${PADDING:${#CPU_USAGE}}" ${CPU_60S_LOAD} "${PADDING:${#CPU_60S_LOAD}}" ${MEM_USAGE} "${PADDING:${#MEM_USAGE}}"
  printf "\n"
  sleep ${INTERVAL_DURATION}
done

# Example Output
#timestamp               cpu%      load      mem%     
#2020-02-21 16:55:11     100.00%   8.44      88.57%         
#2020-02-21 16:55:16     96.82%    8.40      77.59%         
#2020-02-21 16:55:21     2.20%     7.73      76.15%         
#2020-02-21 16:55:26     0.65%     7.11      76.15%         
#2020-02-21 16:55:31     0.58%     6.54      76.15%   






# Output:
# Timestamp
# CPU Load Average (1 minute average)
# Memory Utilization
# PHP-FPM Active Connections
# Nginx Active Connections

# Variables
NGINX_URL="https://www.example.com/nginx_status/"
NGINX_CURL_RESOLVE="www.example.com:443:127.0.0.1"
PHP_FPM_URL="http://localhost:8080/php-status?json"

TIME_PAD='                      '
PADDING='        '

# Header info
CUR_TIME='timestamp'
CPU_USAGE="cpu%"
CPU_60S_LOAD="load"
MEM_USAGE="mem%"
PHP_FPM_ACTIVE_PROCESSES="php-fpm"
NGINX_ACTIVE_CONNECTIONS="nginx"

CUR_PROC_STAT=$(grep 'cpu ' /proc/stat)

ECHO_HEADER=1
while true; do
  [ ${ECHO_HEADER} -eq 1 ] && ECHO_HEADER=0 && printf "%s %s %s %s %s %s %s %s %s %s %s %s\n" ${CUR_TIME} "${TIME_PAD:${#CUR_TIME}}" ${CPU_USAGE} "${PADDING:${#CPU_USAGE}}" ${CPU_60S_LOAD} "${PADDING:${#CPU_60S_LOAD}}" ${MEM_USAGE} "${PADDING:${#MEM_USAGE}}" ${PHP_FPM_ACTIVE_PROCESSES} "${PADDING:${#PHP_FPM_ACTIVE_PROCESSES}}" ${NGINX_ACTIVE_CONNECTIONS} "${PADDING:${#NGINX_ACTIVE_CONNECTIONS}}"
  CUR_TIME=$(date '+%Y-%m-%d %H:%M:%S')
  LAST_PROC_STAT=${CUR_PROC_STAT}
  CUR_PROC_STAT=$(grep 'cpu ' /proc/stat)
  CPU_USAGE=$(cat <(echo "${LAST_PROC_STAT}") <(echo "${CUR_PROC_STAT}") | awk -v RS="" '{printf "%.2f%%", ($13-$2+$15-$4)*100/($13-$2+$15-$4+$16-$5) }')
  CPU_60S_LOAD=$(cat /proc/loadavg | cut -d" " -f1)
  MEM_USAGE=$(free -m | awk 'NR==2{printf "%.2f%%", $3*100/$2 }')
  PHP_FPM_STATUS=$(curl -s ${PHP_FPM_URL})
  PHP_FPM_ACTIVE_PROCESSES=$(echo "${PHP_FPM_STATUS}" | jq '."active processes"')
  NGINX_STATUS=$(curl -s --resolve "${NGINX_CURL_RESOLVE}" ${NGINX_URL})
  NGINX_ACTIVE_CONNECTIONS=$(echo "${NGINX_STATUS}" | grep "Active connections: " | cut -d":" -f2)
  NGINX_ACTIVE_CONNECTIONS="${NGINX_ACTIVE_CONNECTIONS##*( )}"
  printf "%s %s %s %s %s %s %s %s %s %s %s %s" ${CUR_TIME} "${TIME_PAD:${#CUR_TIME}}" ${CPU_USAGE} "${PADDING:${#CPU_USAGE}}" ${CPU_60S_LOAD} "${PADDING:${#CPU_60S_LOAD}}" ${MEM_USAGE} "${PADDING:${#MEM_USAGE}}" ${PHP_FPM_ACTIVE_PROCESSES} "${PADDING:${#PHP_FPM_ACTIVE_PROCESSES}}" ${NGINX_ACTIVE_CONNECTIONS} "${PADDING:${#NGINX_ACTIVE_CONNECTIONS}}"
  printf "\n"
  sleep 5
done



# On a physical/dedicated server you may be able to get IPC metrics to determine stalled CPU time.

# http://www.brendangregg.com/blog/2017-05-09/cpu-utilization-is-wrong.html
# http://www.brendangregg.com/perf.html
# https://github.com/brendangregg/perf-tools
yum install perf





# CPU Usage Info
# https://askubuntu.com/questions/274349/getting-cpu-usage-realtime
# https://unix.stackexchange.com/questions/69185/getting-cpu-usage-same-every-time/69194#69194

# Print Number of Processing Units
# used to show the number of processing unit present on your computer physical cores + hyper threads
nproc

# Shows CPU Architecture Info
# prints CPU architecture information from sysfs and /proc/cpuinfo
lscpu


# sar Info
# https://www.cyberciti.biz/tips/how-do-i-find-out-linux-cpu-utilization.html

# Display todays CPU usage stats for every minute since midnight
sar

# display cpu load averages (1m 5m 15m)
cat /proc/loadavg

# Poll every 60 seconds 80 times (60 seconds * 80 attempts = 80 minutes)
# CPU Stats
sar -u 60 80

# Load Average Stats
sar -q 60 80

# Memory Stats
sar -r 60 80

# Network Interface Stats
sar -n DEV 60


# For additional Network Stats you can use bmon for monitoring network utilization real-time
yum install bmon
# Passing the -b flag will report stats in bits/s instead of bytes
bmon -b

# Found a simple script to report network throughput in more of a log form calculating form /proc/net/dev
# https://gist.github.com/dagelf/ab2bad26ce96fa8d79b0834cd8cab549
# Saving the contents to something like ~/n.sh you can call it like this:
~/n.sh em1

# Keep in mind this reports in bytes/s rather than bit/s though the last line could be modified like this:
# printf "In: %12i Mb/s | Out: %12i Mb/s | Total: %12i Mb/s\n" $(($INSPEED/1024/1024*8)) $(($OUTSPEED/1024/1024*8)) $((($INSPEED+$OUTSPEED)/1024/1024*8)) ;
# My forked version of the script: https://gist.github.com/mttjohnson/3cadbdcca8832f60ed9546ad38af4b10