Shows differences between two hosts. Very helpful with diagnosing why something works on one host, but not on the other.

diff-hosts.sh

#!/bin/bash

if [ $# -lt 3 ]; then
    echo "not enough arguments"
    echo "usage: $0 <host> <host> <module|modules|all>"
    exit 1
fi

COLUMNS=$(tput cols)
HOSTA="$1"
HOSTB="$2"

diff_vars() {
    printf -v VARA "%s\n%s" "$HOSTA" "$1"
    printf -v VARB "%s\n%s" "$HOSTB" "$2"
    diff --color \
      --expand-tabs \
      --side-by-side \
      --suppress-common-lines \
      --width="$COLUMNS" \
      <(echo "$VARA") <(echo "$VARB")
}

diff_apt() {
    echo "### APT"
    OUT_HOSTA=$(ssh "$HOSTA" apt list --installed 2>/dev/null)
    OUT_HOSTB=$(ssh "$HOSTB" apt list --installed 2>/dev/null)
    diff_vars "$OUT_HOSTA" "$OUT_HOSTB"
}

diff_etc() {
    echo "### /etc"
    OUT_HOSTA=$(ssh "$HOSTA" "sudo find /etc -type f | sort | sudo xargs md5sum" | awk '{ print $2 "\t" $1 }')
    OUT_HOSTB=$(ssh "$HOSTB" "sudo find /etc -type f | sort | sudo xargs md5sum" | awk '{ print $2 "\t" $1 }')
    diff_vars "$OUT_HOSTA" "$OUT_HOSTB"
}

diff_file() {
    if [[ -z "$1" ]]; then
        echo "file: path missing"
        exit 1
    fi
    echo "### $1"
    OUT_HOSTA=$(ssh "$HOSTA" sudo cat "$1")
    OUT_HOSTB=$(ssh "$HOSTB" sudo cat "$1")
    diff_vars "$OUT_HOSTA" "$OUT_HOSTB"
}

diff_kernel() {
    echo "### Kernel"
    OUT_HOSTA=$(ssh "$HOSTA" uname -r)
    OUT_HOSTB=$(ssh "$HOSTB" uname -r)
    OUT_HOSTA+=$(ssh "$HOSTA" cat /proc/cmdline)
    OUT_HOSTB+=$(ssh "$HOSTB" cat /proc/cmdline)
    diff_vars "$OUT_HOSTA" "$OUT_HOSTB"
}

case "$3" in
    apt)
    diff_apt
    ;;
    etc)
    diff_etc
    ;;
    file)
    diff_file "$4"
    ;;
    kernel)
    diff_kernel
    ;;
    all)
    diff_apt
    diff_etc
    diff_kernel
    ;;
    modules)
    echo "available modules:"
    echo "  apt          list installed packages"
    echo "  etc          list differing files in /etc"
    echo "  file <path>  diffs file with <path>"
    echo "  kernel       check for differing kernel version and command line"
    echo
    echo "  all          runs apt, etc, kernel"
    ;;
    *)
    echo "unknown module: $3"
    ;;
esac