Show performance information about NVMe PCie drives or any PCIe sysfs device, based on https://community.mellanox.com/s/article/understanding-pcie-configuration-for-maximum-performance

pcie-throughput.sh

#!/usr/bin/env bash

# BEGIN: PCIe-related things
_read_attr() {
    local dir="$1" name="$2"
    local path=$(printf "%s/%s" "$dir" "$name")

    cat "$path"
}

_read_device_current_link_width() {
    _read_attr "$1" "current_link_width"
}

_read_device_current_link_speed() {
    _read_attr "$1" "current_link_speed" | egrep -o '[0-9\.]*'
}

_read_device_max_link_width() {
    _read_attr "$1" "max_link_width"
}

_read_device_max_link_speed() {
    _read_attr "$1" "max_link_speed" | egrep -o '[0-9\.]*'
}

_calc_pcie_gen_by_speed() {
    local pcie_speed="$1"
    local pcie_gen=''

    case "$pcie_speed" in
        2.5)
            pcie_gen="1"
	    ;;
        5)
            pcie_gen="2"
	    ;;
        8)
            pcie_gen="3"
	    ;;
        16)
            pcie_gen="4"
	    ;;
    esac

    echo -n "${pcie_gen}"
}

_calc_pcie_encoding_by_gen() {
    local pcie_gen="$1"
    local pcie_encoding=''

    case "$pcie_gen" in
        1)
            pcie_encoding="1/5" # 8b/10b
	    ;;
        2)
            pcie_encoding="1/5" # 8b/10b
	    ;;
        3)
            pcie_encoding="2/130" # 128b/130b
	    ;;
        4)
            pcie_encoding="2/130" # 128b/130b
	    ;;
    esac

    echo -n "${pcie_encoding}"
}

_calc_math() {
    awk "BEGIN{print $*}";
}

_calc_pcie_throughput() {
    local pcie_width="$1" pcie_speed="$2" pcie_ecc_overhead="$3"
    local pcie_headers_overhead=1

    _calc_math "${pcie_speed} * ${pcie_width} * (1 - ${pcie_ecc_overhead}) - ${pcie_headers_overhead}"
}

_calc_device_info() {
    local pdevice="${1}"

    DEVICE_CUR_WIDTH=$(_read_device_current_link_width "$pdevice")
    DEVICE_CUR_SPEED=$(_read_device_current_link_speed "$pdevice")
    DEVICE_CUR_GEN=$(_calc_pcie_gen_by_speed "${DEVICE_CUR_SPEED}")
    DEVICE_CUR_ECC_OVERHEAD=$(_calc_pcie_encoding_by_gen "${DEVICE_CUR_GEN}")
    DEVICE_CUR_THROUGHPUT=$(_calc_pcie_throughput "${DEVICE_CUR_WIDTH}" "${DEVICE_CUR_SPEED}" "${DEVICE_CUR_ECC_OVERHEAD}")

    DEVICE_MAX_WIDTH=$(_read_device_max_link_width "$pdevice")
    DEVICE_MAX_SPEED=$(_read_device_max_link_speed "$pdevice")
    DEVICE_MAX_GEN=$(_calc_pcie_gen_by_speed "${DEVICE_MAX_SPEED}")
    DEVICE_MAX_ECC_OVERHEAD=$(_calc_pcie_encoding_by_gen "${DEVICE_MAX_GEN}")
    DEVICE_MAX_THROUGHPUT=$(_calc_pcie_throughput "${DEVICE_MAX_WIDTH}" "${DEVICE_MAX_SPEED}" "${DEVICE_MAX_ECC_OVERHEAD}")
}

_clear_device_info() {
    unset DEVICE_CUR_WIDTH DEVICE_CUR_SPEED DEVICE_MAX_GEN DEVICE_CUR_ECC_OVERHEAD DEVICE_CUR_THROUGHPUT
    unset DEVICE_MAX_WIDTH DEVICE_MAX_SPEED DEVICE_MAX_GEN DEVICE_MAX_ECC_OVERHEAD DEVICE_MAX_THROUGHPUT
}

_print_device_info() {
    local device_path="$1" log="$2"
    local _log='echo -e' _prefix=''

    if [[ $(type -t "${log}") == "function" ]]; then
        _log="${log}"
    else
	_prefix="${log}"
    fi

    _calc_device_info "$device_path"

    ${_log} "${_prefix}PCIe generation: ${DEVICE_CUR_GEN} (max.: ${DEVICE_MAX_GEN})"
    ${_log} "${_prefix}PCIe width: x${DEVICE_CUR_WIDTH} (max.: x${DEVICE_MAX_WIDTH})"
    ${_log} "${_prefix}PCIe speed: ${DEVICE_CUR_SPEED} GT/s (max.: ${DEVICE_MAX_SPEED} GT/s)"
    ${_log} "${_prefix}PCIe throughput: ${DEVICE_CUR_THROUGHPUT} Gigabit/s (max.: ${DEVICE_MAX_THROUGHPUT} Gigabit/s)"
    
    [ "${DEVICE_CUR_THROUGHPUT}" != "${DEVICE_MAX_THROUGHPUT}" ] && ${_log} "${_prefix}Device throughput is downgraded!"

    _clear_device_info
}

# END: PCIe-related things

# BEGIN NVMe subsystem & device

_read_subsys_model() {
    _read_attr "$1" "model"
}

_read_subsys_serial() {
    _read_attr "$1" "serial"
}

_read_subsys_nqn() {
    _read_attr "$1" "subsysnqn"
}

_read_subsys_fw_rev() {
    _read_attr "$1" "firmware_rev"
}

_read_nvme_transport() {
    _read_attr "$1" "transport"
}

_calc_device_path() {
    printf "%s/device" "$@"
}

# END: NVMe subsystem & device

# BEGIN: Logging
_log() {
    echo -e "$@"
}

log_subsys() {
    _log "$(printf '[Subsystem: %s] %s' "${SUBSYSTEM_NAME}" "$@")"
}

log_nvme() {
    log_subsys "$(printf '[NVMe: %s] %s' "${NVME_NAME}" "$@")"
}

# END: Logging

# BEGIN: Main

# Allow dumping information about custom sysfs PCIe devices(i.e. Mellanox adapters)
if [[ -n "$@" ]]; then
    _print_device_info "$@"
    exit 0
fi

# Fallback to default iteration over NVMe subsystems
for psubsys in /sys/class/nvme-subsystem/*; do
    SUBSYSTEM_NAME=$(basename $psubsys)

    log_subsys "Model: $(_read_subsys_model "${psubsys}")"
    log_subsys "Serial: $(_read_subsys_serial "${psubsys}")"
    log_subsys "NQN: $(_read_subsys_nqn "${psubsys}")"
    log_subsys "Firmware revision: $(_read_subsys_fw_rev "${psubsys}")"

    for pnvme in "$psubsys"/nvme*; do
        NVME_NAME=$(basename "$pnvme")
	NVME_TRANSPORT=$(_read_nvme_transport "$pnvme")

	log_subsys "Found NVMe: ${NVME_NAME} (transport: ${NVME_TRANSPORT})"

        [[ "$NVME_TRANSPORT" -ne "pcie" ]] && {
            log_nvme "${NVME_TRANSPORT} is not the PCIe transport, skipping..."
            continue
        }

        pdevice=$(_calc_device_path "$pnvme")
	_print_device_info "$pdevice" "log_nvme"
    done

    _log
done

# END: Main