I have an AQC107 10Gb NIC installed in my PC running AlmaLinux 9. AlmaLinux 9 supports the card well enough but I found that a number of network settings need to be changed to get the best performance.
Of course these settings also apply to 8, and I think these should also work on 10 as well but I have not yet tested them.
Make the following changes:
# Maximum number of packets queued on the input side (default: 1000)
sysctl -w net.core.netdev_max_backlog=30000
# Maximum receive socket buffer size (default: 212992)
sysctl -w net.core.rmem_max=134217728
# Maximum send socket buffer size (default: 212992)
sysctl -w net.core.wmem_max=134217728
# Minimum, initial and max TCP receive buffer size (default: 4096 87380 6291456)
sysctl -w net.ipv4.tcp_rmem="10240 131072 134217728"
# Minimum, initial and max TCP send buffer size (default: 4096 20480 4194304)
sysctl -w net.ipv4.tcp_wmem="10240 131072 134217728"
# Google's BBR congestion control algorithm (default: cubic)
# https://research.google/pubs/pub45646/
sysctl -w net.ipv4.tcp_congestion_control=bbr
# BBR requires fq for queue management (default: fq_codel)
sysctl -w net.core.default_qdisc=fq
# If you are using jumbo frames set this to avoid MTU black holes (default: 0)
sysctl -w net.ipv4.tcp_mtu_probing=1
You can check the current/default values before making any changes:
sysctl net.core.netdev_max_backlog
sysctl net.core.rmem_max
sysctl net.core.wmem_max
sysctl net.ipv4.tcp_rmem
sysctl net.ipv4.tcp_wmem
sysctl net.ipv4.tcp_congestion_control
sysctl net.core.default_qdisc
sysctl net.ipv4.tcp_mtu_probing
Defaults on Linux 8-10 are as follows:
sysctl -w net.core.netdev_max_backlog=1000
sysctl -w net.core.rmem_max=212992
sysctl -w net.core.wmem_max=212992
sysctl -w net.ipv4.tcp_rmem="4096 87380 6291456"
sysctl -w net.ipv4.tcp_wmem="4096 87380 6291456"
sysctl -w net.ipv4.tcp_congestion_control=cubic
sysctl -w net.core.default_qdisc=fq_codel
sysctl -w net.ipv4.tcp_mtu_probing=0
For the changes to survive between reboots, add the following to /etc/sysctl.d/99-linux.internal.conf
# Maximum number of packets queued on the input side (default: 1000)
net.core.netdev_max_backlog = 30000
# Maximum receive socket buffer size (default: 212992)
net.core.rmem_max = 134217728
# Maximum send socket buffer size (default: 212992)
net.core.wmem_max = 134217728
# Minimum, initial and max TCP receive buffer size (default: 4096 87380 6291456)
net.ipv4.tcp_rmem = 10240 131072 134217728
# Minimum, initial and max TCP send buffer size (default: 4096 20480 4194304)
net.ipv4.tcp_wmem = 10240 131072 134217728
# Google's BBR congestion control algorithm (default: cubic)
# https://research.google/pubs/pub45646/
net.ipv4.tcp_congestion_control = bbr
# BBR requires fq for queue management (default: fq_codel)
net.core.default_qdisc = fq
# If you are using jumbo frames set this to avoid MTU black holes (default: 0)
net.ipv4.tcp_mtu_probing = 1
Once added to /etc/sysctl.d/99-linux.internal.conf, you can reload the settings as follows:
sysctl --load=/etc/sysctl.d/99-linux.internal.conf
By default BBR is not enabled on Linux 8-10, you can check what is currently enabled as follows:
[root@linux]# sysctl net.ipv4.tcp_available_congestion_control
net.ipv4.tcp_available_congestion_control = reno cubic
You can list all the modules present on your system as follows:
[root@linux]# ls -la /lib/modules/$(uname -r)/kernel/net/ipv4/
.
tcp_bbr.ko.xz
.
The module you are looking for is tcp_bbr.ko.xz.
Enable it with modprobe:
[root@linux]# modprobe -v -a tcp_bbr
insmod /lib/modules/4.18.0-193.14.2.el8_2.x86_64/kernel/net/ipv4/tcp_bbr.ko.xz
You can check for it as follows:
[root@linux]# sysctl net.ipv4.tcp_available_congestion_control
net.ipv4.tcp_available_congestion_control = reno cubic bbr