setup-bw-qos.sh

#!/bin/sh
#
# Run this script under openwrt box
#

MAX_DL_RATE=1000
MAX_UP_RATE=1000

tc() {
	local ret
	/usr/sbin/tc "$@"
	ret=$?
	echo "[$ret] tc $@" >> /tmp/qos.log
	return $ret
}

iptables() {
	local ret
	/usr/sbin/iptables "$@"
	ret=$?
	echo "[$ret] iptables $@" >> /tmp/qos.log
	return $ret
}

ip() {
	local ret
	/usr/sbin/ip "$@"
	ret=$?
	echo "[$ret] ip $@" >> /tmp/qos.log
	return $ret
}

insmod() {
	local ret
	/usr/sbin/insmod "$@"
	ret=$?
	echo "[$ret] insmod $@" >> /tmp/qos.log
	return $ret
}

rmmod() {
	local ret
	/usr/sbin/rmmod "$@"
	ret=$?
	echo "[$ret] rmmod $@" >> /tmp/qos.log
	return $ret
}

# $1: ifname to rate limit
# $2: download IFB ifname
# $3: upload IFB ifname
# $4: upload min rate (Mbit)
# $5: upload max rate (Mbit)
# $6: download min rate (Mbit)
# $7: download max rate (Mbit)
# $8: priority
_set_interface_qos() {
	local ifname=$1
	local dl_ifname=$2
	local up_ifname=$3
	local download_min_rate=$4
	local download_max_rate=$5
	local upload_min_rate=$6
	local upload_max_rate=$7
	local priority=$8
	local cnt=0
	
	tc qdisc del dev $ifname root
	tc qdisc del dev $ifname ingress

	# Download
	cnt=$(tc class show dev ${dl_ifname} | wc -l)
	tc qdisc add dev ${ifname} root handle 1: htb
	tc filter add dev ${ifname} parent 1: protocol all prio 1 u32 match u32 0 0 action connmark action mirred egress redirect dev ${dl_ifname}
	tc class add dev ${dl_ifname} parent 1:1 classid 1:1$cnt htb rate ${download_min_rate}mbit ceil ${download_max_rate}mbit prio ${priority}
	tc qdisc add dev ${dl_ifname} handle 1${cnt}:0 parent 1:1${cnt} sfq perturb 5
	tc filter add dev ${dl_ifname} pref 1 protocol ip handle 1${cnt} fw flowid 1:1${cnt}
	
	iptables -t mangle -A qosmark -m physdev --physdev-in ${ifname} -j MARK --set-mark 1${cnt}
	iptables -t mangle -A qosmark -m mark --mark 1${cnt} -j RETURN
	
	# Upload
	cnt=$(tc class show dev ${up_ifname} | wc -l)
	tc qdisc add dev ${ifname} handle ffff: ingress
	tc filter add dev ${ifname} parent ffff: protocol all prio 1 u32 match u32 0 0 action connmark action mirred egress redirect dev ${up_ifname}
	tc class add dev ${up_ifname} parent 1:1 classid 1:1${cnt} htb rate ${upload_min_rate}mbit ceil ${upload_max_rate}mbit prio ${priority}
	tc qdisc add dev ${up_ifname} handle 1${cnt}:0 parent 1:1${cnt} sfq perturb 5
	tc filter add dev ${up_ifname} pref 1 protocol ip handle 1${cnt} fw flowid 1:1${cnt}
}

stop_qos() {
	rmmod ifb
	insmod ifb numifbs=2
	
	ip link del dev ifb0
	ip link del dev ifb1
	
	iptables -t mangle -F qosmark
	iptables -t mangle -D PREROUTING -j qosmark
	iptables -t mangle -X qosmark
	iptables -t mangle -D POSTROUTING -j CONNMARK --save-mark

	tc qdisc del dev download-ifb root 2>/dev/null
	tc qdisc del dev upload-ifb root 2>/dev/null

	ip link set dev download-ifb down 2>/dev/null
	ip link set dev upload-ifb down 2>/dev/null

	ip link del dev download-ifb 2>/dev/null
	ip link del dev upload-ifb 2>/dev/null
}

init_qos() {
	ip link add dev download-ifb name download-ifb type ifb
	ip link set dev download-ifb up

	ip link add dev upload-ifb name upload-ifb type ifb
	ip link set dev upload-ifb up

	echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables

	# Download
	tc qdisc add dev download-ifb root handle 1: htb
	tc class add dev download-ifb parent 1: classid 1:1 htb rate 1mbit ceil ${MAX_DL_RATE}mbit
	
	# Upload
	tc qdisc add dev upload-ifb root handle 1: htb
	tc class add dev upload-ifb parent 1: classid 1:1 htb rate 1mbit ceil ${MAX_UP_RATE}mbit
	
	iptables -t mangle -N qosmark
	iptables -t mangle -A qosmark -j CONNMARK --restore-mark
	iptables -t mangle -A qosmark -m mark ! --mark 0 -j ACCEPT
	iptables -t mangle -A PREROUTING -j qosmark
	iptables -t mangle -A POSTROUTING -j CONNMARK --save-mark
}

setup_qos() {
	rm -f /tmp/qos.log
	
	stop_qos
	init_qos

	_set_interface_qos ath0 download-ifb upload-ifb 20 20 10 10 0
	_set_interface_qos ath1 download-ifb upload-ifb 40 40 20 20 1
	_set_interface_qos ath2 download-ifb upload-ifb 80 80 40 40 2
}

show_qos() {
	local ifnames="ath0 ath1 ath2"
	
	echo "tc -s -d qdisc list | grep htb"
	echo "------------------------------"
	tc -s -d qdisc list | grep htb
	echo ""
	
	echo "tc -s -d qdisc list | grep ingress"
	echo "----------------------------------"
	tc -s -d qdisc list | grep ingress
	echo ""
	
	echo "tc -s -d cl show dev download-ifb"
	echo "---------------------------------"
	tc -s -d cl show dev download-ifb
	echo ""

	echo "tc -s -d cl show dev upload-ifb"
	echo "-------------------------------"
	tc -s -d cl show dev upload-ifb
	echo ""
	
	for i in $(echo $ifnames); do
		echo "tc -s -d fi show dev $i"
		echo "---------------------------"
		tc -s -d fi show dev $i
		echo ""
	done
	
	echo "tc -s -d fi show dev download-ifb"
	echo "---------------------------------"
	tc -s -d fi show dev download-ifb
	echo ""

	echo "tc -s -d fi show dev upload-ifb"
	echo "-------------------------------"
	tc -s -d fi show dev upload-ifb
	echo ""

	echo "iptables -t mangle -L qosmark -nv"
	echo "----------------------------------"
	iptables -t mangle -L qosmark -nv
	echo ""
}

main() {
	local argc=$1; shift
	
	case "$1" in
		"")
			setup_qos
		;;
		"show")
			show_qos
		;;	
	esac
}

main $# "$@"

exit 0

Setup following QoS Setting

ath0 - Download Max/Min Rate: 20M/20M , Upload Max/Min Rate: (10M/10M) , Priority 0

ath1 - Download Max/Min Rate: 40M/40M , Upload Max/Min Rate: (20M/20M) , Priority 1

ath2 - Download Max/Min Rate: 80M/80M , Upload Max/Min Rate: (40M/40M) , Priority 2

# ./setup-bw-qos.sh 

# cat /tmp/qos.log 
[0] rmmod ifb
[0] insmod ifb numifbs=2
[0] ip link del dev ifb0
[0] ip link del dev ifb1
[0] iptables -t mangle -F qosmark
[0] iptables -t mangle -D PREROUTING -j qosmark
[0] iptables -t mangle -X qosmark
[0] iptables -t mangle -D POSTROUTING -j CONNMARK --save-mark
[1] tc qdisc del dev download-ifb root
[1] tc qdisc del dev upload-ifb root
[1] ip link set dev download-ifb down
[1] ip link set dev upload-ifb down
[1] ip link del dev download-ifb
[1] ip link del dev upload-ifb
[0] ip link add dev download-ifb name download-ifb type ifb
[0] ip link set dev download-ifb up
[0] ip link add dev upload-ifb name upload-ifb type ifb
[0] ip link set dev upload-ifb up
[0] tc qdisc add dev download-ifb root handle 1: htb
[0] tc class add dev download-ifb parent 1: classid 1:1 htb rate 1mbit ceil 1000mbit
[0] tc qdisc add dev upload-ifb root handle 1: htb
[0] tc class add dev upload-ifb parent 1: classid 1:1 htb rate 1mbit ceil 1000mbit
[0] iptables -t mangle -N qosmark
[0] iptables -t mangle -A qosmark -j CONNMARK --restore-mark
[0] iptables -t mangle -A qosmark -m mark ! --mark 0 -j ACCEPT
[0] iptables -t mangle -A PREROUTING -j qosmark
[0] iptables -t mangle -A POSTROUTING -j CONNMARK --save-mark
[0] tc qdisc del dev ath0 root
[0] tc qdisc del dev ath0 ingress
[0] tc class show dev download-ifb
[0] tc qdisc add dev ath0 root handle 1: htb
[0] tc filter add dev ath0 parent 1: protocol all prio 1 u32 match u32 0 0 action connmark action mirred egress redirect dev download-ifb
[0] tc class add dev download-ifb parent 1:1 classid 1:11 htb rate 20mbit ceil 20mbit prio 0
[0] tc qdisc add dev download-ifb handle 11:0 parent 1:11 sfq perturb 5
[0] tc filter add dev download-ifb pref 1 protocol ip handle 11 fw flowid 1:11
[0] iptables -t mangle -A qosmark -m physdev --physdev-in ath0 -j MARK --set-mark 11
[0] iptables -t mangle -A qosmark -m mark --mark 11 -j RETURN
[0] tc class show dev upload-ifb
[0] tc qdisc add dev ath0 handle ffff: ingress
[0] tc filter add dev ath0 parent ffff: protocol all prio 1 u32 match u32 0 0 action connmark action mirred egress redirect dev upload-ifb
[0] tc class add dev upload-ifb parent 1:1 classid 1:11 htb rate 10mbit ceil 10mbit prio 0
[0] tc qdisc add dev upload-ifb handle 11:0 parent 1:11 sfq perturb 5
[0] tc filter add dev upload-ifb pref 1 protocol ip handle 11 fw flowid 1:11
[0] tc qdisc del dev ath1 root
[0] tc qdisc del dev ath1 ingress
[0] tc class show dev download-ifb
[0] tc qdisc add dev ath1 root handle 1: htb
[0] tc filter add dev ath1 parent 1: protocol all prio 1 u32 match u32 0 0 action connmark action mirred egress redirect dev download-ifb
[0] tc class add dev download-ifb parent 1:1 classid 1:12 htb rate 40mbit ceil 40mbit prio 1
[0] tc qdisc add dev download-ifb handle 12:0 parent 1:12 sfq perturb 5
[0] tc filter add dev download-ifb pref 1 protocol ip handle 12 fw flowid 1:12
[0] iptables -t mangle -A qosmark -m physdev --physdev-in ath1 -j MARK --set-mark 12
[0] iptables -t mangle -A qosmark -m mark --mark 12 -j RETURN
[0] tc class show dev upload-ifb
[0] tc qdisc add dev ath1 handle ffff: ingress
[0] tc filter add dev ath1 parent ffff: protocol all prio 1 u32 match u32 0 0 action connmark action mirred egress redirect dev upload-ifb
[0] tc class add dev upload-ifb parent 1:1 classid 1:12 htb rate 20mbit ceil 20mbit prio 1
[0] tc qdisc add dev upload-ifb handle 12:0 parent 1:12 sfq perturb 5
[0] tc filter add dev upload-ifb pref 1 protocol ip handle 12 fw flowid 1:12
[0] tc qdisc del dev ath2 root
[0] tc qdisc del dev ath2 ingress
[0] tc class show dev download-ifb
[0] tc qdisc add dev ath2 root handle 1: htb
[0] tc filter add dev ath2 parent 1: protocol all prio 1 u32 match u32 0 0 action connmark action mirred egress redirect dev download-ifb
[0] tc class add dev download-ifb parent 1:1 classid 1:13 htb rate 80mbit ceil 80mbit prio 2
[0] tc qdisc add dev download-ifb handle 13:0 parent 1:13 sfq perturb 5
[0] tc filter add dev download-ifb pref 1 protocol ip handle 13 fw flowid 1:13
[0] iptables -t mangle -A qosmark -m physdev --physdev-in ath2 -j MARK --set-mark 13
[0] iptables -t mangle -A qosmark -m mark --mark 13 -j RETURN
[0] tc class show dev upload-ifb
[0] tc qdisc add dev ath2 handle ffff: ingress
[0] tc filter add dev ath2 parent ffff: protocol all prio 1 u32 match u32 0 0 action connmark action mirred egress redirect dev upload-ifb
[0] tc class add dev upload-ifb parent 1:1 classid 1:13 htb rate 40mbit ceil 40mbit prio 2
[0] tc qdisc add dev upload-ifb handle 13:0 parent 1:13 sfq perturb 5
[0] tc filter add dev upload-ifb pref 1 protocol ip handle 13 fw flowid 1:13

# ./setup-bw-qos.sh show
tc -s -d qdisc list | grep htb
------------------------------
qdisc htb 1: dev ath0 root refcnt 2 r2q 10 default 0 direct_packets_stat 8 ver 3.17 direct_qlen 2
qdisc htb 1: dev ath1 root refcnt 2 r2q 10 default 0 direct_packets_stat 10 ver 3.17 direct_qlen 2
qdisc htb 1: dev ath2 root refcnt 2 r2q 10 default 0 direct_packets_stat 10 ver 3.17 direct_qlen 2
qdisc htb 1: dev download-ifb root refcnt 2 r2q 10 default 0 direct_packets_stat 28 ver 3.17 direct_qlen 32
qdisc htb 1: dev upload-ifb root refcnt 2 r2q 10 default 0 direct_packets_stat 0 ver 3.17 direct_qlen 32

tc -s -d qdisc list | grep ingress
----------------------------------
qdisc ingress ffff: dev ath0 parent ffff:fff1 ---------------- 
qdisc ingress ffff: dev ath1 parent ffff:fff1 ---------------- 
qdisc ingress ffff: dev ath2 parent ffff:fff1 ---------------- 

tc -s -d cl show dev download-ifb
---------------------------------
class htb 1:11 parent 1:1 leaf 11: prio 0 quantum 200000 rate 20Mbit ceil 20Mbit linklayer ethernet burst 1600b/1 mpu 0b overhead 0b cburst 1600b/1 mpu 0b overhead 0b level 0 
 Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) 
 rate 0bit 0pps backlog 0b 0p requeues 0 
 lended: 0 borrowed: 0 giants: 0
 tokens: 10000 ctokens: 10000

class htb 1:1 root rate 1Mbit ceil 1Gbit linklayer ethernet burst 1600b/1 mpu 0b overhead 0b cburst 1375b/1 mpu 0b overhead 0b level 7 
 Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) 
 rate 0bit 0pps backlog 0b 0p requeues 0 
 lended: 0 borrowed: 0 giants: 0
 tokens: 200000 ctokens: 187

class htb 1:13 parent 1:1 leaf 13: prio 2 quantum 200000 rate 80Mbit ceil 80Mbit linklayer ethernet burst 1600b/1 mpu 0b overhead 0b cburst 1600b/1 mpu 0b overhead 0b level 0 
 Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) 
 rate 0bit 0pps backlog 0b 0p requeues 0 
 lended: 0 borrowed: 0 giants: 0
 tokens: 2500 ctokens: 2500

class htb 1:12 parent 1:1 leaf 12: prio 1 quantum 200000 rate 40Mbit ceil 40Mbit linklayer ethernet burst 1600b/1 mpu 0b overhead 0b cburst 1600b/1 mpu 0b overhead 0b level 0 
 Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) 
 rate 0bit 0pps backlog 0b 0p requeues 0 
 lended: 0 borrowed: 0 giants: 0
 tokens: 5000 ctokens: 5000


tc -s -d cl show dev upload-ifb
-------------------------------
class htb 1:11 parent 1:1 leaf 11: prio 0 quantum 125000 rate 10Mbit ceil 10Mbit linklayer ethernet burst 1600b/1 mpu 0b overhead 0b cburst 1600b/1 mpu 0b overhead 0b level 0 
 Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) 
 rate 0bit 0pps backlog 0b 0p requeues 0 
 lended: 0 borrowed: 0 giants: 0
 tokens: 20000 ctokens: 20000

class htb 1:1 root rate 1Mbit ceil 1Gbit linklayer ethernet burst 1600b/1 mpu 0b overhead 0b cburst 1375b/1 mpu 0b overhead 0b level 7 
 Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) 
 rate 0bit 0pps backlog 0b 0p requeues 0 
 lended: 0 borrowed: 0 giants: 0
 tokens: 200000 ctokens: 187

class htb 1:13 parent 1:1 leaf 13: prio 2 quantum 200000 rate 40Mbit ceil 40Mbit linklayer ethernet burst 1600b/1 mpu 0b overhead 0b cburst 1600b/1 mpu 0b overhead 0b level 0 
 Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) 
 rate 0bit 0pps backlog 0b 0p requeues 0 
 lended: 0 borrowed: 0 giants: 0
 tokens: 5000 ctokens: 5000

class htb 1:12 parent 1:1 leaf 12: prio 1 quantum 200000 rate 20Mbit ceil 20Mbit linklayer ethernet burst 1600b/1 mpu 0b overhead 0b cburst 1600b/1 mpu 0b overhead 0b level 0 
 Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) 
 rate 0bit 0pps backlog 0b 0p requeues 0 
 lended: 0 borrowed: 0 giants: 0
 tokens: 10000 ctokens: 10000


tc -s -d fi show dev ath0
---------------------------
filter parent 1: protocol all pref 1 u32 
filter parent 1: protocol all pref 1 u32 fh 800: ht divisor 1 
filter parent 1: protocol all pref 1 u32 fh 800::800 order 2048 key ht 800 bkt 0 terminal flowid ??? 
  match 00000000/00000000 at 0
        action order 1:  connmark       Action statistics:
        Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) 
        backlog 0b 0p requeues 0 

        action order 2: mirred (Egress Redirect to device download-ifb) stolen
        index 21 ref 1 bind 1 installed 272 sec used 23 sec
        Action statistics:
        Sent 1264 bytes 8 pkt (dropped 0, overlimits 0 requeues 0) 
        backlog 0b 0p requeues 0 


tc -s -d fi show dev ath1
---------------------------
filter parent 1: protocol all pref 1 u32 
filter parent 1: protocol all pref 1 u32 fh 800: ht divisor 1 
filter parent 1: protocol all pref 1 u32 fh 800::800 order 2048 key ht 800 bkt 0 terminal flowid ??? 
  match 00000000/00000000 at 0
        action order 1:  connmark       Action statistics:
        Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) 
        backlog 0b 0p requeues 0 

        action order 2: mirred (Egress Redirect to device download-ifb) stolen
        index 23 ref 1 bind 1 installed 272 sec used 20 sec
        Action statistics:
        Sent 1396 bytes 10 pkt (dropped 0, overlimits 0 requeues 0) 
        backlog 0b 0p requeues 0 


tc -s -d fi show dev ath2
---------------------------
filter parent 1: protocol all pref 1 u32 
filter parent 1: protocol all pref 1 u32 fh 800: ht divisor 1 
filter parent 1: protocol all pref 1 u32 fh 800::800 order 2048 key ht 800 bkt 0 terminal flowid ??? 
  match 00000000/00000000 at 0
        action order 1:  connmark       Action statistics:
        Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) 
        backlog 0b 0p requeues 0 

        action order 2: mirred (Egress Redirect to device download-ifb) stolen
        index 25 ref 1 bind 1 installed 272 sec used 18 sec
        Action statistics:
        Sent 1396 bytes 10 pkt (dropped 0, overlimits 0 requeues 0) 
        backlog 0b 0p requeues 0 


tc -s -d fi show dev download-ifb
---------------------------------
filter parent 1: protocol ip pref 1 fw 
filter parent 1: protocol ip pref 1 fw handle 0xb classid 1:11 
filter parent 1: protocol ip pref 1 fw handle 0xc classid 1:12 
filter parent 1: protocol ip pref 1 fw handle 0xd classid 1:13 

tc -s -d fi show dev upload-ifb
-------------------------------
filter parent 1: protocol ip pref 1 fw 
filter parent 1: protocol ip pref 1 fw handle 0xb classid 1:11 
filter parent 1: protocol ip pref 1 fw handle 0xc classid 1:12 
filter parent 1: protocol ip pref 1 fw handle 0xd classid 1:13 

iptables -t mangle -L qosmark -nv
----------------------------------
Chain qosmark (1 references)
 pkts bytes target     prot opt in     out     source               destination         
  502 71985 CONNMARK   all  --  *      *       0.0.0.0/0            0.0.0.0/0            CONNMARK restore
    0     0 ACCEPT     all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match ! 0x0
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            PHYSDEV match --physdev-in ath0 MARK set 0xb
    0     0 RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0xb
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            PHYSDEV match --physdev-in ath1 MARK set 0xc
    0     0 RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0xc
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            PHYSDEV match --physdev-in ath2 MARK set 0xd
    0     0 RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match 0xd