Qubes-os port forwarding to allow external connections

qvm-portfwd-iptables

#!/bin/bash

# Frédéric Pierret <[email protected]>

# Adapted from previous work:
# - https://gist.github.com/daktak/f887352d564b54f9e529404cc0eb60d5
# - https://gist.github.com/jpouellet/d8cd0eb8589a5b9bf0c53a28fc530369
# - https://gist.github.com/Joeviocoe/6c4dc0c283f6d6c5b1a3f5af8793292b

[ "$DEBUG" = 1 ] && set -x

ip() {
	qvm-prefs -g -- "$1" ip
}

netvm() {
	qvm-prefs -g -- "$1" netvm
}

forward() {
	local action="$1"
	local from_qube="$2"
	local to_qube="$3"
	local port="$4"
	local proto="$5"
	local persistent="$6"

	local iface
	local from_ip
	local to_ip
	local nft_cmd
	local nft_handle

	# TODO: Handle multiple interfaces in sys-net. It currently catches only the first physical interface.
	iface=$(qvm-run -p -u root "$from_qube" "ifconfig | grep cast -B 1 --no-group-separator | grep -vE '^(vif|lo)' | grep -oE '^[^: ]+' | head -1")
	from_ip=$(qvm-run -p -u root "$from_qube" "hostname -I | cut -d ' ' -f 1")
	to_ip=$(ip "$to_qube")

	if [ "x$from_ip" = "xNone" ]; then
		local from_ip=
	fi

	if [[ "$action" = "clear" ]]; then
		echo "$from_qube: Clearing Port Forwarding from $from_qube iptables" >&2
		qvm-run -p -u root "$from_qube" "iptables-save | grep -v 'PortFwd $from_qube' | iptables-restore"

		nft_cmd="nft list table ip qubes-firewall -a | tr -d '\"' | grep 'iifname $iface accept # handle' | awk '{print \$NF}'"
		nft_handle=$(qvm-run -p -u root "$from_qube" "$nft_cmd")

		if [[ $nft_handle =~ ^[0-9]+$ ]]; then
			qvm-run -p -u root "$from_qube" "nft delete rule ip qubes-firewall forward handle $nft_handle"
		fi
		qvm-run -p -u root "$from_qube" "sed -i '/PortFwd $from_qube>$to_qube:$proto$port/d' /rw/config/rc.local"
		qvm-run -p -u root "$from_qube" "sed -i '/PortFwd $from_qube>$to_qube:$proto$port/d' /rw/config/rc.local"
		if ! qvm-run -p -u root "$from_qube" "grep -q 'PortFwd' /rw/config/rc.local"; then
			qvm-run -p -u root "$from_qube" "sed -i '/nft add rule ip qubes-firewall forward meta iifname $iface accept/d' /rw/config/rc.local"
		fi
	else
		echo "$from_qube: Forwarding on $iface port $port to $to_qube ($from_ip -> $to_ip)" >&2
		forward_rule1="iptables -t nat -A PREROUTING -i $iface -p $proto ${from_ip:+-d} $from_ip --dport $port -j DNAT --to-destination $to_ip -m comment --comment \"'PortFwd $from_qube>$to_qube:$proto$port'\""
		forward_rule2="iptables -I FORWARD 2 -i $iface -p $proto ${to_ip:+-d} $to_ip --dport $port -m conntrack --ctstate NEW -j ACCEPT -m comment --comment \"'PortFwd $from_qube>$to_qube:$proto$port'\""
		forward_rule3="nft add rule ip qubes-firewall forward meta iifname $iface accept"

		qvm-run -p -u root "$from_qube" "iptables-save | grep -v 'PortFwd $from_qube>$to_qube:$proto$port' | iptables-restore"
		qvm-run -p -u root "$from_qube" "$forward_rule1"
		qvm-run -p -u root "$from_qube" "$forward_rule2"
		qvm-run -p -u root "$from_qube" "$forward_rule3"
		if [ "$persistent" = 1 ]; then
			qvm-run -p -u root "$from_qube" "echo $forward_rule1 >> /rw/config/rc.local"
			qvm-run -p -u root "$from_qube" "echo $forward_rule2 >> /rw/config/rc.local"
			if ! qvm-run -p -u root "$from_qube" "grep -q 'nft add rule ip qubes-firewall forward meta iifname $iface accept' /rw/config/rc.local"; then
				qvm-run -p -u root "$from_qube" "echo $forward_rule3 >> /rw/config/rc.local"
				# Ensure rc.local is executable
				qvm-run -p -u root "$from_qube" "chmod +x /rw/config/rc.local"
			fi
		fi 
	fi
}

input() {
	local action="$1"
	local qube="$2"
	local port="$3"
	local proto="$4"
	local persistent="$5"

	if [[ "$action" = "clear" ]]; then
		echo "$qube: Clearing Port Forwarding from $qube iptables" >&2
		qvm-run -p -u root "$qube" "iptables-save | grep -v 'PortFwd $qube' | iptables-restore"
		qvm-run -p -u root "$qube" "sed -i '/PortFwd $qube:$proto$port/d' /rw/config/rc.local"
	else
		echo "$qube: Allowing input to port $port" >&2
		qvm-run -p -u root "$qube" "iptables-save | grep -v 'PortFwd $qube:$proto$port' | iptables-restore"
	
		input_rule="iptables -I INPUT 5 -p $proto --dport $port -m conntrack --ctstate NEW -j ACCEPT -m comment --comment \"'PortFwd $qube:$proto$port'\""
		qvm-run -p -u root "$qube" "$input_rule"
		if [ "$persistent" = 1 ]; then
			qvm-run -p -u root "$qube" "echo $input_rule >> /rw/config/rc.local"
			# Ensure rc.local is executable
			qvm-run -p -u root "$qube" "chmod +x /rw/config/rc.local"
		fi
	fi
}

recurse_netvms() {
	local action="$1"
	local this_qube="$2"
	local port="$3"
	local proto="$4"
	local persistent="$5"

	local outer_dom

	outer_dom=$(netvm "$this_qube")
	if [[ -n "$outer_dom" && "$outer_dom" != "None" ]]; then
		forward "$action" "$outer_dom" "$this_qube" "$port" "$proto" "$persistent"
		recurse_netvms "$action" "$outer_dom" "$port" "$proto" "$persistent"
	fi
}

usage() {
	echo "Usage: ${0##*/} --action ACTION --qube QUBE --port PORT --proto PROTO --persistent" >&2
	echo "" >&2
	echo "Exemple: " >&2
	echo " -> ${0##*/} --action create --qube work --port 22" >&2
	echo " -> ${0##*/} --action create --qube work --port 444 --proto udp --persistent" >&2
	echo " -> ${0##*/} --action clear --qube work --port 22" >&2
	echo " -> ${0##*/} --action clear --qube work --port 444 --proto udp" >&2
	echo "" >&2
	echo "Default value for PROTO is 'tcp' and iptables are not persistent"
	exit 1
}

if ! OPTS=$(getopt -o a:q:p:n:s --long action:,qube:,port:,proto:,persistent -n "$0" -- "$@"); then
    echo "An error occurred while parsing options." >&2
    exit 1
fi

eval set -- "$OPTS"

while [[ $# -gt 0 ]]; do
    case "$1" in
        -a | --action ) ACTION="$2"; shift ;;
        -q | --qube ) QUBE="$2"; shift ;;
        -p | --port ) PORT="$2"; shift ;;
        -n | --proto ) PROTO="$2"; shift ;;
        -s | --persistent ) PERSISTENT=1; shift ;;
    esac
    shift
done

if [ -z "$PROTO" ]; then
	PROTO="tcp"
fi

if { [ "$ACTION" != "create" ] || [ "$ACTION" == "clear" ]; } && { [ -z "$QUBE" ] || [ -z "$PORT" ]; }; then
	usage
fi

if ! qvm-check "$QUBE" > /dev/null 2>&1; then
	echo "Qube '$QUBE' not found." >&2
	exit 1
fi

input "$ACTION" "$QUBE" "$PORT" "$PROTO" "$PERSISTENT"
recurse_netvms "$ACTION" "$QUBE" "$PORT" "$PROTO" "$PERSISTENT"

I found a bug. It is not able to find the network interface on the sys-net VM unless the network is actually connected.

[user@dom0 ~]$ ./qvm-portfwd-iptables.sh --action create --qube my-debian-qube --port 22
my-debian-qube: Allowing input to port 22
sys-firewall-lan: Forwarding on eth0 port 22 to my-debian-qube (10.137.0.41 -> 10.137.0.63)
sys-net-lan: Forwarding on  port 22 to sys-firewall-lan (10.137.0.22 -> 10.137.0.41)
Bad argument `tcp'
Try `iptables -h' or 'iptables --help' for more information.
Bad argument `tcp'
Try `iptables -h' or 'iptables --help' for more information.
Error: syntax error, unexpected accept
add rule ip qubes-firewall forward meta iifname accept
                                                ^^^^^^

my-debian-qube is debian-10. However, the same error crops up when I use a fedora-32 VM.

I modified the qvm-portfwd script to pass -v to the relevant qvm-run commands, and now we can see which commands are failing:

[user@dom0 ~]$ ./qvm-portfwd-iptables.sh --action create --qube my-debian-qube --port 22
my-debian-qube: Allowing input to port 22
sys-firewall-lan: Forwarding on eth0 port 22 to my-debian-qube (10.137.0.41 -> 10.137.0.63)
Running 'iptables-save | grep -v 'PortFwd sys-firewall-lan>my-debian-qube:tcp22' | iptables-restore' on sys-firewall-lan
Running 'iptables -t nat -A PREROUTING -i eth0 -p tcp -d 10.137.0.41 --dport 22 -j DNAT --to-destination 10.137.0.63 -m comment --comment "'PortFwd sys-firewall-lan>my-debian-qube:tcp22'"' on sys-firewall-lan
Running 'iptables -I FORWARD 2 -i eth0 -p tcp -d 10.137.0.63 --dport 22 -m conntrack --ctstate NEW -j ACCEPT -m comment --comment "'PortFwd sys-firewall-lan>my-debian-qube:tcp22'"' on sys-firewall-lan
Running 'nft add rule ip qubes-firewall forward meta iifname eth0 accept' on sys-firewall-lan
sys-net-lan: Forwarding on  port 22 to sys-firewall-lan (10.137.0.22 -> 10.137.0.41)
Running 'iptables-save | grep -v 'PortFwd sys-net-lan>sys-firewall-lan:tcp22' | iptables-restore' on sys-net-lan
Running 'iptables -t nat -A PREROUTING -i  -p tcp -d 10.137.0.22 --dport 22 -j DNAT --to-destination 10.137.0.41 -m comment --comment "'PortFwd sys-net-lan>sys-firewall-lan:tcp22'"' on sys-net-lan
Bad argument `tcp'
Try `iptables -h' or 'iptables --help' for more information.
sys-net-lan: command failed with code: 2
Running 'iptables -I FORWARD 2 -i  -p tcp -d 10.137.0.41 --dport 22 -m conntrack --ctstate NEW -j ACCEPT -m comment --comment "'PortFwd sys-net-lan>sys-firewall-lan:tcp22'"' on sys-net-lan
Bad argument `tcp'
Try `iptables -h' or 'iptables --help' for more information.
sys-net-lan: command failed with code: 2
Running 'nft add rule ip qubes-firewall forward meta iifname  accept' on sys-net-lan
Error: syntax error, unexpected accept
add rule ip qubes-firewall forward meta iifname accept
                                                ^^^^^^
sys-net-lan: command failed with code: 1

Here is one error:

Running 'iptables -I FORWARD 2 -i  -p tcp -d 10.137.0.41 --dport 22 -m conntrack --ctstate NEW -j ACCEPT -m comment --comment "'PortFwd sys-net-lan>sys-firewall-lan:tcp22'"' on sys-net-lan

There is no interface name being passed to the -i flag: it's doing -i -p tcp when it should be doing something like -i eth0 -p tcp

Therefore I add an echo statement into the script to print the value of the $iface variable. This shows it is not finding any interface. (The sys-net-lan VM has a physical interface, ens6)

I check the sys-net-lan VM and find the Ethernet cable is unplugged. After plugging in the cable, the script is able to find the interface and configure port forwarding without any problem.

Suggested bug fix: add a test statement to line 36 of the script to check whether the $iface variable is empty. If empty, the script should exit with an error message. Suggested error message:

echo "Unable to find an outward-facing network interface on ${from_qube}, can't continue configuring port-forwarding. Make sure the network connection on ${from_qube} is up and connected to a network."

An alternate and perhaps better fix would be to improve the grep statement so that it may detect network interfaces with no active link.

On my system, for the net vm line 35 finds the right interface for the external network, but line 36 outputs the IP of the internal qubes interface. Therefore the rules never apply to external traffic comming from the network.

I fixed it like that:

@@ -35,3 +35,3 @@ forward() {
	iface=$(qvm-run -p -u root "$from_qube" "ifconfig | grep cast -B 1 --no-group-separator | grep -vE '^(vif|lo)' | grep -oE '^[^: ]+' | head -1")
-	from_ip=$(qvm-run -p -u root "$from_qube" "hostname -I | cut -d ' ' -f 1")
+	from_ip=$(qvm-run -p -u root "$from_qube" "ifconfig | grep cast -B 1 --no-group-separator | grep '$iface' -A1 | grep -Eo 'inet [0-9]+(\.[0-9]+){3}' | cut -d ' ' -f 2")
	to_ip=$(ip "$to_qube")

This queries the IPv4 address of the interface found in line 35 and doesn't rely on the order of IPs that are returned by hostname.
(This won't work for IPv6-only interfaces without modification.)

#jfyi in https://forum.qubes-os.org/t/port-forwarding-to-allow-external-connections-qvm-portfwd-iptables/18870 ,
i have attempted to grasp the "essentials" of what this script does (minus the undo). But also:

• replaced all short form options
• renamed iface to incomingInterface
• renamed from_ip to incomingPacketRecipient
• renamed to_ip to outgoingPacketRecipient

What seems to be the essence is this:

iptables -I INPUT 5 -p $proto --dport $port -m conntrack --ctstate NEW -j ACCEPT -m comment --comment "'PortFwd $qube:$proto$port'"

And then traversing the net vms:

iptables -t nat -A PREROUTING -i $iface -p $proto ${from_ip:+-d} $from_ip --dport $port -j DNAT --to-destination $to_ip -m comment --comment "'PortFwd $from_qube>$to_qube:$proto$port'"
iptables -I FORWARD 2 -i $iface -p $proto ${to_ip:+-d} $to_ip --dport $port -m conntrack --ctstate NEW -j ACCEPT -m comment --comment "'PortFwd $from_qube>$to_qube:$proto$port'"
nft add rule ip qubes-firewall forward meta iifname $iface accept