FreeBSD 15 Bridges, VLANs and Jails - Nice!

00 - Overview

Complex network topology between host and jail with vlan aware bridges and epair trunks. Network set up to ping from one side to the other while traversing the trunk at the host/jail border half a dozen times. tcpdump shows the vlan at the crossings.

01 - vlan-test.conf

# /etc/jail.conf.d/vlan-test.conf

vlan-test {
    host.hostname = vlan-test;
    path = /usr/jails/vlan-test;
    mount.devfs;
    devfs_ruleset = "14";    # Ruleset required for tcpdump, not for bridges
    allow.raw_sockets;

    vnet;
    vnet.interface = "epair0b";

    exec.prestart  = "ifconfig epair0 create up";
    exec.prestart += "ifconfig epair0a up";

    exec.start     = "/bin/sh /etc/rc";

    # Create the test network
    exec.poststart = "/usr/local/sbin/vlan-test-create.sh";

    exec.prestop   = "/usr/local/sbin/vlan-test-destroy.sh";
    exec.stop      = "/bin/sh /etc/rc.shutdown";
    exec.poststop  = "ifconfig epair0a destroy 2>/dev/null";

    persist;
}

02 - vlan-test-create.sh

#!/bin/sh

#  /usr/local/sbin/vlan-test-create.sh

# Ping Path Test
#
# The network snakes around from Host to Jail three times:
#   192.168.1.1 <=> Host <=> Jail <=> Host <=> 192.168.1.2
# and uses VLANs (10, 20, 30) to enable the crossing.
#
# The only epair between the host and the jail is epair0, marked as a
# trunk so it can carry VLAN tagged packets. The host and jail have
# managed switches whose switchports attached to epair0 are marked as
# trunks carrying the VLANs. Two devices exist (.1 and .2) one one each
# side. An ICMP packet snakes back and forth across the trunk moving
# across VLANs to PING and ACK between the devices. When attached to
# the trunk tcpdump shows this.
#
# Normally, we'd imagine a wire hairpinned between VLANs 10/20 and
# also VLANs 20/30 to make this happen. Unfortunately, the bridge
# a single epair won't hairpin traffic, so we had to hack a solution.
#
# How this is wired:
#
#
# epair1 (192.168.1.1)-+                   Jail Hairpin
#                      |
#           epair4     |                      epair2
#         /        \   |                    /        \
#    panel2        bridge0---epair0---bridge1        panel3
#         \        /                    |   \        /
#           epair5                      |     epair3      
#                                       |
#        Host Hairpin                   +-epair6 (192.168.1.2)
#
#
# Logically, epair4<=>panel2<=>epair5 should be a single epair, but
# that's a hairpin with the both ends of the epair connected to the
# same bridge. And, FreeBSD won't allow traffic to flow across it and
# has no configuration to allow that. So, we added two extra "panels"
# (dumb bridges) and two extra epairs to fool the managed switches
# (bridge0|1) to create the hairpin.
#
# NO ONE WOULD EVER DO THIS!!! No one needs this in any production
# system. This is solely for testing to show that epair0 can trunk
# VLANs between two FreeBSD 15 bridges with vlan running through them.
#
# If we isolate tcpdump output to either epair0a|b to see the
# ICMP packet cross VLANs to and from the remote device, we should see
# six crossings: 10>20>30>30>20>10
#
# tcpdump -nnv -e -ttt -i epair0a 'not (port 22 or port 67 or port 68) and not (vlan and udp port 67)'
# jexec vlan-test tcpdump -nnv -e -ttt -i epair0b 'not (port 22 or port 67 or port 68) and not (vlan and udp port 67)'
#
# ping -o 192.168.1.2                    # ping from host to jail device
# jexec vlan-test ping -o 192.168.1.1    # ping from jail to host device


# ----------------------------------------------------------------------
# Create elements in order of ping packet travel
#
# 192.168.1.1 <=> 10 <=> 20 <=> 30 <=> 192.168.1.2 

# ----------------------------------------------------------------------
# This is the only important piece and what we are testing

# Allow epair0 to trunk vlans
ifconfig              epair0a -vlanhwfilter up
ifconfig -j vlan-test epair0b -vlanhwfilter up

# Create host and jail bridges and connect with trunked epair0
ifconfig              bridge0 create up
ifconfig              bridge0 vlanfilter addm epair0a tagged 10,20,30
ifconfig -j vlan-test bridge1 create up
ifconfig -j vlan-test bridge1 vlanfilter addm epair0b tagged 10,20,30
# ----------------------------------------------------------------------


# EVERYTHING BELOW THIS IS TEST BITS

# Create and attach endpoint epair for host
ifconfig              epair1  create up
ifconfig              epair1b up
ifconfig              epair1a inet 192.168.1.1/24
ifconfig              bridge0 addm epair1b untagged 10 up

# Create jail hairpin
#   epair2 <=> bridge2 <=> epair3
#   VLAN: 10 <=> 20
ifconfig -j vlan-test epair2  create up
ifconfig -j vlan-test epair2b create up
ifconfig -j vlan-test epair3  create up
ifconfig -j vlan-test epair3b create up
ifconfig -j vlan-test bridge2 create up
ifconfig -j vlan-test bridge1 addm epair2a untagged 10
ifconfig -j vlan-test bridge2 addm epair2b addm epair3a
ifconfig -j vlan-test bridge1 addm epair3b untagged 20

# Create host hairpin
#   epair4 <=> bridge3 <=> epair5
#   VLAN: 20 <=> 30
ifconfig              epair4  create up
ifconfig              epair4b up
ifconfig              epair5  create up
ifconfig              epair5b up
ifconfig              bridge3 create up
ifconfig              bridge0 addm epair4a untagged 20
ifconfig              bridge3 addm epair4b addm epair5a
ifconfig              bridge0 addm epair5b untagged 30

# Create and attach endpoint epair for jail
ifconfig -j vlan-test epair6 create up
ifconfig -j vlan-test bridge1 addm epair6a untagged 30
ifconfig -j vlan-test epair6b inet 192.168.1.2/24 up


# ----------------------------------------------------------------------
# Further testing not documented above

# --------------------------------
# Test 2
# This creates another trunk epair7 and splinters vlan 20 (epair7b.20) off of it
ifconfig -j vlan-test epair7 create -vlanhwfilter up
ifconfig -j vlan-test epair7b -vlanhwfilter up
ifconfig -j vlan-test bridge1 vlanfilter addm epair7a tagged 10,20,30 up
ifconfig -j vlan-test epair7b.20 create inet 192.168.1.3/24 up

# > ifconfig -j vlan-test epair7b.20
# epair7b.20: flags=1008843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
#         options=0
#         ether d6:16:6d:2d:a7:7d
#         inet 192.168.1.3 netmask 0xffffff00 broadcast 192.168.1.255
#         groups: vlan
#         vlan: 20 vlanproto: 802.1q vlanpcp: 0 parent interface: epair7b
#         media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
#         status: active
#         nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>

03 - vlan-test-destroy.sh

#!/bin/sh

# /usr/local/sbin/vlan-test-destroy.sh

# Clean up
ifconfig bridge0 destroy 2>/dev/null
ifconfig bridge3 destroy 2>/dev/null
ifconfig epair1a destroy 2>/dev/null
ifconfig epair4a destroy 2>/dev/null
ifconfig epair5a destroy 2>/dev/null
ifconfig -j vlan-test bridge1 destroy 2>/dev/null
ifconfig -j vlan-test bridge2 destroy 2>/dev/null
ifconfig -j vlan-test epair2a destroy 2>/dev/null
ifconfig -j vlan-test epair3a destroy 2>/dev/null
ifconfig -j vlan-test epair6a destroy 2>/dev/null

ifconfig -j vlan-test epair7a destroy 2>/dev/null
ifconfig -j vlan-test epair7b.20 destroy 2>/dev/null
ifconfig -j vlan-test bridge0.10 destroy 2>/dev/null
ifconfig -j vlan-test bridge0.30 destroy 2>/dev/null

04 - devfs.rules

# /etc/devfs.rules

[devfsvnet_jail_bpf=14]
add include $devfsrules_hide_all
add include $devfsrules_unhide_basic
add include $devfsrules_unhide_login
add include $devfsrules_jail
add include $devfsrules_jail_vnet
add path bpf* unhide

05 - tcpdump.md

Check out the vlan column.

> tcpdump -nnv -e -ttt -i epair0a 'not (port 22 or port 67 or port 68) and not (vlan and udp port 67)'
tcpdump: listening on epair0a, link-type EN10MB (Ethernet), snapshot length 262144 bytes
 00:00:00.000000 58:9c:fc:10:b0:ff > 9e:3e:dc:bd:3e:ed, ethertype 802.1Q (0x8100), length 102: vlan 10, p 0, ethertype IPv4 (0x0800), (tos 0x0, ttl 64, id 58842, offset 0, flags [none], proto ICMP (1), length 84)
    192.168.1.1 > 192.168.1.2: ICMP echo request, id 24584, seq 0, length 64
 00:00:00.000021 58:9c:fc:10:b0:ff > 9e:3e:dc:bd:3e:ed, ethertype 802.1Q (0x8100), length 102: vlan 20, p 0, ethertype IPv4 (0x0800), (tos 0x0, ttl 64, id 58842, offset 0, flags [none], proto ICMP (1), length 84)
    192.168.1.1 > 192.168.1.2: ICMP echo request, id 24584, seq 0, length 64
 00:00:00.000005 58:9c:fc:10:b0:ff > 9e:3e:dc:bd:3e:ed, ethertype 802.1Q (0x8100), length 102: vlan 30, p 0, ethertype IPv4 (0x0800), (tos 0x0, ttl 64, id 58842, offset 0, flags [none], proto ICMP (1), length 84)
    192.168.1.1 > 192.168.1.2: ICMP echo request, id 24584, seq 0, length 64
 00:00:00.000010 9e:3e:dc:bd:3e:ed > 58:9c:fc:10:b0:ff, ethertype 802.1Q (0x8100), length 102: vlan 30, p 0, ethertype IPv4 (0x0800), (tos 0x0, ttl 64, id 22483, offset 0, flags [none], proto ICMP (1), length 84)
    192.168.1.2 > 192.168.1.1: ICMP echo reply, id 24584, seq 0, length 64
 00:00:00.000002 9e:3e:dc:bd:3e:ed > 58:9c:fc:10:b0:ff, ethertype 802.1Q (0x8100), length 102: vlan 20, p 0, ethertype IPv4 (0x0800), (tos 0x0, ttl 64, id 22483, offset 0, flags [none], proto ICMP (1), length 84)
    192.168.1.2 > 192.168.1.1: ICMP echo reply, id 24584, seq 0, length 64
 00:00:00.000002 9e:3e:dc:bd:3e:ed > 58:9c:fc:10:b0:ff, ethertype 802.1Q (0x8100), length 102: vlan 10, p 0, ethertype IPv4 (0x0800), (tos 0x0, ttl 64, id 22483, offset 0, flags [none], proto ICMP (1), length 84)
    192.168.1.2 > 192.168.1.1: ICMP echo reply, id 24584, seq 0, length 64