POC to attach a VM to a CNI NIC using libvirt

kube-vm-network.sh

#!/bin/sh

# POC to connect a libvirt VM to a Pod NIC created by CNI

# Resources:
# - rancher-vm networking https://github.com/rancher/vm/blob/master/docs/networking.md
# - qemu-ifup https://gist.github.com/Wayt/26d25002d00d7cb4348499a145456999
# - cni https://github.com/containernetworking/cni/blob/master/scripts/exec-plugins.sh

# yum install -y macchanger

toupper() { tr [[:lower:]] [[:upper:]] ; }
onhost() { nsenter --mount=/host-proc/1/ns/mnt --net=/host-proc/1/ns/net $@ ; }

IFNAME=eth42

all_containers() { onhost docker ps --format "{{.ID}}" ; }
container_net_ns() { onhost docker inspect $1 | onhost jq -r '.[].NetworkSettings.SandboxKey' ; }
container_ifs() { onhost nsenter --net=$1 ip a ; }

my_network_namespace_containerid()
{
    for CONTAINER in $(all_containers);
    do
        CONTAINER_NET_NS=$(container_net_ns $CONTAINER)
        [[ -z "$CONTAINER_NET_NS" ]] && continue
        CONTAINER_IFS=$(container_ifs $CONTAINER_NET_NS)
        if [[ "x$CONTAINER_IFS" = "x$(ip a)" ]]; then
            echo $CONTAINER
        fi
    done
}

container_pid()
{
    onhost docker inspect $1 | onhost jq -r ".[0].State.Pid"
}

export MY_NETWORK_NAMESPACE_CONTAINERID=$(my_network_namespace_containerid)
echo containerid $MY_NETWORK_NAMESPACE_CONTAINERID

export MY_NETWORK_NAMESPACE_PID=$(container_pid $MY_NETWORK_NAMESPACE_CONTAINERID)
echo pid $MY_NETWORK_NAMESPACE_PID

cni_add_if()
{
    ip link del $IFNAME || :
    cat <<EOF | onhost tee /tmp/k
#!/bin/sh
set -x ; \
export PATH=/opt/cni/bin/:\$PATH \
CNI_COMMAND=ADD \
CNI_CONTAINERID=$MY_NETWORK_NAMESPACE_CONTAINER \
CNI_NETNS=/proc/$MY_NETWORK_NAMESPACE_PID/ns/net \
CNI_IFNAME=$IFNAME \
CNI_PATH=/opt/cni/bin/ ; \
env ; /opt/cni/bin/bridge < /etc/cni/net.d/k8s.conf
EOF
    onhost chmod a+x /tmp/k
    onhost systemd-run /tmp/k
    #onhost rm -v /tmp/k
}

cni_remember_but_hide_if()
{
    nic_field() { ip address show $IFNAME | awk "{ for(i=0;i<NF;i++) { if (\$i == \"$1\") { print \$(i+1); next; } } }" ; }
    export NIC_ORIG_IP=$(nic_field inet)
    export NIC_IP=$(nic_field inet | egrep -o "^[0-9.]+")
    # FIXME libvirt detects if this network is already used, that's why we need a higher class
    export NIC_PREFIX=$(( $(nic_field inet | egrep -o "[0-9]+$") - 8 ))
    export NIC_MAC=$(nic_field link/ether | toupper)
    # FIXME DNS IP needs to created much smarter
    export DNS_IP=${NIC_IP%.*}.254

    ip address del $NIC_ORIG_IP dev $IFNAME
    #ip link set $IFNAME address $(od /dev/urandom -w6 -tx1 -An|sed -e 's/ //' -e 's/ /:/g'|head -n 1)
    # FIXME do I really need to change this.
    macchanger -a $IFNAME
}

vir_create_net_w_dhcp()
{
    virsh net-destroy net-$IFNAME || :
    virsh net-undefine net-$IFNAME || :
    cat <<EON | virsh net-define /dev/stdin
<network>
  <name>net-$IFNAME</name>
  <bridge name="br-$IFNAME" />
  <ip address="$DNS_IP" prefix="$NIC_PREFIX" localPtr="yes">
    <dhcp>
      <range start="$NIC_IP" end="$NIC_IP"/>
      <host mac="$NIC_MAC" name="FIXME_HOSTNAME" ip="$NIC_IP"/>
    </dhcp>
  </ip>
</network>
EON
    virsh net-start net-$IFNAME
    virsh net-dumpxml net-$IFNAME
    
    # avoid that pod uses br for traffic
    ip route del 10.0.0.0/8 dev br-$IFNAME

    ip link set dev $IFNAME master br-$IFNAME up    
    ip link set dev br-$IFNAME-nic up
}



cni_add_if
cni_remember_but_hide_if
vir_create_net_w_dhcp


virsh undefine testvm || :
virsh destroy testvm || :
virsh define /dev/stdin <<EOF
<domain type="qemu">
  <name>testvm</name>
  <memory unit="MB">64</memory>
  <os>
    <type>hvm</type>
  </os>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <interface type="network">
      <source network="net-$IFNAME"></source>
      <mac address="$NIC_MAC" />
      <model type='virtio'/>
    </interface>
    <disk device="disk" snapshot="external" type="network">
      <source protocol="iscsi" name="iqn.2017-01.io.kubevirt:sn.42/2"><!-- 2 alpine -->
        <host name="iscsi-demo-target" port="3260"></host>
      </source>
      <target dev="vda"></target>
      <driver cache="none" name="qemu" type="raw"></driver>
    </disk>
    <console type="pty"></console>
  </devices>
</domain>
EOF
virsh start testvm
virsh console testvm