{multus-cni, sriov-cni, centralip, sriov-device-plugin} Dockerfile and Daemonset for M-CORD

centralip-ipam-cni.json

{
    "type": "sriov",
    "ipam": {
        "type": "centralip",
        "ipType": "cluster",
        "etcdURL": "http://127.0.0.1:31079",
        "network": "13.1.1.0/26",
        "subnetMin": "13.1.1.2",
        "subnetMax": "13.1.1.63"
    }
}

Dockerfile

# Build multus plugin
FROM golang:1.10 AS multus
RUN git clone -q --depth 1 https://github.com/intel/multus-cni.git /go/src/github.com/intel/multus-cni
WORKDIR /go/src/github.com/intel/multus-cni
RUN ./build

# Build sriov plugin
FROM golang:1.10 AS sriov-cni
RUN git clone -q -b dev/k8s-deviceid-model https://github.com/Intel-Corp/sriov-cni.git /go/src/github.com/intel-corp/sriov-cni
WORKDIR /go/src/github.com/intel-corp/sriov-cni
RUN ./build

# Build sriov device plugin
FROM golang:1.10 AS sriov-dp
RUN git clone -q https://github.com/intel/sriov-network-device-plugin.git /go/src/github.com/intel/sriov-network-device-plugin
WORKDIR /go/src/github.com/intel/sriov-network-device-plugin
RUN make

# Build centralip ipam plugin
FROM golang:1.10 AS centralip-ipam
RUN go get -u github.com/kardianos/govendor
RUN git clone -q https://github.com/John-Lin/ovs-cni.git /go/src/github.com/John-Lin/ovs-cni
WORKDIR /go/src/github.com/John-Lin/ovs-cni
RUN govendor sync && ./build.sh

# Build vfioveth plugin
FROM busybox as vfioveth
RUN wget -O /bin/vfioveth https://gist.githubusercontent.com/krsna1729/c2ed195c26202831ef8f79e319196913/raw/e2cd8486a4fa1cb576a8ddbdcb1839152c46d497/vfioveth && \
    wget -O /bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && \
    chmod +x /bin/vfioveth /bin/jq

# Final image
FROM centos/systemd
WORKDIR /tmp/cni/bin
COPY --from=multus /go/src/github.com/intel/multus-cni/bin/multus .
COPY --from=sriov-cni /go/src/github.com/intel-corp/sriov-cni/bin/sriov .
COPY --from=centralip-ipam /go/src/github.com/John-Lin/ovs-cni/bin/centralip .
COPY --from=vfioveth /bin/vfioveth .
COPY --from=vfioveth /bin/jq .
WORKDIR /usr/bin
COPY --from=sriov-dp /go/src/github.com/intel/sriov-network-device-plugin/build/sriovdp .

multus-sriov-ipam-ds.yaml

---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  # name must match the spec fields below, and be in the form: <plural>.<group>
  name: network-attachment-definitions.k8s.cni.cncf.io
spec:
  # group name to use for REST API: /apis/<group>/<version>
  group: k8s.cni.cncf.io
  # version name to use for REST API: /apis/<group>/<version>
  version: v1
  # either Namespaced or Cluster
  scope: Namespaced
  names:
    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
    plural: network-attachment-definitions
    # singular name to be used as an alias on the CLI and for display
    singular: network-attachment-definition
    # kind is normally the CamelCased singular type. Your resource manifests use this.
    kind: NetworkAttachmentDefinition
    # shortNames allow shorter string to match your resource on the CLI
    shortNames:
    - net-attach-def
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: multus-sa
  namespace: kube-system
---
apiVersion: v1
kind: Secret
metadata:
  name: multus-sa-secret
  namespace: kube-system
  annotations:
    kubernetes.io/service-account.name: multus-sa
type: kubernetes.io/service-account-token
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: multus-pod-networks-lister
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["get"]
- apiGroups: [""]
  resources: ["pods/status"]
  verbs: ["update"]
- apiGroups: ["k8s.cni.cncf.io"]
  resources: ["*"]
  verbs: ["get"]
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: multus-scripts
  namespace: kube-system
data:
  install-certs.sh: |
    # Copied from Calico
    # https://github.com/projectcalico/cni-plugin/blob/master/k8s-install/scripts/install-cni.sh
    touch /host/etc/cni/net.d/multus-kubeconfig
    chmod 600 /host/etc/cni/net.d/multus-kubeconfig
    SERVICE_ACCOUNT_PATH=/var/run/secrets/multus/serviceaccount
    KUBE_CA_FILE=$SERVICE_ACCOUNT_PATH/ca.crt
    TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')"
    SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
    cat > /host/etc/cni/net.d/multus-kubeconfig <<EOF
    # Kubeconfig file for Multus CNI plugin.
    apiVersion: v1
    kind: Config
    clusters:
    - name: local
      cluster:
        server: https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
        $TLS_CFG
    users:
    - name: multus
      user:
        token: "${SERVICEACCOUNT_TOKEN}"
    contexts:
    - name: multus-context
      context:
        cluster: local
        user: multus
    current-context: multus-context
    EOF
  install-multus-conf.sh: |
    # copied from https://github.com/intel/multus-cni/blob/master/images/entrypoint.sh
    rm -f /host/etc/cni/net.d/00-multus.conf
    MASTER_PLUGIN="$(ls /host/etc/cni/net.d | grep -E '\.conf(list)?$' | head -1)"
    MASTER_PLUGIN_JSON="$(cat /host/etc/cni/net.d/$MASTER_PLUGIN)"
    cat > /host/etc/cni/net.d/00-multus.conf <<EOF
    {
      "name": "multus-cni-network",
      "type": "multus",
      "logFile": "/var/log/multus.log",
      "logLevel": "debug",
      "kubeconfig": "/etc/cni/net.d/multus-kubeconfig",
      "delegates": [
                    $MASTER_PLUGIN_JSON
       ]
    }
    EOF
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: multus-rb
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: multus-pod-networks-lister
subjects:
- kind: ServiceAccount
  name: multus-sa
  namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: sriov-device-plugin
  namespace: kube-system
spec:
  selector:
      matchLabels:
        name: sriov-device-plugin
  template:
    metadata:
      labels:
        name: sriov-device-plugin
    spec:
      initContainers:
      - name: multus
        image: krsna1729/multus-sriov:k8s-1.13
        command: [ "bash", "-c" ]
        args:
        - cp /tmp/cni/bin/{multus,sriov,centralip,vfioveth,jq} /host/opt/cni/bin/;
          /tmp/multus/install-multus-conf.sh;
          /tmp/multus/install-certs.sh;
          systemctl stop kubelet;
          echo "Restarting crio/containerd, kubelet";
          systemctl restart containerd;
          systemctl restart crio;
          systemctl restart kubelet;
        volumeMounts:
        - name: usr-bin
          mountPath: /host/usr/bin
        - name: cni-bin
          mountPath: /host/opt/cni/bin
        - name: multus-sa
          mountPath: /var/run/secrets/multus/serviceaccount
        - name: multus-scripts
          mountPath: /tmp/multus
        - name: cni-conf
          mountPath: /host/etc/cni/net.d
        - name: dbus
          mountPath: /var/run/dbus
        - name: systemd
          mountPath: /run/systemd
      containers:
      - name: sriovdp
        image: krsna1729/multus-sriov:k8s-1.13
        command: [ "sh", "-c" ]
        args:
        - /usr/bin/sriovdp --logtostderr -v 10;
          sleep infinity;
        volumeMounts:
        - name: net
          mountPath: /sys/class/net
          readOnly: true
        - name: dp-sock
          mountPath: /var/lib/kubelet/device-plugins/
          readOnly: false
        - mountPath: /etc/pcidp
          name: sriov-config
          readOnly: false
      volumes:
      - name: sriov-config
        configMap:
          name: sriov-config
      - name: usr-bin
        hostPath:
          path: /usr/bin
      - name: cni-bin
        hostPath:
          path: /opt/cni/bin
      - name: multus-sa
        secret:
          secretName: multus-sa-secret
      - name: multus-scripts
        configMap:
          defaultMode: 511
          name: multus-scripts
      - name: cni-conf
        hostPath:
          path: /etc/cni/net.d
      - name: dbus
        hostPath:
          path: /var/run/dbus
      - name: systemd
        hostPath:
          path: /run/systemd
      - name: net
        hostPath:
          path: /sys/class/net
      - name: dp-sock
        hostPath:
          path: /var/lib/kubelet/device-plugins/
      hostNetwork: true
      hostPID: true

sriov-net.yaml

---
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
  name: sriov-net
  annotations:
    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
spec:
  config: '{
    "cniVersion":"0.2.0",
    "type": "vfioveth",
    "name": "sriov-net",
    "ipam": {
        "type": "host-local",
        "subnet": "198.19.0.0/24",
        "rangeStart": "198.19.0.100",
        "rangeEnd": "198.19.0.200",
        "gateway": "198.19.0.1"
    }
}'

sriov-pod.yaml

---
apiVersion: v1
kind: Pod
metadata:
  name: test-sriov
  annotations:
      k8s.v1.cni.cncf.io/networks: sriov-net
spec:
  containers:
  - name: busy
    image: busybox
    command: [ "top" ]
    resources:
      limits:
        intel.com/sriov: '1'

sriov.service

[Unit]
Description=Create VFs on ens785f0 ens785f1 interfaces

[Service]
Type=oneshot
ExecStart=/usr/bin/sriov.sh ens785f0 ens785f1

[Install]
WantedBy=default.target

sriov.sh

#!/bin/bash
# Usage: sriov.sh ens785f0 ens785f1 ...

set -o errexit
set -o pipefail
set -o nounset

reset_pf() {
	local pf=$1
	echo "Resetting $pf"
	echo 0 | tee /sys/class/net/$pf/device/sriov_numvfs
}

set_pf() {
	local pf=$1
	local NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_totalvfs)
	echo "Enabling $NUM_VFS for $pf"
	echo $NUM_VFS | tee /sys/class/net/$pf/device/sriov_numvfs
	ip link set $pf up
}

bind_vfs_vfio() {
	local pf=$1
	local pci=$(readlink /sys/devices/pci*/*/*/net/$pf/device | awk '{print substr($1,10)}')
	echo "Binding VFs of PF $pf ($pci) to vfio-pci"
	for i in $(ls -l /sys/devices/pci*/*/$pci | awk '"virtfn"==substr($9,1,6) {print substr($11,4)}'); do
		echo $i | tee /sys/bus/pci*/*/$i/driver/unbind
		echo vfio-pci | tee /sys/devices/pci*/*/$i/driver_override
		echo $i | tee /sys/bus/pci/drivers/vfio-pci/bind
	done
}

setup_vfs() {
	local pf=$1
	local NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_totalvfs)
	echo "Setting up VFs of PF $pf"
	for ((i = 0; i < ${NUM_VFS}; i++)); do
		ip link set dev $pf vf $i state enable
		ip link set dev $pf vf $i mac \
			$(printf '00:80:86:%02X:%02X:%02X\n' $((RANDOM % 256)) $((RANDOM % 256)) $((RANDOM % 256)))
		# ip link set $pf vf $i spoofchk off
	done
}

for pf in "$@"; do
	reset_pf $pf
	set_pf $pf
	bind_vfs_vfio $pf
	setup_vfs $pf
done

https://gist.github.com/krsna1729/c2ed195c26202831ef8f79e319196913