Pursuant to microshift SCC bug chasing on Kubernetes #kubevirt-dev slack thread
- Install Fedora 35 Server on kvm device equipped VM
- Configure with br0 interface connected to local network
- Install Kubevirt dependencies and deploy vm:
##################################################################################
# Install Microshift
sudo dnf copr enable -y @redhat-et/microshift
sudo dnf module enable -y cri-o:1.21
sudo dnf install -y firewalld kernel-modules kubernetes-client helm dracut-squash squashfs-tools squashfuse fuse jq
sudo dnf install -y firewalld kubernetes-client helm jq cri-o cri-tools microshift tmux NetworkManager-tui
sudo systemctl enable --now crio
sudo systemctl enable --now microshift
sudo systemctl disable firewalld
curl --output /tmp/virtctl -L https://github.com/kubevirt/kubevirt/releases/download/$(curl -s https://api.github.com/repos/kubevirt/kubevirt/releases/latest | awk -F '[",]' '/tag_name/{print $4}')/virtctl-$(curl -s https://api.github.com/repos/kubevirt/kubevirt/releases/latest | awk -F '[",]' '/tag_name/{print $4}')-linux-amd64
sudo install -o root -g root -m 0755 /tmp/virtctl /usr/local/bin/virtctl
sudo grubby --update-kernel=ALL --args 'selinux=0 intel_iommu=on iommu=pt rd.driver.pre=vfio-pci pci=realloc'
mkdir -p ~/.kube
touch ~/.kube/config
chmod 600 ~/.kube/config
sudo cat /var/lib/microshift/resources/kubeadmin/kubeconfig > ~/.kube/config
sudo ln -s /usr/bin/kubectl /usr/bin/k
kubectl get po -A
sudo shutdown -r now
##################################################################################
# Install Kubevirt
# Cert Manager (dependency for kubevirt webhook)
helm repo add jetstack https://charts.jetstack.io; helm repo update
helm upgrade --install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --set installCRDs=true
# Kubevirt
curl -sL https://github.com/kubevirt/kubevirt/releases/download/$(curl -s https://api.github.com/repos/kubevirt/kubevirt/releases/latest | awk -F '[",]' '/tag_name/{print $4}')/kubevirt-operator.yaml | kubectl apply -f -
curl -sL https://github.com/kubevirt/kubevirt/releases/download/$(curl -s https://api.github.com/repos/kubevirt/kubevirt/releases/latest | awk -F '[",]' '/tag_name/{print $4}')/kubevirt-cr.yaml | kubectl apply -f -
cat <<EOF | kubectl apply -f -
---
apiVersion: kubevirt.io/v1
kind: KubeVirt
metadata:
name: kubevirt
namespace: kubevirt
spec:
configuration:
developerConfiguration:
featureGates:
- LiveMigration
- DataVolumes
- ExpandDisks
- ExperimentalIgnitionSupport
- Sidecar
- HostDevices
- Snapshot
- HotplugVolumes
- ExperimentalVirtiofsSupport
- GPU
EOF
watch kubectl get po -A
kubectl -n kubevirt wait kv kubevirt --for condition=Available
##################################################################################
# Install Multus - Fail method A - Mutually exclusive with method B
kubectl apply -f https://github.com/kubevirt/cluster-network-addons-operator/releases/download/v0.77.0/namespace.yaml
kubectl apply -f https://github.com/kubevirt/cluster-network-addons-operator/releases/download/v0.77.0/network-addons-config.crd.yaml
kubectl apply -f https://github.com/kubevirt/cluster-network-addons-operator/releases/download/v0.77.0/operator.yaml
cat <<EOF | kubectl apply -f -
---
apiVersion: networkaddonsoperator.network.kubevirt.io/v1
kind: NetworkAddonsConfig
metadata:
name: cluster
spec:
imagePullPolicy: IfNotPresent
multus: {}
EOF
kubectl describe networkaddonsconfig cluster
##################################################################################
# Install Multus - Fail method B - Mutually exclusive with method A
kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick-plugin.yml
kubectl get pods --all-namespaces | grep -i multus
kubectl logs -n kube-system kube-multus-ds-zqbnz
##################################################################################
# Configure net-attach-def
cat <<EOF | kubectl apply -f -
apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
name: nadbr0
spec:
config: '{"cniVersion":"0.3.1","name":"br0","plugins":[{"type":"bridge","bridge":"br0","ipam":{}},{"type":"tuning"}]}'
EOF
kubectl get net-attach-def -oyaml
##################################################################################
# Deploy simple pod network NAT Virtual Machine (working)
kubectl apply -f https://gist.githubusercontent.com/usrbinkat/500a2d7314d938f6582f36059a7c1b29/raw/f6ccbce3461424531a23c70fd55002521b0104ac/virtualmachine-jammy-pod-nat.yml
kubectl get events -w
virtctl console jammy
kubectl delete vm jammy
watch kubectl get po -A
##################################################################################
# Deploy multus bridged network NAT Virtual Machine (failing)
# Create user ssh key credential secret
ls ~/.ssh/id_rsa.pub >/dev/null || ssh-keygen
kubectl create secret generic kubevirt-sshpubkey-kc2user --from-file=key1=$HOME/.ssh/id_rsa.pub --dry-run=client -oyaml | kubectl apply -f -
# Deploy VM
kubectl apply -f https://gist.githubusercontent.com/usrbinkat/500a2d7314d938f6582f36059a7c1b29/raw/f6ccbce3461424531a23c70fd55002521b0104ac/virtualmachine-jammy-multus-bridge.yml
kubectl get events -w
... truncated ...
Warning SyncFailed virtualmachineinstance/jammy failed to configure vmi network: setup failed, err: failed plugging phase1 at nic 'net1': Link not found
... truncated ...