Learn how to use kubevirt with an example

kubevirt.md

01_Test_environment

kubernetes v1.31
cni : flannel v0.25.6
cri : cri-o v1.31

01_install_kubernetes

• (주의) hostname이 k8s-master로 변경됩니다.

wget https://gist.github.com/taking/024d5fdb7393b9ea1f1eb84c07cd992d/raw/49cbefd8dfd833e1c0cd47cf3fd3b9038adf40a8/k8s-v1.31-crio-auto-installation.sh
chmod +x k8s-v1.31-crio-auto-installation.sh
sudo ./k8s-v1.31-crio-auto-installation.sh k8s-master master

02_install_helm

curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

03_install-longhorn.sh

#!/bin/bash

# Usage: ./02_install-longhorn.sh install|uninstall

if [ "$#" -ne 1 ]; then
  echo "Usage: $0 {install|uninstall}"
  exit 1
fi

if [ "$1" == "install" ]; then
  echo "Installing Longhorn..."

  helm repo add longhorn https://charts.longhorn.io
  helm repo update longhorn

  mkdir -p /data/longhorn
  helm install longhorn longhorn/longhorn \
      --create-namespace \
      --namespace longhorn-system \
      --set defaultSettings.defaultDataPath="/data/longhorn" \
      --set defaultSettings.defaultDataLocality="best-effort" \
      --wait

  kubectl patch storageclass longhorn -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
  kubectl get sc
  echo "Longhorn installed."
  
elif [ "$1" == "uninstall" ]; then
  echo "Uninstalling Longhorn..."

  kubectl patch settings.longhorn.io orphan-auto-deletion -n longhorn-system --type=merge -p '{"value": "true"}'
  helm uninstall longhorn -n longhorn-system --wait
  
  rm -rf /data/longhorn/*

  echo "Longhorn uninstalled."
  
else
  echo "Invalid argument: $1"
  echo "Usage: $0 {install|uninstall}"
  exit 1
fi

04_install-kubevirt.sh

#!/bin/bash

# Usage: ./02_install-kubevirt.sh install|uninstall

export VERSION=$(basename $(curl -s -w %{redirect_url} https://github.com/kubevirt/kubevirt/releases/latest))

if [ "$#" -ne 1 ]; then
  echo "Usage: $0 {install|uninstall}"
  exit 1
fi

if [ "$1" == "install" ]; then
  echo "Installing KubeVirt version $VERSION..."
  kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-operator.yaml
  kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-cr.yaml
  echo "KubeVirt installed."
  
elif [ "$1" == "uninstall" ]; then
  echo "Uninstalling KubeVirt version $VERSION..."
  kubectl delete -f https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-cr.yaml
  kubectl delete -f https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-operator.yaml
  echo "KubeVirt uninstalled."
  
else
  echo "Invalid argument: $1"
  echo "Usage: $0 {install|uninstall}"
  exit 1
fi

05_install-cdi.sh

#!/bin/bash

# Usage: ./03_install-cdi.sh install|uninstall

export VERSION=$(basename $(curl -s -w %{redirect_url} https://github.com/kubevirt/containerized-data-importer/releases/latest))

if [ "$#" -ne 1 ]; then
  echo "Usage: $0 {install|uninstall}"
  exit 1
fi

if [ "$1" == "install" ]; then
  echo "Installing KubeVirt-CDI version $VERSION..."
  kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/$VERSION/cdi-operator.yaml
  kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/$VERSION/cdi-cr.yaml
  echo "KubeVirt-CDI installed."
  
elif [ "$1" == "uninstall" ]; then
  echo "Uninstalling KubeVirt-CDI version $VERSION..."
  kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/$VERSION/cdi-operator.yaml
  kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/$VERSION/cdi-cr.yaml
  echo "KubeVirt-CDI uninstalled."
  
else
  echo "Invalid argument: $1"
  echo "Usage: $0 {install|uninstall}"
  exit 1
fi

06_install-virtctl.sh

#!/bin/bash

# Usage: ./04_install-virtctl.sh install|uninstall

export VERSION=$(curl -s https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt)
export UNAME=$(uname | sed 's/^Darwin$/darwin/; s/^Linux$/linux/')
export ARCH=$(uname -m | sed 's/^aarch64$/arm64/; s/^x86_64$/amd64/')

if [ "$#" -ne 1 ]; then
  echo "Usage: $0 {install|uninstall}"
  exit 1
fi

if [ "$1" == "install" ]; then
  echo "Installing Virtctl version $VERSION..."
  wget https://github.com/kubevirt/kubevirt/releases/download/$VERSION/virtctl-$VERSION-$UNAME-$ARCH
  sudo mv virtctl-$VERSION-$UNAME-$ARCH /usr/local/bin/virtctl
  sudo chmod +x /usr/local/bin/virtctl
  sudo rm -f virtctl-$VERSION-$UNAME-$ARCH
  
  virtctl version
  echo "Virtctl installed."
  
elif [ "$1" == "uninstall" ]; then
  echo "Uninstalling Virtctl version $VERSION..."

  VIRTCTL_PATH=$(which virtctl 2>/dev/null)
  if [ -n "$VIRTCTL_PATH" ]; then
    sudo rm -f "$VIRTCTL_PATH"
  fi

  echo "Virtctl uninstalled."
  
else
  echo "Invalid argument: $1"
  echo "Usage: $0 {install|uninstall}"
  exit 1
fi

07_kubevirt_option.sh

#!/bin/bash

# Usage: ./06_kubevirt_option.sh

echo "patching KubeVirt FeatureGates..."

# featureGates list - https://github.com/kubevirt/kubevirt/blob/main/pkg/virt-config/feature-gates.go#L26
cat <<EOF | kubectl apply -f -
apiVersion: kubevirt.io/v1
kind: KubeVirt
metadata:
  name: kubevirt
  namespace: kubevirt
spec:
  configuration:
    developerConfiguration:
      featureGates:
        - Root
        - DataVolumes
        - HostDisk
        - PersistentReservation
        - VMPersistentState
        - GPU
EOF

echo "KubeVirt FeatureGates Patched."

08_install-h5ai.sh

#!/bin/bash

# Usage: ./05_install-h5ai.sh install|uninstall

if [ "$#" -ne 1 ]; then
  echo "Usage: $0 {install|uninstall}"
  exit 1
fi

if [ "$1" == "install" ]; then
    echo "Installing h5ai..."

    cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    taking.service: h5ai
  name: h5ai
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      taking.service: h5ai
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        taking.service: h5ai
    spec:
      containers:
        - env:
            - name: PUID
              value: "1000"
            - name: PGID
              value: "1000"
            - name: TZ
              value: "Asia/Seoul"
          image: genomehubs/h5ai:latest
          name: h5ai
          imagePullPolicy: Always
          ports:
            - containerPort: 8080
              protocol: TCP
          volumeMounts:
            - mountPath: '/var/www/html/images'
              name: storage-volume
      restartPolicy: Always
      volumes:
        - name: storage-volume
          hostPath:
            type: DirectoryOrCreate
            path: '/mnt/data'
---
apiVersion: v1
kind: Service
metadata:
  name: h5ai
  namespace: default
  labels:
    taking.service: h5ai
spec:
  selector:
    taking.service: h5ai
  ports:
    - protocol: TCP
      port: 8080
      targetPort: 8080
      nodePort: 31111
  type: NodePort
EOF

    echo "h5ai installed."

elif [ "$1" == "uninstall" ]; then
    echo "Uninstalling h5ai..."

    # Delete the deployment and service
    kubectl delete deployment h5ai
    kubectl delete service h5ai

    echo "h5ai uninstalled."

else
  echo "Invalid argument: $1"
  echo "Usage: $0 {install|uninstall}"
  exit 1
fi

09_download_ubuntu-24_04.sh

#!/bin/bash

if [ "$(id -u)" != "0" ]; then
  echo "Run as root."
  exit 1
fi

if [ -f /mnt/data/Ubuntu-22_04-LTS-Noble_Numbat.img ]; then
  echo "Ubuntu-22_04-LTS-Noble_Numbat.img exists."
else
  wget https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img -O /mnt/data/Ubuntu-22_04-LTS-Noble_Numbat.img
  
  chmod +x /mnt/data/*
fi

10_ubuntu-2404-container.sh (ubuntu / ubuntu)

# ubuntu 24.04 + container
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Namespace
metadata:
  name: kubevirt-ubuntu
  labels:
    pod-security.kubernetes.io/enforce: privileged
---
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
  name: vm-ubuntu
  namespace: kubevirt-ubuntu
spec:
  running: true
  template:
    metadata:
      labels:
        kubevirt.io/size: small
        kubevirt.io/domain: ubuntu
    spec:
      domain:
        devices:
          disks:
            - name: containerdisk
              disk:
                bus: virtio
            - name: cloudinitdisk
              disk:
                bus: virtio
          interfaces:
          - name: default
            masquerade: {}
        resources:
          requests:
            memory: 2096M
      networks:
      - name: default
        pod: {}
      volumes:
        - name: containerdisk
          containerDisk:
            image: quay.io/containerdisks/ubuntu:24.04
        - name: cloudinitdisk
          cloudInitNoCloud:
            userData: |-
              #cloud-config
              password: ubuntu
              chpasswd: { expire: False }
---
apiVersion: v1
kind: Service
metadata:
  name: vm-ubuntu-ssh
  namespace: kubevirt-ubuntu
spec:
  ports:
  - name: ssh
    port: 22
    protocol: TCP
    targetPort: 22
  selector:
    kubevirt.io/size: small
    kubevirt.io/domain: ubuntu
  type: NodePort
EOF

11_ubuntu-2404-dataVolume.sh (ubuntu / ubuntu)

• List of matrix supported by CDI

# ubuntu 24.04 + Containerized-Data-Importer (CDI)
cat <<EOF | kubectl apply -f -
---
apiVersion: v1
kind: Namespace
metadata:
  name: kubevirt-ubuntu
  labels:
    pod-security.kubernetes.io/enforce: privileged
---
apiVersion: cdi.kubevirt.io/v1beta1
kind: DataVolume
metadata:
  name: vm-ubuntu-dv
  namespace: kubevirt-ubuntu
spec:
  source:
    http:
        url: "https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img"
  storage:
    resources:
      requests:
        storage: 10Gi
---
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
  name: vm-ubuntu
  namespace: kubevirt-ubuntu
spec:
  running: true
  template:
    metadata:
      labels:
        kubevirt.io/size: small
        kubevirt.io/domain: ubuntu-2404
    spec:
      domain:
        devices:
          disks:
            - name: rootdisk
              disk:
                bus: virtio
            - name: cloudinitdisk
              disk:
                bus: virtio
          interfaces:
          - name: default
            masquerade: {}
        resources:
          requests:
            memory: 2096M
      networks:
      - name: default
        pod: {}
      volumes:
        - name: rootdisk
          dataVolume:
            name: vm-ubuntu-dv
        - name: cloudinitdisk
          cloudInitNoCloud:
            userData: |-
              #cloud-config
              password: ubuntu
              chpasswd: { expire: False }
              ssh_pwauth: True
              disable_root: true
              pacakge_update:  true
              packages:
              - qemu-guest-agent
              runcmd:
              - [ systemctl, start, qemu-guest-agent ]
---
apiVersion: v1
kind: Service
metadata:
  name: vm-ubuntu-ssh
  namespace: kubevirt-ubuntu
spec:
  ports:
  - name: ssh
    port: 22
    protocol: TCP
    targetPort: 22
  selector:
    kubevirt.io/size: small
    kubevirt.io/domain: ubuntu-2404
  type: NodePort
EOF

DataVolume :: spec.source.registry.url

apiVersion: cdi.kubevirt.io/v1beta1
kind: DataVolume
metadata:
  name: vm-ubuntu-dv
  namespace: kubevirt-ubuntu
spec:
  source:
    registry:
        url: "docker://quay.io/containerdisks/ubuntu:24.04"
  storage:
    resources:
      requests:
        storage: 10Gi

12_rocky-9-dataVolume.sh (rocky / rocky)

• List of matrix supported by CDI

# rocky 9 / dataVolume
cat <<EOF | kubectl apply -f -
---
apiVersion: v1
kind: Namespace
metadata:
  name: kubevirt-rocky
  labels:
    pod-security.kubernetes.io/enforce: privileged
---
apiVersion: cdi.kubevirt.io/v1beta1
kind: DataVolume
metadata:
  name: vm-rocky-dv
  namespace: kubevirt-rocky
spec:
  source:
    http:
        url: "http://h5ai.default.svc.cluster.local:8080/images/Rocky-9-GenericCloud-Base.x86_64.qcow2"
  storage:
    resources:
      requests:
        storage: 10Gi
---
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
  name: vm-rocky
  namespace: kubevirt-rocky
spec:
  running: true
  template:
    metadata:
      labels:
        kubevirt.io/size: small
        kubevirt.io/domain: rocky-9
    spec:
      domain:
        devices:
          disks:
            - name: rootdisk
              disk:
                bus: virtio
            - name: cloudinitdisk
              disk:
                bus: virtio
          interfaces:
          - name: default
            masquerade: {}
        resources:
          requests:
            memory: 2096M
      networks:
      - name: default
        pod: {}
      volumes:
        - name: rootdisk
          dataVolume:
            name: vm-rocky-dv
        - name: cloudinitdisk
          cloudInitNoCloud:
            userData: |-
              #cloud-config
              password: rocky
              chpasswd: { expire: False }
              ssh_pwauth: True
              disable_root: true
              pacakge_update:  true
              packages:
              - qemu-guest-agent
              runcmd:
              - [ systemctl, start, qemu-guest-agent ]
---
apiVersion: v1
kind: Service
metadata:
  name: vm-rocky-ssh
  namespace: kubevirt-rocky
spec:
  ports:
  - name: ssh
    port: 22
    protocol: TCP
    targetPort: 22
  selector:
    kubevirt.io/size: small
    kubevirt.io/domain: rocky-9
  type: NodePort
EOF

13_windows-11-dv-templates-url.sh (admin / admin)

# windows 11 / dataVolumeTemplates / url
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Namespace
metadata:
  name: kubevirt-win11
  labels:
    pod-security.kubernetes.io/enforce: privileged
---
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
  labels:
    kubevirt.io/os: windows
  name: vm-win11
  namespace: kubevirt-win11
spec:
  running: true
  template:
    metadata:
      labels:
        kubevirt.io/vm: win-11
    spec:
      terminationGracePeriodSeconds: 0
      domain:
        cpu:
          cores: 4
        clock:
          timezone: "Asia/Seoul"
        #machine:
        #  type: q35
        resources:
          requests:
            cpu: 4
            memory: 8192M
        #features:
        #  smm: {}
        #firmware:
        #  bootloader:
        #    efi:
        #      secureBoot: false
        #    #efi:
        #    #  persistent: true
        #    #  secureBoot: true
        devices:
          autoattachGraphicsDevice: true
          #tpm: {}
            #persistent: true
          disks:
            - bootOrder: 1
              disk:
                bus: virtio
              name: datavolumedisk1
            #- disk:
            #    bus: virtio
            #  name: host-disk
            - disk:
                bus: virtio
              name: cloudinitdisk
      volumes:
        - dataVolume: #Note the type is dataVolume
            name: vm-win11-dv
          name: datavolumedisk1
        - cloudInitNoCloud:
            userData: |
              # add any custom logic you want to occur on startup here.
              echo "cloud-init script execution"
          name: cloudinitdisk
  dataVolumeTemplates: # Automatically a PVC of size 2Gi is created
    - metadata:
        name: vm-win11-dv
        namespace: kubevirt-win11
      spec:
        pvc:
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 80Gi
        source: #This is the source where the ISO file resides
          http:
            url: http://h5ai.default.svc.cluster.local:8080/images/Windows-11_21H2-PRO-70G.qcow2
---
apiVersion: v1
kind: Service
metadata:
  name: vm-win11-rdp
  namespace: kubevirt-win11
  labels:
    kubevirt.io/vm: win-11
spec:
  externalTrafficPolicy: Cluster
  ports:
  - name: rdp
    port: 3389
    protocol: TCP
    targetPort: 3389
  selector:
    kubevirt.io/vm: win-11
  type: NodePort
---
apiVersion: v1
kind: Service
metadata:
  name: vm-win11-vnc
  namespace: kubevirt-win11
  labels:
    kubevirt.io/vm: win-11
spec:
  externalTrafficPolicy: Cluster
  ports:
  - port: 5900
    protocol: TCP
    targetPort: 5900
    name: vnc
  selector:
    kubevirt.io/vm: win-11
  type: NodePort
EOF

99_kubevirt-clean-up.sh

# the namespace of kubevirt installation
export namespace=kubevirt
export labels=("operator.kubevirt.io" "operator.cdi.kubevirt.io" "kubevirt.io" "cdi.kubevirt.io")
export namespaces=(default ${namespace} "<other-namespaces that has kubevirst resources>")

kubectl -n ${namespace} delete kv kubevirt
kubectl -n ${namespace} patch kv kubevirt --type=json -p '[{ "op": "remove", "path": "/metadata/finalizers" }]'
kubectl get vmis --all-namespaces -o=custom-columns=NAME:.metadata.name,NAMESPACE:.metadata.namespace,FINALIZERS:.metadata.finalizers --no-headers | grep foregroundDeleteVirtualMachine | while read p; do
    arr=($p)
    name="${arr[0]}"
    namespace="${arr[1]}"
    kubectl patch vmi $name -n $namespace --type=json -p '[{ "op": "remove", "path": "/metadata/finalizers" }]'
done

for i in ${namespaces[@]}; do
    for label in ${labels[@]}; do
        kubectl -n ${i} delete deployment -l ${label}
        kubectl -n ${i} delete ds -l ${label}
        kubectl -n ${i} delete rs -l ${label}
        kubectl -n ${i} delete pods -l ${label}
        kubectl -n ${i} delete services -l ${label}
        kubectl -n ${i} delete pvc -l ${label}
        kubectl -n ${i} delete rolebinding -l ${label}
        kubectl -n ${i} delete roles -l ${label}
        kubectl -n ${i} delete serviceaccounts -l ${label}
        kubectl -n ${i} delete configmaps -l ${label}
        kubectl -n ${i} delete secrets -l ${label}
        kubectl -n ${i} delete jobs -l ${label}
    done
done

for label in ${labels[@]}; do
    kubectl delete validatingwebhookconfiguration -l ${label}
    kubectl delete pv -l ${label}
    kubectl delete clusterrolebinding -l ${label}
    kubectl delete clusterroles -l ${label}
    kubectl delete customresourcedefinitions -l ${label}
    kubectl delete scc -l ${label}
    kubectl delete apiservices -l ${label}

    kubectl get apiservices -l ${label} -o=custom-columns=NAME:.metadata.name,FINALIZERS:.metadata.finalizers --no-headers | grep foregroundDeletion | while read p; do
        arr=($p)
        name="${arr[0]}"
        kubectl -n ${i} patch apiservices $name --type=json -p '[{ "op": "remove", "path": "/metadata/finalizers" }]'
    done
done

TroubleShoot

• Turn on the Virtualization Technology (VT) feature on the motherboard.

medge@medgew01:~$ sudo /usr/sbin/kvm-ok
INFO: /dev/kvm does not exist
HINT: sudo modprobe kvm_intel
INFO: Your CPU supports KVM extensions
INFO: KVM (vmx) is disabled by your BIOS
HINT: Enter your BIOS setup and enable Virtualization Technology (VT),
      and then hard poweroff/poweron your system
KVM acceleration can NOT be used

• If you're not in a KVM environment, use a soft emulator

# https://github.com/kubevirt/kubevirt/blob/main/docs/software-emulation.md
kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}'