kvm.sh

#!/bin/bash

set -e

verlte() {
	[[ "$1" == "$(echo -e "$1\n$2" | sort -V | head -n1)" ]]
}

verlt() {
	[[ "$1" == "$2" ]] && return 1 || verlte $1 $2
}

image_format() {
	IMAGE_FORMAT=raw
	IMAGE_FORMAT_FILE=$(file "${DISK_PATH}" 2>/dev/null || echo not)
	if [[ $IMAGE_FORMAT_FILE =~ .*QCOW.* ]]; then
		IMAGE_FORMAT=qcow2
	fi
	echo $IMAGE_FORMAT
}

QEMU_BINARY=/usr/bin/qemu-system-x86_64
#QEMU_BINARY=$HOME/Development/qemu/x86_64-softmmu/qemu-system-x86_64
SEED=0
CPUS=2
KERNEL=( )
MEMORY=2048
MEMORY_FLAGS=( -m $MEMORY )
NUMA_NODES=0
DISK_BUS_TYPE=1 # 0 = IDE, 1 = AHCI, 2 = virtio-scsi, 3 = virtio-blk, 4 = virtio-blk data-plane, 5 = NVMe
NIC_TYPE=0 # 0 = rtl8139, 1 = e1000, 2 = virtio
SCSI_BUS=1
DISK_IDX=0
GRAPHICS_FLAGS=( -vga vmware )
INPUT_FLAGS=( -usbdevice tablet )
EXTRA_FLAGS=( )
BALLOON=( )
NIC=( )
DISK=( )
DISK_PATHS=( )
RTC_FLAGS=( )
ENABLE_VNC=1
CDROM=( )
CPU_FLAGS=( -cpu host )
SERIAL=( -monitor stdio )

macaddr() {
	hexchars="0123456789abcdef"
	end=
	RANDOM=$SEED
	for i in {1..3} ; do
		RND=$RANDOM
		end=${end}:$(echo -n ${hexchars:$(( $RND % 16 )):1})

		RND=$RANDOM
		end=${end}$(echo -n ${hexchars:$(( $RND % 16 )):1})
	done
	echo 00:1c:42$end
}

TEMP=`getopt -o b:C:c:i:k:m:n:r:agosvw \
	--long apple,gui,serial,virtio,windows,blockif:,block:,cdrom:,cpus:,index:,kernel:.memory:,nic:,numa-ndoes:,ramdisk: \
	-- "$@"`

if [ $? -ne 0 ]; then
	echo "Invalid flags"
	exit 1
fi

eval set -- "$TEMP"

while true; do
	case "$1" in
		-a | --apple)
			# "a" for "Apple"
			CPU_FLAGS=( -cpu core2duo )
			EXTRA_FLAGS+=(
				-smbios type=2
				-device isa-applesmc,osk="ourhardworkbythesewordsguardedpleasedontsteal(c)AppleComputerInc"
			)
			INPUT_FLAGS=( -usb -device usb-mouse -device usb-kbd )
			DISK_BUS_TYPE=1
			shift
			;;
		-i | --index)
			# "i" for "index"
			SEED=$2
			shift 2
			;;
		-b | --block )
			# "b" for "block"
			DISK_PATHS+=( "$2" )
			shift 2
			;;
		-C | --cpus)
			# "C" for "CPU count"
			CPUS=$2
			shift 2
			;;
		-k | --kernel)
			# "k" for "kernel"
			KERNEL=( -kernel "$2" -append "loglevel=6 root=/dev/sda console=ttyS0,115200n8 earlyprintk=verbose,serial,ttyS0" )
			shift 2
			;;
		-r | --ramdisk)
			# "r" for "ramdisk"
			INITRD=( -initrd "$2" )
			shift 2
			;;
		-m | --memory)
			# "m" for "memory"
			MEMORY=$2
			shift 2
			;;
		-n | --numa-nodes)
			# "n" for "NUMA"
			NUMA_NODES=$2
			shift 2
			;;
		-g | --gui)
			ENABLE_VNC=0
			shift
			;;
		-c | --cdrom)
			# "c" for "cdrom"
			CDROM+=( -drive "file=$2,if=none,id=isocd.$SCSI_BUS,format=raw" )
			CDROM+=( -device "ide-cd,bus=ide.$SCSI_BUS,drive=isocd.$SCSI_BUS,id=cd-disk$SCSI_BUS,unit=0" )
			SCSI_BUS=$(( $SCSI_BUS + 1 ))
			shift 2
			;;
		--nic)
			NIC_TYPE=$2
			shift 2
			;;
		--blockif)
			DISK_BUS_TYPE=$2
			shift 2
			;;
		-v | --virtio)
			DISK_BUS_TYPE=2
			NIC_TYPE=2
			shift
			;;
		-s | --serial)
			SERIAL=( -nographic )
			shift
			;;
		-w | --windows)
			# virtio-scsi hangs on install, most recently with
			# virtio-win-0.1.113. Enforce using AHCI.
			DISK_BUS_TYPE=1

			# Kernels before 4.5.x don't have hv-synic or hv-stimer
			HV_STIMER=,hv-synic,hv-stimer
			if verlt "$(uname -r)" "4.5.0"; then
				HV_STIMER=
			fi

			CPU_FLAGS=( -cpu host,hv-spinlocks=0xffff,hv-vapic,hv-time,hv-relaxed,hv-vpindex,hv-runtime,hv-crash,hv-reset$HV_STIMER )
			RTC_FLAGS=( -rtc base=localtime )
			shift
			;;
		--)
			shift
			break
			;;
		*)
			echo "Unknown argument '$1'"
			exit 1
			;;
	esac
done

MACADDR=$(macaddr)
if [ $ENABLE_VNC -ne 0 ]; then
	GRAPHICS_FLAGS+=( -vnc 0.0.0.0:$SEED )
fi

echo "Attaching NIC with MAC address ${MACADDR}..."

MEMORY_FLAGS=(
	-m $MEMORY
)

if [ "$NUMA_NODES" -lt 1 ]; then
	NUMA_NODES=1
	if [ "$(cat /proc/sys/vm/nr_hugepages)" -gt 0 ]; then
		MEMORY_FLAGS+=(
			-mem-path /dev/hugepages
		)
	else
		echo always | sudo tee /sys/kernel/mm/transparent_hugepage/enabled >/dev/null
	fi
else
	MEM_DIVIDED=$(( $MEMORY / $NUMA_NODES ))
	CPUS_PER_NODE=$(( $CPUS / $NUMA_NODES ))

	if [ "$CPUS_PER_NODE" -lt 1 ]; then
		echo "Not enough CPUs for NUMA count $NUMA_NODES, set with -C $NUMA_NODES or higher"
		exit 1
	fi

	HOST_NUMA_NODES=$(lscpu | awk '/^NUMA node\(s\):/ { print $3 }')

	NODE=0
	S=0
	E=$(( $CPUS_PER_NODE - 1 ))
	for CPU in $(seq 1 $NUMA_NODES); do
		# e.g.
		# -object memory-backend-file,mem-path=/dev/hugepages,size=131072M,id=ram-node0,host-nodes=0,policy=bind
		# -numa node,nodeid=0,memdev=ram-node0,cpus=0-31
		# -object memory-backend-file,mem-path=/dev/hugepages,size=131072M,id=ram-node1,host-nodes=1,policy=bind
		# -numa node,nodeid=1,memdev=ram-node1,cpus=32-63
		# [...]
		if [ $NODE -lt $HOST_NUMA_NODES ]; then
			MEMBIND=",host-nodes=${NODE},policy=bind"
		fi
		MEMORY_FLAGS+=(
			-object memory-backend-file,mem-path=/dev/hugepages,size=${MEM_DIVIDED}M,id=ram-node${NODE}${MEMBIND}
			-numa node,nodeid=${NODE},memdev=ram-node${NODE},cpus=${S}-${E}
		)
		NODE=$(( $NODE + 1 ))
		S=$(( $E + 1 ))
		E=$(( $S + $CPUS_PER_NODE - 1 ))
	done
fi

SOCKETS=$NUMA_NODES
THREADS=2
CORES=$(( $CPUS / $SOCKETS / $THREADS ))
if [ $CPUS -eq 1 ]; then
	THREADS=1
	CORES=1
fi
if [ $CORES -lt 1 ]; then
	echo "Not enough CPUS"
	exit 1
fi
CPU_FLAGS+=(
	-smp sockets=$SOCKETS,cores=$CORES,threads=$THREADS
)

case $DISK_BUS_TYPE in
	0) # IDE
		DISK+=(
			#-device ahci,id=ide
			-device piix4-ide,id=ide0
		)
		for DISK_PATH in "${DISK_PATHS[@]}"; do
			IMAGE_FORMAT=$(image_format "${DISK_PATH}")
			DISK+=( -drive "id=hdd.$DISK_IDX,file=$DISK_PATH,format=$IMAGE_FORMAT,if=none,cache=none,aio=native,discard=unmap" )
			#DISK+=( -drive "id=hdd.$DISK_IDX,file=$DISK_PATH,if=ide,discard=unmap" )
			DISK+=( -device "ide-drive,bus=ide0.$DISK_IDX,drive=hdd.$DISK_IDX" )
			DISK_IDX=$(( $DISK_IDX + 1 ))
		done
		;;
	1) # AHCI
		DISK+=(
			-device "ioh3420,bus=pcie.0,addr=1c.0,multifunction=on,port=1,chassis=1,id=root.1"
			-device "ahci,bus=pcie.0,id=ahci0,multifunction=on"
		)
		for DISK_PATH in "${DISK_PATHS[@]}"; do
			IMAGE_FORMAT=$(image_format "${DISK_PATH}")
			DISK+=( -drive "id=hdd.$DISK_IDX,file=$DISK_PATH,format=$IMAGE_FORMAT,if=none,bus=$DISK_IDX,cache=none,aio=native,discard=unmap" )
			DISK+=( -device "ide-hd,bus=ahci0.$DISK_IDX,drive=hdd.$DISK_IDX" )
			DISK_IDX=$(( $DISK_IDX + 1 ))
		done
		;;
	2) # virtio-scsi
		DISK+=( -device "virtio-scsi-pci,id=scsi0" )
		for DISK_PATH in "${DISK_PATHS[@]}"; do
			IMAGE_FORMAT=$(image_format "${DISK_PATH}")
			DISK+=( -drive "id=hdd.$DISK_IDX,file=$DISK_PATH,format=$IMAGE_FORMAT,if=none,cache=none,aio=native,discard=unmap" )
			DISK+=( -device "scsi-hd,bus=scsi0.$DISK_IDX,drive=hdd.$DISK_IDX" )
			DISK_IDX=$(( $DISK_IDX + 1 ))
		done
		;;
	3) # virtio-blk
		for DISK_PATH in "${DISK_PATHS[@]}"; do
			IMAGE_FORMAT=$(image_format "${DISK_PATH}")
			DISK+=( -drive "id=hdd.$DISK_IDX,file=$DISK_PATH,format=$IMAGE_FORMAT,if=none,cache=none,aio=native,discard=unmap" )
			DISK+=( -device "virtio-blk-pci,drive=hdd.$DISK_IDX,scsi=off,config-wce=off,request-merging=off" )
			DISK_IDX=$(( $DISK_IDX + 1 ))
		done
		;;
	4) # virtio-blk data plane
		for DISK_PATH in "${DISK_PATHS[@]}"; do
			IMAGE_FORMAT=$(image_format "${DISK_PATH}")
			DISK+=( -object "iothread,id=iothread.$DISK_IDX" )
			DISK+=( -drive "id=hdd.$DISK_IDX,file=$DISK_PATH,format=$IMAGE_FORMAT,if=none,cache=none,aio=native,discard=unmap" )
			DISK+=( -device "virtio-blk-pci,drive=hdd.$DISK_IDX,scsi=off,config-wce=off,iothread=iothread.$DISK_IDX" )
			DISK_IDX=$(( $DISK_IDX + 1 ))
		done
		;;
	5) # NVMe
		for DISK_PATH in "${DISK_PATHS[@]}"; do
			IMAGE_FORMAT=$(image_format "${DISK_PATH}")
			DISK+=( -drive "id=hdd.$DISK_IDX,file=$DISK_PATH,format=$IMAGE_FORMAT,if=none,cache=none,aio=native,discard=unmap" )
			DISK+=( -device "nvme,drive=hdd.$DISK_IDX,serial=HDD$DISK_IDX" )
			DISK_IDX=$(( $DISK_IDX + 1 ))
		done
		;;
	*)
		echo "Unknown disk bus type"
		exit 1
		;;
esac

case $NIC_TYPE in
	0) # rtl8139
		NIC=(
			-netdev tap,id=net0
			-device rtl8139,netdev=net0,id=vnet0,addr=5,mac=$MACADDR
		)
		;;
	1) # e1000
		NIC=(
			-netdev tap,id=net0
			-device e1000-82545em,netdev=net0,id=vnet0,addr=5,mac=$MACADDR
		)
		;;
	2) # virtio
		NIC=(
			-netdev tap,id=net0
			-device virtio-net-pci,netdev=net0,id=vnet0,mac=$MACADDR
		)
		;;
	*)
		echo "Unknown NIC type"
		exit 1
		;;
esac

BALLOON=( -balloon none )

set -x

sudo setcap cap_net_admin=ep $QEMU_BINARY

[ -d /dev/hugepages ] && sudo chmod 1777 /dev/hugepages

exec $QEMU_BINARY \
	-M q35 \
	-global isa-fdc.driveA= \
	-enable-kvm \
	"${CPU_FLAGS[@]}" \
	"${RTC_FLAGS[@]}" \
	"${SERIAL[@]}" \
	"${MEMORY_FLAGS[@]}" \
	-parallel none \
	"${INPUT_FLAGS[@]}" \
	"${BALLOON[@]}" \
	"${EXTRA_FLAGS[@]}" \
	"${KERNEL[@]}" \
	"${INITRD[@]}" \
	"${DISK[@]}" \
	"${NIC[@]}" \
	"${CDROM[@]}" \
	"${GRAPHICS_FLAGS[@]}" \
	-boot cd \
	-no-reboot