All-in-one script to run Xen on qemu-system-aarch64

xen-qemu-aarch64.sh

#!/bin/bash -ex

# Copyright (c) 2018, DornerWorks, Ltd.
# Author: Stewart Hildebrand

# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
# following conditions are met:

# 1.   Redistributions of source code must retain the above copyright notice, this list of conditions and the following
# disclaimer.

# THIS SOFTWARE IS PROVIDED BY DORNERWORKS ''AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
# DORNERWORKS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

WRKDIR=$(pwd)/
NPROC=$(nproc)

USERNAME=dornerworks
PASSWORD=dornerworks
SALT=dw
HASHED_PASSWORD=$(perl -e "print crypt(\"${PASSWORD}\",\"${SALT}\");")

sudo apt install zlib1g-dev libglib2.0-dev libpixman-1-dev libssl-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu pkg-config-aarch64-linux-gnu iasl curl qemu-utils qemu-user-static git build-essential kpartx flex bison device-tree-compiler u-boot-tools automake autoconf

if [ ! -d qemu ]; then
    git clone git://git.qemu.org/qemu.git
fi

if [ ! -d xen ]; then
    git clone git://xenbits.xen.org/xen.git
fi

if [ ! -d linux ]; then
    git clone --depth 1 --branch linux-4.14.y git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git linux
fi

# Download Ubuntu Base file system (https://wiki.ubuntu.com/Base)
ROOTFSURL=http://cdimage.ubuntu.com/ubuntu-base/releases/18.04.1/release/
ROOTFS=ubuntu-base-18.04.1-base-arm64.tar.gz
if [ ! -s ${ROOTFS} ]; then
    curl -OLf ${ROOTFSURL}${ROOTFS}
fi

# Build qemu
cd qemu
mkdir -p build
cd build
if [ ! -f config.status ]; then
    ../configure --target-list=aarch64-softmmu
fi
if [ ! -s aarch64-softmmu/qemu-system-aarch64 ]; then
    make -j ${NPROC}
fi
cd ${WRKDIR}

# Build Xen
if [ ! -s ${WRKDIR}xen/xen/xen ]; then
    cd ${WRKDIR}xen
    if [ ! -s xen/.config ]; then
        make -C xen XEN_TARGET_ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- CONFIG_EARLY_PRINTK=pl011,0x09000000,115200 defconfig
        #make -C xen XEN_TARGET_ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- CONFIG_EARLY_PRINTK=pl011,0x09000000,115200 menuconfig
        #sed -i 's/# CONFIG_DEVICE_TREE_DEBUG is not set/CONFIG_DEVICE_TREE_DEBUG=y/' xen/.config
    fi
    make XEN_TARGET_ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- CONFIG_EARLY_PRINTK=pl011,0x09000000,115200 dist-xen -j ${NPROC}
    cd ${WRKDIR}
fi

# Build Linux
if [ ! -s ${WRKDIR}linux/arch/arm64/boot/Image ]; then
    cd ${WRKDIR}linux
    cp arch/arm64/configs/defconfig arch/arm64/configs/xenback_defconfig
    echo "CONFIG_XEN_BLKDEV_BACKEND=y" >> arch/arm64/configs/xenback_defconfig
    echo "CONFIG_XEN_NETDEV_BACKEND=y" >> arch/arm64/configs/xenback_defconfig
    make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- xenback_defconfig
    # Uncomment the following line if you wish to change the kernel configuration
    #make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- menuconfig
    echo "Building kernel. This takes a while. To monitor progress, open a new terminal and use \"tail -f buildoutput.log\""
    make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- -j ${NPROC} > ${WRKDIR}buildoutput.log 2> ${WRKDIR}buildoutput2.log
    cd ${WRKDIR}
fi


MNTRAMDISK=/mnt/ramdisk/
MNTROOTFS=/mnt/qemu-arm64-rootfs/
MNTBOOT=${MNTROOTFS}boot/
if [ ! -s qemuxen.img ]; then
    IMGFILE=${MNTRAMDISK}qemuxen.img
else
    IMGFILE=qemuxen.img
    IMG_EXISTS=yes
fi

unmountstuff () {
  sudo umount ${MNTROOTFS}proc || true
  sudo umount ${MNTROOTFS}dev/pts || true
  sudo umount ${MNTROOTFS}dev || true
  sudo umount ${MNTROOTFS}sys || true
  sudo umount ${MNTROOTFS}tmp || true
  sudo umount ${MNTBOOT} || true
  sudo umount ${MNTROOTFS} || true
}

mountstuff () {
  sudo mkdir -p ${MNTROOTFS}
  sudo mount ${LOOPDEVROOTFS} ${MNTROOTFS}
  sudo mkdir -p ${MNTBOOT}
  sudo mount ${LOOPDEVBOOT} ${MNTBOOT}
  sudo mount -o bind /proc ${MNTROOTFS}proc
  sudo mount -o bind /dev ${MNTROOTFS}dev
  sudo mount -o bind /dev/pts ${MNTROOTFS}dev/pts
  sudo mount -o bind /sys ${MNTROOTFS}sys
  sudo mount -o bind /tmp ${MNTROOTFS}tmp
}

finish () {
  cd ${WRKDIR}
  sudo sync
  unmountstuff
  sudo kpartx -dvs ${IMGFILE} || true
  sudo rmdir ${MNTROOTFS} || true
  mv ${IMGFILE} . || true
  sudo umount ${MNTRAMDISK} || true
  sudo rmdir ${MNTRAMDISK} || true
}

trap finish EXIT


sudo mkdir -p ${MNTRAMDISK}
sudo mount -t tmpfs -o size=3g tmpfs ${MNTRAMDISK}

if [ "${IMG_EXISTS}" != "yes" ]; then
    qemu-img create ${IMGFILE} 2G
    parted ${IMGFILE} --script -- mklabel msdos
    parted ${IMGFILE} --script -- mkpart primary fat32 2048s 264191s
    parted ${IMGFILE} --script -- mkpart primary ext4 264192s -1s
fi

LOOPDEVS=$(sudo kpartx -avs ${IMGFILE} | awk '{print $3}')
LOOPDEVBOOT=/dev/mapper/$(echo ${LOOPDEVS} | awk '{print $1}')
LOOPDEVROOTFS=/dev/mapper/$(echo ${LOOPDEVS} | awk '{print $2}')

if [ "${IMG_EXISTS}" != "yes" ]; then
    sudo mkfs.vfat ${LOOPDEVBOOT}
    sudo mkfs.ext4 ${LOOPDEVROOTFS}

    sudo fatlabel ${LOOPDEVBOOT} BOOT
    sudo e2label ${LOOPDEVROOTFS} QemuUbuntu

    sudo mkdir -p ${MNTROOTFS}
    sudo mount ${LOOPDEVROOTFS} ${MNTROOTFS}

    sudo tar -C ${MNTROOTFS} -xf ${ROOTFS}
    sudo umount ${MNTROOTFS}
fi

mountstuff

sudo cp `which qemu-aarch64-static` ${MNTROOTFS}usr/bin/

sudo cp ${WRKDIR}linux/arch/arm64/boot/Image ${MNTBOOT}

cd ${WRKDIR}linux
sudo make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- INSTALL_MOD_PATH=${MNTROOTFS} modules_install > ${WRKDIR}modules_install.log
cd ${WRKDIR}

# /etc/resolv.conf is required for internet connectivity in chroot. It will get overwritten by dhcp, so don't get too attached to it.
sudo chroot ${MNTROOTFS} bash -c 'echo "nameserver 8.8.8.8" > /etc/resolv.conf'
sudo chroot ${MNTROOTFS} bash -c 'echo "nameserver 2001:4860:4860::8888" >> /etc/resolv.conf'

if [ "${IMG_EXISTS}" != "yes" ]; then
    sudo sed -i -e "s/# deb /deb /" ${MNTROOTFS}etc/apt/sources.list
    sudo chroot ${MNTROOTFS} apt-get update
fi
# Install the dialog package and others first to squelch some warnings
sudo chroot ${MNTROOTFS} apt-get -y install dialog apt-utils
sudo chroot ${MNTROOTFS} apt-get -y upgrade
sudo chroot ${MNTROOTFS} apt-get -y install systemd systemd-sysv sysvinit-utils sudo udev rsyslog kmod util-linux sed netbase dnsutils ifupdown isc-dhcp-client isc-dhcp-common less vim net-tools iproute2 iputils-ping libnss-mdns iw software-properties-common ethtool dmsetup hostname iptables logrotate lsb-base lsb-release plymouth psmisc tar tcpd libsystemd-dev symlinks uuid-dev libc6-dev libncurses-dev libglib2.0-dev build-essential bridge-utils zlib1g-dev patch libpixman-1-dev libyajl-dev libfdt-dev libaio-dev

# Change the shared library symlinks to relative instead of absolute so they play nice with cross-compiling
sudo chroot ${MNTROOTFS} symlinks -c /usr/lib/aarch64-linux-gnu/

if [ "${IMG_EXISTS}" != "yes" ]; then
cd ${WRKDIR}xen

# TODO: --with-xenstored=oxenstored

# Ask the native compiler what system include directories it searches through.
SYSINCDIRS=$(echo $(sudo chroot ${MNTROOTFS} bash -c "echo | gcc -E -Wp,-v -o /dev/null - 2>&1" | grep "^ " | sed "s|^ /| -isystem${MNTROOTFS}|"))
SYSINCDIRSCXX=$(echo $(sudo chroot ${MNTROOTFS} bash -c "echo | g++ -x c++ -E -Wp,-v -o /dev/null - 2>&1" | grep "^ " | sed "s|^ /| -isystem${MNTROOTFS}|"))

LDFLAGS="-Wl,-rpath-link=${MNTROOTFS}lib/aarch64-linux-gnu -Wl,-rpath-link=${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
./configure \
    PYTHON_PREFIX_ARG=--install-layout=deb \
    --enable-systemd \
    --disable-xen \
    --enable-tools \
    --disable-docs \
    --disable-stubdom \
    --prefix=/usr \
    --with-xenstored=xenstored \
    --build=x86_64-linux-gnu \
    --host=aarch64-linux-gnu \
    CC="aarch64-linux-gnu-gcc --sysroot=${MNTROOTFS} -nostdinc ${SYSINCDIRS} -B${MNTROOTFS}lib/aarch64-linux-gnu -B${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
    CXX="aarch64-linux-gnu-g++ --sysroot=${MNTROOTFS} -nostdinc ${SYSINCDIRSCXX} -B${MNTROOTFS}lib/aarch64-linux-gnu -B${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
    PKG_CONFIG_PATH=${MNTROOTFS}usr/lib/aarch64-linux-gnu/pkgconfig:${MNTROOTFS}usr/share/pkgconfig

LDFLAGS="-Wl,-rpath-link=${MNTROOTFS}lib/aarch64-linux-gnu -Wl,-rpath-link=${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
make dist-tools \
    CROSS_COMPILE=aarch64-linux-gnu- XEN_TARGET_ARCH=arm64 \
    CC="aarch64-linux-gnu-gcc --sysroot=${MNTROOTFS} -nostdinc ${SYSINCDIRS} -B${MNTROOTFS}lib/aarch64-linux-gnu -B${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
    CXX="aarch64-linux-gnu-g++ --sysroot=${MNTROOTFS} -nostdinc ${SYSINCDIRSCXX} -B${MNTROOTFS}lib/aarch64-linux-gnu -B${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
    PKG_CONFIG_PATH=${MNTROOTFS}usr/lib/aarch64-linux-gnu/pkgconfig:${MNTROOTFS}usr/share/pkgconfig \
    QEMU_PKG_CONFIG_FLAGS=--define-variable=prefix=${MNTROOTFS}usr \
    -j $(nproc)

sudo --preserve-env PATH=${PATH} \
LDFLAGS="-Wl,-rpath-link=${MNTROOTFS}lib/aarch64-linux-gnu -Wl,-rpath-link=${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
make install-tools \
    CROSS_COMPILE=aarch64-linux-gnu- XEN_TARGET_ARCH=arm64 \
    CC="aarch64-linux-gnu-gcc --sysroot=${MNTROOTFS} -nostdinc ${SYSINCDIRS} -B${MNTROOTFS}lib/aarch64-linux-gnu -B${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
    CXX="aarch64-linux-gnu-g++ --sysroot=${MNTROOTFS} -nostdinc ${SYSINCDIRSCXX} -B${MNTROOTFS}lib/aarch64-linux-gnu -B${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
    PKG_CONFIG_PATH=${MNTROOTFS}usr/lib/aarch64-linux-gnu/pkgconfig:${MNTROOTFS}usr/share/pkgconfig \
    QEMU_PKG_CONFIG_FLAGS=--define-variable=prefix=${MNTROOTFS}usr \
    DESTDIR=${MNTROOTFS}

sudo chroot ${MNTROOTFS} systemctl enable xen-qemu-dom0-disk-backend.service
sudo chroot ${MNTROOTFS} systemctl enable xen-init-dom0.service
sudo chroot ${MNTROOTFS} systemctl enable xenconsoled.service
sudo chroot ${MNTROOTFS} systemctl enable xendomains.service
sudo chroot ${MNTROOTFS} systemctl enable xen-watchdog.service

cd ${WRKDIR}
fi

# It seems like the xen tools configure script selects a few too many of these backend driver modules, so we override it with a simpler list.
# /usr/lib/modules-load.d/xen.conf
cat > tmp-qemuxen-script-generated-xen.conf <<EOF
xen-evtchn
xen-gntdev
xen-gntalloc
xen-blkback
xen-netback
EOF
sudo cp tmp-qemuxen-script-generated-xen.conf ${MNTROOTFS}usr/lib/modules-load.d/xen.conf
rm tmp-qemuxen-script-generated-xen.conf

# /etc/hostname
HOSTNAME=ubuntu
sudo bash -c "echo ${HOSTNAME} > ${MNTROOTFS}etc/hostname"

# /etc/hosts
cat > tmp-qemuxen-script-generated-hosts <<EOF
127.0.0.1	localhost
127.0.1.1	${HOSTNAME}

# The following lines are desirable for IPv6 capable hosts
::1     ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters

EOF
sudo cp tmp-qemuxen-script-generated-hosts ${MNTROOTFS}etc/hosts
rm tmp-qemuxen-script-generated-hosts

# /etc/fstab
cat > tmp-qemuxen-script-generated-fstab <<EOF
proc            /proc           proc    defaults          0       0
/dev/vda1       /boot           vfat    defaults          0       2
/dev/vda2       /               ext4    defaults,noatime  0       1
EOF
sudo cp tmp-qemuxen-script-generated-fstab ${MNTROOTFS}etc/fstab
rm tmp-qemuxen-script-generated-fstab

# /etc/network/interfaces.d/eth0br0
cat > tmp-qemuxen-script-generated-interfaces <<EOF
auto eth0
iface eth0 inet manual

auto br0
iface br0 inet dhcp
    bridge_ports eth0
EOF
sudo cp tmp-qemuxen-script-generated-interfaces ${MNTROOTFS}etc/network/interfaces.d/eth0br0
rm tmp-qemuxen-script-generated-interfaces
sudo chmod 0600 ${MNTROOTFS}etc/network/interfaces.d/eth0br0

# User account setup
if [ "${IMG_EXISTS}" != "yes" ]; then
    sudo chroot ${MNTROOTFS} useradd -s /bin/bash -G adm,sudo -m -p ${HASHED_PASSWORD} ${USERNAME}
fi

LINUX_ADDR=0x40600000
KERNEL_SIZE=$(printf '0x%x\n' $(ls -l linux/arch/arm64/boot/Image | awk '{print $5}'))
DTBFILE=virt.dtb

QEMUOPTS="-gdb tcp::3333 \
-machine virt,gic_version=3 -machine virtualization=true \
-cpu cortex-a53 -smp 4 -m 2G -nographic \
-netdev user,id=net0,hostfwd=tcp::2222-:22 -device virtio-net-device,netdev=net0 \
-kernel ${WRKDIR}xen/xen/xen \
-device loader,file=${WRKDIR}linux/arch/arm64/boot/Image,addr=${LINUX_ADDR} \
-drive if=virtio,file=${IMGFILE},format=raw"

unmountstuff
${WRKDIR}qemu/build/aarch64-softmmu/qemu-system-aarch64 ${QEMUOPTS} -machine dumpdtb=${DTBFILE}

dtc -I dtb -O dts -o virt.dts ${DTBFILE}

patch -p1 <<EOF
diff --git a/virt.dts b/virt.dts
--- a/virt.dts
+++ b/virt.dts
@@ -385,5 +385,13 @@
 
 	chosen {
 		stdout-path = "/pl011@9000000";
+		xen,xen-bootargs = "console=dtuart dtuart=/pl011@9000000 dom0_mem=512M";
+		xen,dom0-bootargs = "console=hvc0 ignore_loglevel clk_ignore_unused root=/dev/vda2";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		module@0 {
+			compatible = "multiboot,module", "multiboot,kernel";
+			reg = <${LINUX_ADDR} ${KERNEL_SIZE}>;
+		};
 	};
 };
EOF

dtc -I dts -O dtb -o ${DTBFILE} virt.dts

${WRKDIR}qemu/build/aarch64-softmmu/qemu-system-aarch64 ${QEMUOPTS} -dtb ${DTBFILE}