Firecracker in MacOS M3+

poc.sh

#!/usr/bin/env bash
set -eux

# Proof of concept: two Firecracker microVMs (Ubuntu + Alpine) boot inside a
# nested-KVM Lima VM on macOS, share an L2 bridge, and answer pings.
# Requires Apple Silicon M3+ and macOS 15+. Nested virtualization is NOT supported on M1/M2.

mise use -g lima@latest

limactl delete --force firecracker-poc 2>/dev/null || true
limactl start --name firecracker-poc \
  --vm-type vz \
  --nested-virt \
  --tty=false \
  template:ubuntu

limactl shell firecracker-poc bash <<'EOF'
set -eux
sudo test -r /dev/kvm   # nested KVM must be exposed to the guest (checked as root: sudo runs firecracker below too)

# limactl shell inherits the host cwd, which Lima mounts read-only -> curl (23).
mkdir -p "$HOME/fc-poc" && cd "$HOME/fc-poc"

sudo apt-get update -qq
sudo apt-get install -y -qq --no-install-recommends squashfs-tools e2fsprogs iproute2 iputils-ping

# ---------------------------------------------------------------- binaries
curl -fsSLO https://github.com/firecracker-microvm/firecracker/releases/download/v1.12.1/firecracker-v1.12.1-aarch64.tgz
tar xzf firecracker-v1.12.1-aarch64.tgz \
  --strip-components=1 \
  --wildcards '*/firecracker-v1.12.1-aarch64'

# Kernel from Firecracker's CI bucket. CONFIG_IP_PNP=y, so the `ip=` boot arg
# configures eth0 in-kernel -- ICMP works on any rootfs, no userspace netcfg.
curl -fsSLO https://s3.amazonaws.com/spec.ccfc.min/firecracker-ci/v1.12/aarch64/vmlinux-6.1.128

# ---------------------------------------------------------------- distro 1: Ubuntu 24.04
curl -fsSLO https://s3.amazonaws.com/spec.ccfc.min/firecracker-ci/v1.12/aarch64/ubuntu-24.04.squashfs
sudo unsquashfs -d squashfs-root ubuntu-24.04.squashfs

# Best-effort guest->guest proof: Ubuntu pings Alpine and shouts on the console.
sudo tee squashfs-root/etc/systemd/system/poc-ping.service >/dev/null <<'UNIT'
[Unit]
Description=POC peer ping
After=network.target
[Service]
Type=oneshot
ExecStart=/bin/sh -c 'ping -c 3 -W 2 172.16.0.3 && echo POC_GUEST_TO_GUEST_OK > /dev/ttyS0'
[Install]
WantedBy=multi-user.target
UNIT
sudo mkdir -p squashfs-root/etc/systemd/system/multi-user.target.wants
sudo ln -sf ../poc-ping.service squashfs-root/etc/systemd/system/multi-user.target.wants/poc-ping.service

truncate -s 800M ubuntu-24.04.ext4
sudo mkfs.ext4 -q -d squashfs-root -F ubuntu-24.04.ext4

# ---------------------------------------------------------------- distro 2: Alpine 3.21
curl -fsSLO https://dl-cdn.alpinelinux.org/alpine/v3.21/releases/aarch64/alpine-minirootfs-3.21.7-aarch64.tar.gz
mkdir -p alpine-root
sudo tar -xzf alpine-minirootfs-3.21.7-aarch64.tar.gz -C alpine-root
# minirootfs targets containers, so give it an init that just stays alive.
sudo tee alpine-root/sbin/poc-init >/dev/null <<'INIT'
#!/bin/sh
mount -t proc  proc /proc  2>/dev/null || true
mount -t sysfs sys  /sys   2>/dev/null || true
echo "POC_ALPINE_UP $(cat /etc/alpine-release 2>/dev/null)"
while :; do sleep 5; done
INIT
sudo chmod +x alpine-root/sbin/poc-init

truncate -s 200M alpine-3.21.ext4
sudo mkfs.ext4 -q -d alpine-root -F alpine-3.21.ext4

# ---------------------------------------------------------------- L2 bridge, one tap per VM
sudo ip link add br0 type bridge 2>/dev/null || true
sudo ip addr replace 172.16.0.1/24 dev br0
sudo ip link set br0 up
for t in tap0 tap1; do
  sudo ip tuntap add "$t" mode tap 2>/dev/null || true
  sudo ip link set "$t" master br0
  sudo ip link set "$t" up
done

write_config() {   # $1=file $2=rootfs $3=tap $4=ip $5=mac $6=mem $7=init
  cat > "$1" <<JSON
{
  "boot-source": {
    "kernel_image_path": "vmlinux-6.1.128",
    "boot_args": "console=ttyS0 reboot=k panic=1 pci=off ip=$4::172.16.0.1:255.255.255.0::eth0:off $7"
  },
  "drives": [{
    "drive_id": "rootfs",
    "path_on_host": "$2",
    "is_root_device": true,
    "is_read_only": false
  }],
  "network-interfaces": [{
    "iface_id": "eth0",
    "host_dev_name": "$3",
    "guest_mac": "$5"
  }],
  "machine-config": { "vcpu_count": 1, "mem_size_mib": $6 }
}
JSON
}

write_config vm-ubuntu.json ubuntu-24.04.ext4 tap0 172.16.0.2 06:00:AC:10:00:02 512 ""
write_config vm-alpine.json alpine-3.21.ext4  tap1 172.16.0.3 06:00:AC:10:00:03 256 "init=/sbin/poc-init"

# ---------------------------------------------------------------- boot both microVMs
sudo ./firecracker-v1.12.1-aarch64 --no-api --config-file vm-ubuntu.json >vm-ubuntu.log 2>&1 &
UBUNTU_PID=$!
sudo ./firecracker-v1.12.1-aarch64 --no-api --config-file vm-alpine.json >vm-alpine.log 2>&1 &
ALPINE_PID=$!

cleanup() {
  sudo pkill -f firecracker-v1.12.1-aarch64 || true
  sudo ip link del br0 2>/dev/null || true
  for t in tap0 tap1; do sudo ip link del "$t" 2>/dev/null || true; done
}
trap cleanup EXIT

sleep 25   # let both guests reach the point where the kernel has configured eth0

# ---------------------------------------------------------------- assertions (set -e = these gate the POC)
kill -0 "$UBUNTU_PID" && kill -0 "$ALPINE_PID"          # both VMMs still alive, i.e. no early exit
grep -q "Linux version" vm-ubuntu.log                    # guest kernel actually executed
grep -q "Linux version" vm-alpine.log
ping -c 3 -W 2 172.16.0.2                                # Ubuntu microVM answers
ping -c 3 -W 2 172.16.0.3                                # Alpine microVM answers

echo "PASS: two microVMs booted under nested KVM and both answer ICMP"
grep -q POC_ALPINE_UP vm-alpine.log && echo "PASS: Alpine userspace reached init"
grep -q POC_GUEST_TO_GUEST_OK vm-ubuntu.log \
  && echo "PASS: Ubuntu microVM pinged Alpine microVM" \
  || echo "note: guest->guest marker absent (needs ping in the Ubuntu rootfs); host->guest proof above still holds"
EOF

echo "POC done. Tear down with: limactl stop firecracker-poc && limactl delete firecracker-poc"