iPXE Netboot.xyz with Docker

_ipxe_setup.md

iPXE with Netboot.xyz

Leverage Docker and Netboot.xyz to host Harvester/Rocky/Liberty installers and kickstart. We have been developing an Harvester Workshop and needed a better way to boot dozens of machines. Netboot.xyz was a simple way to do it. We used https://hub.docker.com/r/linuxserver/netbootxyz to simplify the install.

Then we created a custom menu and loaded the assets. After updating the DHCP settings on the network to point to the machine we are able to boot anything.

Netboot Docker docs : https://netboot.xyz/docs/docker

Deploy Netboot with Docker

• install docker
• create directories mkdir /opt/netboot/{config,assets}
• deploy with docker compose -f docker_compose.yaml up -d
• Add menu.ipxe to /opt/netboot/config/menus/menu.ipxe from below.
• Add assets to /opt/netboot/assets/ directory, aka unpack ISOs to directory.

update dhcpd

Set the DHCPD next-server with netboot.xyz.kpxe or netboot.xyz.efi for uefi.

look at attached files

Take a look at the tree output below. I truncated to show the important files.

Video

https://youtu.be/BGDiaouqSSg

basic_ks.yaml

timezone America/New_York --utc
text
%addon com_redhat_kdump --disable
%end
rootpw --iscrypted --allow-ssh $6$VmlNLErACH.u627t$YM8zJROIhS4CxoR7Z6BVQsLcpM3mhoBUqcYLg.jIiwLXdxqRo0C2d7ATe7ltNVxw8WRW5FE5BPV3d1Nf8I4aa.
network --bootproto=dhcp --device=eth0 --noipv6 --activate
keyboard --xlayouts='us'
lang en_US.UTF-8
url --url="http://192.168.1.220/liberty/BaseOS"
repo --name="AppStream" --baseurl="http://192.168.1.220/liberty/AppStream"

%packages
@^server-product-environment
%end

firstboot --disabled
ignoredisk --only-use=vda
bootloader --location=mbr --timeout=1
zerombr
clearpart --all --initlabel
autopart --type=plain --fstype=xfs --nohome

%packages
@^minimal-environment
sudo
qemu-guest-agent
openssh-server
-alsa*
-microcode_ctl
-iwl*firmware
-dracut-config-rescue
-plymouth
%end

reboot

docker_compose.yaml

services:
  netbootxyz:
    image: lscr.io/linuxserver/netbootxyz
    container_name: netbootxyz
    environment:
      - PUID=1000
      - PGID=1000
      - TZ=America/NewYork
    volumes:
      - /opt/netboot/config:/config
      - /opt/netboot/assets:/assets
    ports:
      - 3000:3000
      - 69:69/udp
      - 80:80
    restart: unless-stopped

harvester_config.yaml

schemeversion: 1
serverurl: ""
token: ilikechicken
os:
    afterinstallchrootcommands: []
    sshauthorizedkeys: []
    writefiles: []
    hostname: um790
    modules:
        - kvm
        - vhost_net
        - nvme
    sysctls:
      kernel.printk: "4 4 1 7"
      kernel.kptr_restrict: "1"
    ntpservers:
        - 0.suse.pool.ntp.org
    dnsnameservers:
        - 192.168.8.1
        - 1.1.1.1
    wifi: []
    password: Pa22word
    environment: {}
    labels:
      topology.kubernetes.io/zone: zone1
      foo: bar
    ssh_authorized_keys:
        - "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA26evmemRbhTtjV9szD9SwcFW9VOD38jDuJmyYYdqoqIltDkpUqDa/V1jxLSyrizhOHrlJtUOj790cxrvInaBNP7nHIO+GwC9VH8wFi4KG/TFj3K8SfNZ24QoUY12rLiHR6hRxcT4aUGnqFHGv2WTqsW2sxz03z+W1qeMqWYJOUfkqKKs2jiz42U+0Kp9BxsFBlai/WAXrQsYC8CcpQSRKdggOMQf04CqqhXzt5Q4Cmago+Fr7HcvEnPDAaNcVtfS5DYLERcX2OVgWT3RBWhDIjD8vYCMBBCy2QUrc4ZhKZfkF9aemjnKLfLcbdpMfb+r7NwJsVQSPKcjYAJOckE8RQ=="
install:
    automatic: true
    skipchecks: true
    mode: create
    managementinterface:
        interfaces:
            - name: enp1s0
              hwaddr: 58:47:ca:71:8f:a6
        method: static
        ip: 192.168.8.5
        subnetmask: 255.255.255.0
        gateway: 192.168.8.1
        defaultroute: true
        bondoptions:
            miimon: "100"
            mode: active-backup
        mtu: 0
        vlanid: 0
    vip: 192.168.8.6
    viphwaddr: ""
    vipmode: static
    forceefi: false
    device: /dev/nvme0n1
    silent: true
    isourl: http://192.168.8.11/harvester/harvester-v1.3.1-amd64.iso
    poweroff: false
    noformat: false
    debug: false
    tty: tty1
    forcegpt: true
    role: default
    withnetimages: false
    wipedisks: false
    forcembr: false
    datadisk: ""
    webhooks: []
    harvester:
        storage_class:
            replica_count: 1
        longhorn:
            defaultsettings:
                guaranteedEngineManagerCPU: 2
                guaranteedReplicaManagerCPU: 2
    persistentpartitionsize: 150Gi
    addons:
      rancher_monitoring:
        enabled: false
      rancher_logging:
        enabled: false
system_settings:
    auto-disk-provision-paths: ""
    #containerd-registry: '{"Mirrors": {"docker.io": {"Endpoints": ["https://myregistry.local:5000"]}}, "Configs": {"myregistry.local:5000": {"Auth": {"Username": "testuser", "Password": "testpassword"}, "TLS": {"InsecureSkipVerify": false}}}}'
    ui-source: bundled

harvester_setup.sh

#!/usr/bin/env bash
# harvester automation

export RED='\x1b[0;31m'
export GREEN='\x1b[32m'
export BLUE='\x1b[34m'
export YELLOW='\x1b[33m'
export NO_COLOR='\x1b[0m'

# variables
vip=192.168.8.6
longPassword=Pa22word1234#
shortPassword=Pa22word

# key pair
keypair="ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA26evmemRbhTtjV9szD9SwcFW9VOD38jDuJmyYYdqoqIltDkpUqDa/V1jxLSyrizhOHrlJtUOj790cxrvInaBNP7nHIO+GwC9VH8wFi4KG/TFj3K8SfNZ24QoUY12rLiHR6hRxcT4aUGnqFHGv2WTqsW2sxz03z+W1qeMqWYJOUfkqKKs2jiz42U+0Kp9BxsFBlai/WAXrQsYC8CcpQSRKdggOMQf04CqqhXzt5Q4Cmago+Fr7HcvEnPDAaNcVtfS5DYLERcX2OVgWT3RBWhDIjD8vYCMBBCy2QUrc4ZhKZfkF9aemjnKLfLcbdpMfb+r7NwJsVQSPKcjYAJOckE8RQ== [email protected]"

# set functions for debugging/logging
function info { echo -e "$GREEN[info]$NO_COLOR $1" ;  }
function warn { echo -e "$YELLOW[warn]$NO_COLOR $1" ; }
function fatal { echo -e "$RED[error]$NO_COLOR $1" ; exit 1 ; }
function info_ok { echo -e "$GREEN" "ok" "$NO_COLOR" ; }

#better error checking
command -v curl >/dev/null 2>&1 || { fatal "Curl was not found. Please install" ; }
command -v jq >/dev/null 2>&1 || { fatal "Jq was not found. Please install" ; }
command -v kubectl >/dev/null 2>&1 || { fatal "Kubectl was not found. Please install" ; }

info " - waiting for harvester "
until curl -skf -m 1 --output /dev/null https://$vip/v3-public ; do sleep 1; echo -n  ".";done
sleep 60
info_ok

info " - setting long password"
token=$(curl -sk -X POST https://$vip/v3-public/localProviders/local?action=login -H 'content-type: application/json' -d '{"username":"admin","password":"admin"}' | jq -r .token)

curl -sk https://$vip/v3/users?action=changepassword -H 'content-type: application/json' -H "Authorization: Bearer $token" -d '{"currentPassword":"admin","newPassword":"'$longPassword'"}'
info_ok

# reauthenticate
api_token=$(curl -sk https://$vip/v3/token -H 'content-type: application/json' -H "Authorization: Bearer $token" -d '{"type":"token","description":"automation"}' | jq -r .token)
 
info " - getting kubeconfig"
curl -sk https://$vip/v1/management.cattle.io.clusters/local?action=generateKubeconfig -H "Authorization: Bearer $api_token" -X POST -H 'content-type: application/json' | jq -r .config > $vip.yaml
export KUBECONFIG=$vip.yaml
info_ok

# load password length, image, network, keypair and template
info " - configuring password length, images, network, and keypair"
cat <<EOF | kubectl apply -f -  > /dev/null 2>&1
apiVersion: management.cattle.io/v3
kind: Setting
metadata:
  name: password-min-length
  namespace: cattle-system
value: "8"
---
apiVersion: harvesterhci.io/v1beta1
kind: VirtualMachineImage
metadata:
  name: rocky94
  annotations:
    harvesterhci.io/storageClassName: harvester-longhorn
  labels:
    harvesterhci.io/image-type: raw_qcow2
    harvesterhci.io/os-type: rocky
  namespace: default
spec:
  displayName: rocky94
  retry: 3
  sourceType: download
  storageClassParameters:
    migratable: 'true'
    numberOfReplicas: '3'
    staleReplicaTimeout: '30'
  url: https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2
---
apiVersion: harvesterhci.io/v1beta1
kind: VirtualMachineImage
metadata:
  name: noble
  annotations:
    harvesterhci.io/storageClassName: harvester-longhorn
  labels:
    harvesterhci.io/image-type: raw_qcow2
    harvesterhci.io/os-type: ubuntu
  namespace: default
spec:
  displayName: noble
  retry: 3
  sourceType: download
  storageClassParameters:
    migratable: 'true'
    numberOfReplicas: '3'
    staleReplicaTimeout: '30'
  url: https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-amd64.img
---

apiVersion: harvesterhci.io/v1beta1
kind: KeyPair
metadata:
  name: keypair
  namespace: default
spec:
  publicKey: $keypair

---

apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
  annotations:
    network.harvesterhci.io/route: '{"mode":"auto","serverIPAddr":"","cidr":"","gateway":""}'
  name: vlan1
  namespace: default
spec:
  config: '{"cniVersion":"0.3.1","name":"vlan1","type":"bridge","bridge":"mgmt-br","promiscMode":true,"ipam":{}}'
EOF

# delete if you want the long password
# password shortener - one last time
token=$(curl -sk -X POST https://$vip/v3-public/localProviders/local?action=login -H 'content-type: application/json' -d '{"username":"admin","password":"'$longPassword'"}' | jq -r .token)

api_token=$(curl -sk https://$vip/v3/token -H 'content-type: application/json' -H "Authorization: Bearer $token" -d '{"type":"token","description":"automation"}' | jq -r .token)

curl -sk https://$vip/v3/users?action=changepassword -H 'content-type: application/json' -H "Authorization: Bearer $token" -d '{"currentPassword":"'$longPassword'","newPassword":"'$shortPassword'"}'

menu.ipxe

#!ipxe

:global_vars
# set location of custom netboot.xyz live http assets
set live_endpoint http://192.168.1.220 

:main_menu
clear menu
menu ipxe All The Things
item --gap Default:
item local ${space} Boot from local
item --gap Distributions:
item harvester ${space} Harvester Installer 1.3.0
item rocky ${space} Rocky - Graphical
item rocky_ks ${space} Rocky Basic KickStart
item liberty ${space} Liberty - Graphical
item liberty_ks ${space} Liberty Basic KickStart
choose --default ${menu} menu
echo ${cls}
goto ${menu} ||

:rocky
imgfree
kernel ${live_endpoint}/rocky/images/pxeboot/vmlinuz inst.repo=${live_endpoint}/rocky inst.graphical ip=dhcp ipv6.disable inst.geoloc=0 devfs=nomount initrd=initrd.magic
initrd ${live_endpoint}/rocky/images/pxeboot/initrd.img
boot

:rocky_ks
imgfree
kernel ${live_endpoint}/rocky/images/pxeboot/vmlinuz inst.repo=${live_endpoint}/rocky inst.ks=http://192.168.1.220/rocky/basic_ks.yaml ip=dhcp ipv6.disable inst.geoloc=0 devfs=nomount initrd=initrd.magic
initrd ${live_endpoint}/rocky/images/pxeboot/initrd.img
boot

:harvester
kernel ${live_endpoint}/harvester/harvester-v1.3.0-vmlinuz-amd64 ip=dhcp net.ifnames=1 console=tty1 rd.cos.disable root=live:${live_endpoint}/harvester/harvester-v1.3.0-rootfs-amd64.squashfs rd.noverifyssl harvester.install.iso_url=${live_endpoint}/harvester/harvester-v1.3.0-amd64.iso harvester.install.skipchecks=true install.harvester.longhorn.default_settings.guaranteedReplicaManagerCPU=2 install.harvester.longhorn.default_settings.guaranteedEngineManagerCPU=2
initrd ${live_endpoint}/harvester/harvester-v1.3.0-initrd-amd64
boot

:liberty
imgfree
kernel ${live_endpoint}/liberty/images/pxeboot/vmlinuz inst.repo=${live_endpoint}/liberty inst.graphical ip=dhcp ipv6.disable inst.geoloc=0 devfs=nomount initrd=initrd.magic
initrd ${live_endpoint}/liberty/images/pxeboot/initrd.img
boot

:liberty_ks
imgfree
kernel ${live_endpoint}/liberty/images/pxeboot/vmlinuz inst.repo=${live_endpoint}/liberty inst.ks=http://192.168.1.220/liberty/basic_ks.yaml ip=dhcp ipv6.disable inst.geoloc=0 devfs=nomount initrd=initrd.magic
initrd ${live_endpoint}/liberty/images/pxeboot/initrd.img
boot

:local
echo Booting from local disks ...
exit 1

no_docker.md

use rocky native

mkdir /opt/netboot
yum install tftp-server nginx -y
sed -i 's#/var/lib/tftpboot#/opt/netboot#g' /usr/lib/systemd/system/tftp.service
sed -i -e '0,/\/usr\/share\/nginx\/html;/{s//\/opt\/netboot;\n        location \/ { autoindex on; autoindex_exact_size off; }/}' -e '/sendfile/s/on/off/' /etc/nginx/nginx.conf
systemctl daemon-reload
systemctl enable tftp nginx --now

Then copy the following to /opt/netboot along with OS specific files.

[root@ipxe netboot]# tree -L 1
.
├── harvester
├── liberty
├── menu.ipxe
├── netboot.xyz-undionly.kpxe
├── netboot.xyz.efi
├── netboot.xyz.kpxe
└── rocky

tree.txt

[root@netboot netboot]# tree -L 3
.
├── assets
│   ├── harvester
│   │   ├── harvester-v1.3.0-amd64.iso
│   │   ├── harvester-v1.3.0-initrd-amd64
│   │   ├── harvester-v1.3.0-rootfs-amd64.squashfs
│   │   └── harvester-v1.3.0-vmlinuz-amd64
│   ├── liberty
│   │   ├── AppStream
│   │   ├── BaseOS
│   │   ├── EFI
│   │   ├── EULA
│   │   ├── RPM-GPG-KEY-SUSE_Liberty_Linux
│   │   ├── RPM-GPG-KEY-SUSE_Liberty_Linux_v3
│   │   ├── basic_ks.yaml
│   │   ├── extra_files.json
│   │   ├── images
│   │   ├── isolinux
│   │   └── media.repo
│   └── rocky
│       ├── AppStream
│       ├── BaseOS
│       ├── EFI
│       ├── LICENSE
│       ├── images
│       ├── isolinux
│       └── media.repo
├── config
│   ├── endpoints.yml
│   ├── log
│   │   └── nginx
│   ├── menus
│   │   ├── menu.ipxe
│   │   ├── netboot.xyz-undionly.kpxe
│   │   ├── netboot.xyz.efi
│   │   ├── netboot.xyz.kpxe
│   ├── menuversion.txt
│   └── nginx
│       ├── nginx.conf
│       └── site-confs
└── docker_compose.yaml