Skip to content

Instantly share code, notes, and snippets.

@AfroThundr3007730

AfroThundr3007730/zfs-module-build.service

Last active March 4, 2026 12:51
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save AfroThundr3007730/341e10a368e2c099da6c451632585086 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save AfroThundr3007730/341e10a368e2c099da6c451632585086 to your computer and use it in GitHub Desktop.
Download ZIP
Build binary zfs-module packages for all kernel packages
Raw
zfs-module-build.service
[Unit]
Description=Build binary ZFS module packages and publish repo
RequiresMountsFor=/mnt/pool0/data/app /mnt/pool0/mirror/repo
[Service]
Type=exec
Restart=no
TimeoutSec=10
ExecStart=/mnt/pool0/data/app/repo/debian-zfs/bin/zfs-module-build -y
EnvironmentFile=-/mnt/pool0/data/app/repo/debian-zfs/etc/environment
EnvironmentFile=-%h/.config/systemd/user/zfs-module-build.env
Raw
zfs-module-build.sh
#!/bin/bash
# Build binary zfs-module packages for all kernel packages
# Version 0.10.8 modified 2026-03-03 by AfroThundr
# SPDX-License-Identifier: GPL-3.0-or-later
# For issues or updated versions of this script, browse to the following URL:
# https://gist.github.com/AfroThundr3007730/341e10a368e2c099da6c451632585086
# Run in strict mode
set -eEuo pipefail
shopt -s extdebug extglob nullglob
#----------------------------------------------------------------------#
# MARK: Top Matter
#----------------------------------------------------------------------#
# Print program usage text
print_usage() {
local b='\e[1m' e='\e[1;2m' n='\e[22m' l=${_LE} _______=''
say -h "
${l}$(sed -n "s/# //; s/$/${_LE}/; 2,4p" "$0")
${l}
${l}Usage: ${b}${_ME} (-v | -h | -b | -s | -y)${n}
${l}
${l}Options:
${l}
${l} ${b}-b${n} Run build job queue (internal)
${l} ${b}-h${n} Display script usage information
${l} ${b}-s${n} Run repo signing jobs (internal)
${l} ${b}-v${n} Emit version header and details
${l} ${b}-y${n} Confirm to start the build
${l}
${l}Metadata variables: Default values that may be overridden
${l}
${l} * ${b}LOGFILE${n} File to write script output to
${l} $_______ Default: ${e}${LOGFILE:-(none)}${n}
${l} * ${b}PURGEDAYS${n} Max age of stamp before purging
${l} $_______ Default: ${e}${PURGEDAYS:-(none)}${n}
${l} ${b}MEMLIMIT${n} Set container max memory limit
${l} $_______ Default: ${e}${MEMLIMIT:-(none)}${n}
${l} ${b}ENGINE${n} Container engine to use (in PATH)
${l} $_______ Default: ${e}${ENGINE:-(none)}${n}
${l} ${b}IMAGE${n} Registry URI for base build image
${l} $_______ Default: ${e}${IMAGE:-(none)}${n}
${l} * ${b}MIRROR${n} Repository mirror to check for metadata
${l} $_______ Default: ${e}${MIRROR:-(none)}${n}
${l} ${b}APPDIR${n} Application root directory to map in
${l} $_______ Default: ${e}${APPDIR:-(none)}${n}
${l} ${b}PUBDIR${n} Repository root directory to map in
${l} $_______ Default: ${e}${PUBDIR:-(none)}${n}
${l}
${l}Control variables: Flags enabled if non-empty (some have overloads)
${l}
${l} * ${b}QUIET${n} Suppress output to terminal
${l} * ${b}RETRY${n} Re-attempt build (ignore build stamp)
${l} * ${b}REBUILD${n} Rerun build (ignore built package)
${l} * ${b}STAMPONLY${n} Only refresh timestamps (no build)
${l} * ${b}SRCPULL${n} Force repull source tarballs
${l} * ${b}NOBUILD${n} Skip the build job queue completely
${l} * ${b}BATCH${n} Run multiple concurrent builds per kernel
${l} * ${b}NOPURGE${n} Skip the stale package purge step
${l} * ${b}NOPUBLISH${n} Skip the publish job queue completely
${l} * ${b}REPUBLISH${n} Regenerate repository manifests and release
${l} $_______ Overload: all - republish all architectures
${l} * ${b}EXTRAS${n} Force regenerate extrafiles manifest
${l} ${b}REPULL${n} Force repull container images
${l} ${b}PULLONLY${n} Only pull container images (no build)
${l} ${b}USEQEMU${n} Use QEMU emulation instead of crossbuilding
${l} ${b}SETUP${n} Rerun environment setup even if already present
${l} ${b}OVERWRITE${n} Overwrite script executable if different
${l}
${l}Override variables: Provided as a space-separated list of values
${l}
${l} * ${b}XKVER_LIST${n} List of kernel versions to build
${l} * ${b}XPVER_LIST${n} List of module versions to build
${l} ${b}XRELS_LIST${n} List of release names to build on
${l} ${b}XARCH_LIST${n} List of architectures to build on
${l}
${l}Note: Variables marked with a '*' are passed into the container.
${l}
${l}Note: Override list variables will be validated against values
${l} retrieved from the upstream repository mirrors.
${l}
${l}Note: QEMU emulation is typically slower than cross-compiling and
${l} requires the 'qemu-user-binfmt' package to be installed.
${l}
${l}$(sed -n "s/# //; s/$/${_LE}/; 6,7p" "$0")
"
}
# Print program version
print_version() { say -h "$(sed -n "s/# //; s/$/${_LE}/; 3p" "$0")"; }
# Silent pushd
pushd() { command pushd "$@" >/dev/null; }
# Silent popd
popd() { command popd >/dev/null; }
# Silent apt
qapt() { command apt "$@" &>/dev/null; }
# Print formatted message
say() {
local c1='' c2='' end='\n' fd=1
while (($#)); do
case $1 in
-e) c1='\e[31;1;2m' c2='\e[m' fd=2 && shift ;;
-h) c1='\e[34m' c2='\e[m' && shift ;;
-n) end='' && shift ;;
*) break ;;
esac
done
(($#)) || return 0
# shellcheck disable=SC2059
[[ ${LOGFILE:-} ]] && set -- "${1//\\e+([[0-9;m])/}" "${@:2}" &&
printf -- "${1}${end}" "${@:2}" >>"${LOGFILE}" 2>&- || :
[[ ${QUIET:-} ]] && return
[[ ${_LE} ]] && set -- "${1//\\n/\\r\\n}" "${@:2}"
# shellcheck disable=SC2059
printf -- "${c1}${1}${end:+${_LE}}${end}${c2}" "${@:2}" >&"${fd}" 2>&- || :
}
# Set global variables
set_globals() {
. /etc/os-release
# Executable Script
_ME=${0##*/} _ME=${_ME%.sh}
# Line Ending Format
[[ -t 1 && ! $(stty) =~ -opost ]] && _LE='' || _LE='\r'
# Native Architecture
_ARCH=$(dpkg --print-architecture 2>&- || uname -m)
# Time Output Format
printf -v TIMEFORMAT '%%0lR%b' "${_LE}"
# Locale Override
LC_ALL=C
# Canonical Architecture
CARCH=${CARCH:-${_ARCH}}
# Target Host Architecture
HARCH=${HARCH:-${_ARCH}}
# Debian Component for Package
COMPONENT=contrib
# Binary Package Prefix
MODNAME=zfs-modules
# Source Package Prefix
SRCNAME=zfs-linux
# Release Codename
CNAME=${CNAME:-${VERSION_CODENAME}}
# Internal Data Root
BASEDIR=/srv/zfs
# Build Root
BUILDDIR=/tmp/build
# Repository Root
REPODIR=/srv/repo
# Signing Helper
SIGNER=/usr/local/sbin/module-sign
# Stripping Helper
STRIP=/usr/local/sbin/strip
# Apt Sources File
SOURCES=/etc/apt/sources.list.d/debian.sources
# Location of Distribution Files
DISTDIR=${BASEDIR}/dist/${CNAME}
# Location of Build Log Files
LOGDIR=${BASEDIR}/logs/${CNAME}
# Location of Prepared Source Archives
SRCDIR=${BASEDIR}/source/${CNAME}
# Location of Timestamp Files
STAMPDIR=${BASEDIR}/stamp/${CNAME}
# Repository Relative POOL Path
POOLDIR=pool/${COMPONENT}/${SRCNAME:0:1}/${SRCNAME}
# Required Dependencies for Building
BUILDDEPS=(debhelper fakeroot initramfs-tools lsb-release)
# Reqiored Dependencies for Publication
REPODEPS=(apt-utils ca-certificates curl gawk gnupg xz-utils)
# Output Log File
LOGFILE=${LOGFILE:-}
# Max Age Before Purging
PURGEDAYS=${PURGEDAYS:-30}
# Max Memory Limit
MEMLIMIT=${MEMLIMIT:-1G}
# Container Engine
ENGINE=${ENGINE:-podman}
# Container Registry URI
IMAGE=${IMAGE:-docker.io/library/debian}
# Repository Mirror URL
MIRROR=${MIRROR:-https://deb.debian.org/debian}
# Application Root Directory
APPDIR=${APPDIR:-/opt/debian-zfs}
# Repository Root Directory
PUBDIR=${PUBDIR:-/srv/debian-zfs}
declare -gr \
_ME _LE _ARCH TIMEFORMAT LC_ALL \
CARCH HARCH COMPONENT MODNAME SRCNAME CNAME \
BASEDIR BUILDDIR REPODIR SIGNER STRIP SOURCES \
DISTDIR LOGDIR SRCDIR STAMPDIR POOLDIR \
BUILDDEPS REPODEPS
declare -grx \
LOGFILE PURGEDAYS \
MEMLIMIT ENGINE IMAGE MIRROR APPDIR PUBDIR \
QUIET RETRY REBUILD STAMPONLY SRCPULL NOBUILD \
BATCH NOPURGE NOPUBLISH REPUBLISH EXTRAS \
REPULL PULLONLY USEQEMU SETUP OVERWRITE
declare -gx \
XKVER_LIST XPVER_LIST XRELS_LIST XARCH_LIST
declare -gA TAINTED
}
#----------------------------------------------------------------------#
# MARK: Package Build
#----------------------------------------------------------------------#
# Setup build environment
setup_builder() {
say '[+] Release: %s (%s) | Arch: %s | Component: %s' \
"${NAME}" "${VERSION_CODENAME}" "${HARCH}" "${COMPONENT}"
[[ "${CARCH}" == "${_ARCH}" && "${HARCH}" == "${_ARCH}" ]] &&
say '[+] Native building: %s -> %s' "${CARCH}" "${HARCH}"
[[ "${CARCH}" == "${_ARCH}" ]] ||
say '[+] QEMU emulating: %s -> %s' "${CARCH}" "${_ARCH}"
[[ "${HARCH}" == "${_ARCH}" ]] ||
say '[+] Cross compiling: %s -> %s' "${_ARCH}" "${HARCH}"
sed -i 's/: deb/& deb-src/g; s/: main/& contrib/g' "${SOURCES}"
sed -i "s/${CNAME}-updates/& ${CNAME}-backports/g" "${SOURCES}"
declare -grx DEBIAN_FRONTEND=nonintractive
dpkg --add-architecture "${HARCH}" && qapt update && qapt upgrade -y
mkdir -p "${BUILDDIR}" "${LOGDIR}" "${SRCDIR}" "${STAMPDIR}" \
"${DISTDIR}"{,-backports} "${REPODIR}/${POOLDIR}"
install -m 0755 /dev/null "${SIGNER}"
# shellcheck disable=SC2016
printf >"${SIGNER}" '#!/bin/bash\nshopt -s nullglob
SIGNER=$(find /usr/lib/linux-kbuild-* -type f -name sign-file -print -quit)
for i in $1/{,*/}*.ko; do ${SIGNER} sha256 /etc/keys/db.{key,crt} $i; done'
install -m 0755 /dev/null "${STRIP}"
# shellcheck disable=SC2016
printf >"${STRIP}" '#!/bin/bash\nshopt -s nullglob
for i in $1/{,*/}*.ko; do ${DEB_HOST_GNU_TYPE}-strip -g $i; done'
}
# Get version lists
get_versions() {
mapfile -t KVER_LIST < <(
apt list -a 'linux-headers-*' 2>&- | awk -F'[/ ]' -v arch="${HARCH}" \
'$1 ~ /-[0-9]+\./ && $4 ~ arch {print substr($1, 15) ":" $3}'
)
mapfile -t PVER_LIST < <(
apt-cache showsrc "${SRCNAME}" 2>&- | awk '/^Version:/ {print $2}'
)
[[ ! ${XKVER_LIST[*]:-} ]] || mapfile -t KVER_LIST < <(
read -ra XKVER_LIST <<<"${XKVER_LIST[*]}"
for kver in "${KVER_LIST[@]}"; do
for xkver in "${XKVER_LIST[@]}"; do
[[ "${kver%:*}" == "${xkver}" ]] && printf '%s\n' "${kver}"
done
done
) && declare -gr KVER_LIST && unset XKVER_LIST
[[ ! ${XPVER_LIST[*]:-} ]] || mapfile -t PVER_LIST < <(
read -ra XPVER_LIST <<<"${XPVER_LIST[*]}"
for pver in "${PVER_LIST[@]}"; do
for xpver in "${XPVER_LIST[@]}"; do
[[ "${pver}" == "${xpver}" ]] && printf '%s\n' "${pver}"
done
done
) && declare -gr PVER_LIST && unset XPVER_LIST
}
# Get CC mapping
get_ccmap() {
local abi=gnu arch host
case ${HARCH} in
amd64) arch=x86-64 ;;
arm64) arch=aarch64 ;;
armel) arch=arm abi=gnueabi ;;
armhf) arch=arm abi=gnueabihf ;;
i386) arch=i686 ;;
loong64) arch=loongarch64 ;;
mips64el) abi=gnuabi64 ;;
ppc64el) arch=powerpc64le ;;
esac
host=${arch:-${HARCH}}-linux-${abi}
mapfile -t PKG < <(
apt-cache depends gcc gcc-"${host}" |
awk '/: gcc-[0-9]/ {g = $2} /: libc6/ {l = $2} END {print g "\n" l}'
)
declare -grx PKG CC=/usr/bin/${host/x86-64/x86_64}-${PKG[0]%-"${host}"}
}
# Check disk usage
check_usage() {
local ssize=200 bsize=600 bavail savail bmount smount pkgs
read -rd '\n' bavail bmount savail smount < <(
df -BM "${BUILDDIR}" /usr/src |
awk '$4 ~ /[0-9]+M/ {gsub(/M/, ""); print $4 "\n" $6}'
) || :
mapfile -t pkgs < <(
find /usr/src -maxdepth 1 -type d -name '*-common*' -print | sort -V
) && ((${#pkgs[@]})) && unset 'pkgs[${#pkgs[@]}-1]'
([[ $bmount != "$smount" ]] || ((bavail >= bsize + ssize)) &&
((bavail >= bsize && savail >= ssize)) && ((${#pkgs[@]} == 0))) || {
say '[-] Clearing space for build (purging headers).'
[[ ${_hpkg:-} ]] && qapt remove -y "${_hpkg%%_*}"
qapt remove -y "${pkgs[@]##*/}"
} || {
say -e '[!] Unable to clear enough space for build.' && _die 1
}
}
# Check build preconditions
check_build() {
local file rc stub=${STAMPDIR}/${srcstub}.dsc
[[ ! -f ${STAMPDIR}/${deb} || ${RETRY:-} || ${REBUILD:-} ]] || rc=1
[[ ! -f ${out}/${deb} || ${REBUILD:-} ]] || rc=1
[[ ! ${STAMPONLY:-} ]] || rc=1
[[ -f ${stub} ]] &&
while read -r file; do touch "${STAMPDIR}/${file}"; done <"${stub}"
[[ -f ${STAMPDIR}/${deb} ]] && touch "${STAMPDIR}/${deb}"
return "${rc:-0}"
}
# Install build dependencies
install_deps() {
[[ ${installed:-} ]] && return 0 || installed=1
say '[*] Installing build dependencies.'
qapt install -y --no-install-recommends "${PKG[@]}" "${BUILDDEPS[@]}"
[[ ${VERSION_ID:-} -ne 12 ]] ||
qapt install -y --no-install-recommends "linux-base=$(
apt list -a linux-base 2>&- | awk 'NF == 3 {print $2; exit}'
)"
}
# Prepare source archive
prepare_source() {
local file out stub=${STAMPDIR}/${srcstub}.dsc
[[ -f ${stub} && -f ${tar} && ! ${SRCPULL:-} ]] && return 0
[[ ${SRCPULL:-} && -f ${BUILDDIR}/repull:${srcstub} ]] && return
[[ ${SRCPULL:-} ]] && : >"${BUILDDIR}/repull:${srcstub}"
[[ $pver =~ bpo ]] && out=${DISTDIR}-backports || out=${DISTDIR}
say '[*] Retrieving sources for: %s %s' "${SRCNAME}" "${pver}"
pushd "${BUILDDIR}"
qapt source "${SRCNAME}=${pver}"
for file in "${SRCNAME}"_*; do
: >>"${STAMPDIR}/${file}" && printf '%s\n' "${file}" >>"${stub}"
done
cp "${SRCNAME}"_* "${out}" && TAINTED[${out##*/}]+=s && rm -f "${SRCNAME}"_*
# shellcheck disable=SC2016
sed -i '/module modules/a\\tstrip $(CURDIR)/module' "${src}/debian/rules"
# shellcheck disable=SC2016
sed -i '/strip \$/a\\tmodule-sign $(CURDIR)/module' "${src}/debian/rules"
# shellcheck disable=SC2016
sed -i 's/with-linux-obj=$(KOBJ)/& \\\n\t\t--host=$(DEB_HOST_GNU_TYPE)/g' \
"${src}/debian/rules"
tar cf - -C "${src}" . | gzip -9 >"${tar}" && rm -fr "${src}"
popd # ${BUILDDIR}
}
# Install kernel headers
install_headers() {
local common kbuild
[[ -d /usr/src/linux-headers-${kver} ]] && return 0
say '[*] Installing linux headers for: %s' "${kver}"
IFS=: read -r common kbuild < <(
apt-cache depends linux-headers-"${kver}" |
awk '/common/ {c = $NF} /kbuild/ {k = $NF} END {print c ":" k}'
)
pushd "${BUILDDIR}"
[[ ${_hpkg:-} ]] && qapt remove -y "${_hpkg%%_*}"
declare -g _hpkg=linux-headers-${kver}_${kpver}_${HARCH}.deb
qapt install -y --no-install-recommends "${common}" "${kbuild}"
qapt download linux-headers-"${kver}" &&
dpkg -i --force-all "${_hpkg}" &>/dev/null && rm -f "./${_hpkg}"
popd # ${BUILDDIR}
declare -gx KOBJ=/usr/src/linux-headers-"${kver}" KSRC=/usr/src/"${common}"
}
# Build kernel modules
build_modules() {
say '[*] [%03d] Building: %s | Kernel: %s | Package: %s' \
"${id}" "${MODNAME}" "${kver}" "${pver}"
pushd "${BUILDDIR}"
rm -fr "${build}" && mkdir "${build}" && tar xf "${tar}" -C "${build}"
pushd "${build}"
dpkg-architecture -c fakeroot nice -n 19 ionice -c 3 debian/rules \
override_dh_binary-modules 1>"${log1}" 2>"${log2}" || {
: >"${STAMPDIR}/${deb}" && popd # ${build}
say -e '[!] [%03d] Errors during build, check logs:\n--- [%03d] %s' \
"${id}" "${id}" "${log2}"
gzip -9f "${log1}" "${log2}" && rm -fr "${build}" && return 0
} && {
: >"${STAMPDIR}/${deb}" && popd # ${build}
say '[+] [%03d] Package build completed successfully.' "${id}"
cp "${deb}" "${out}" && : >"taint:${out##*/}"
gzip -9f "${log1}" "${log2}" && rm -fr "${build}" "${deb}"
apt info "${out}/${deb}" 2>&- | awk -v id="${id}" -v l="${_LE}" \
'{printf "--- [%03d] %s%s\n", id, $0, l} /^Description/ {exit}'
}
popd # ${BUILDDIR}
}
# Remove unneeded headers
prune_headers() {
((id)) && say '[+] All module package builds complete.'
((id)) || say '[-] No module package builds occurred.'
qapt purge -y linux-{image,headers,kbuild}-'*' && qapt autoremove -y
}
# Purge expired packages
clean_packages() {
[[ ${NOPURGE:-} ]] && return 0
local stamps stamp file dir queue count
mapfile -t stamps < <(
find "${STAMPDIR}" -type f -mtime +"${PURGEDAYS}" -printf '%f\n' -delete
) && ((${#stamps[@]})) || return 0
say '[*] Purging stale module packages.'
for stamp in "${stamps[@]}"; do
say '--- Purging stale package: %s' "${stamp##*/}"
queue=() count=0
for file in "${DISTDIR}"{,-backports}/"${stamp}"; do
queue+=("${file}") dir=${file%/*} dir=${dir##*/}
case ${file##*.} in
deb) TAINTED[dir]+=b ;;
dsc) TAINTED[dir]+=s ;;
esac
done
for _ in "${BASEDIR}"/dist/*/"${stamp}"; do ((++count)); done
((count == 1)) && queue+=("${REPODIR}/${POOLDIR}/${stamp}")
case ${stamp##*.} in
deb) queue+=("${LOGDIR}/${stamp}".{1,2}.log.gz) ;;
dsc) queue+=("${SRCDIR}/${stamp%.*}".tar.gz) ;;
esac
rm -f "${queue[@]}"
done
say ' [+] Purging stale packages complete.'
}
# Publish packages to repo
publish_packages() {
for _ in "${BUILDDIR}"/taint:*; do TAINTED[${_#*:}]+=b; done
[[ ${!TAINTED[*]} || ${REPUBLISH:-} ]] && local candidates=(
"${DISTDIR}"{,-backports}/{"${SRCNAME}"_*,"${MODNAME}"-*"${HARCH}"*}
) && ((${#candidates[@]})) || return 0
say '[+] Copying new packages to pool.'
[[ $(stat -c %g "${BASEDIR}") -eq 65534 ]] ||
find "${BASEDIR}" ! -user 1000 -print0 | xargs -0r chown 1000 2>&-
cp -nt "${REPODIR}/${POOLDIR}" "${candidates[@]}"
}
# Run package build jobs
build_packages() {
[[ ${NOBUILD:-} ]] && return 0
local id=0 kver kpver pver
local verstub srcstub deb src tar build log1 log2 out
say '=== Begin ZFS package build job queue.'
setup_builder
get_versions
get_ccmap
pushd "${BASEDIR}"
for kver in "${KVER_LIST[@]}"; do
kpver=${kver#*:} kver=${kver%:*}
declare -gx KVERS=${kver}
check_usage
for pver in "${PVER_LIST[@]}"; do
verstub=${kver}_${pver}_${HARCH}
srcstub=${SRCNAME}_${pver}
deb=${MODNAME}-${verstub}.deb
src=${BUILDDIR}/${SRCNAME}-${pver%%-*}
tar=${SRCDIR}/${srcstub}.tar.gz
build=${BUILDDIR}/${SRCNAME}-${verstub}
log1=${LOGDIR}/${deb}.1.log
log2=${LOGDIR}/${deb}.2.log
[[ $pver =~ bpo || $kpver =~ bpo ]] &&
out=${DISTDIR}-backports || out=${DISTDIR}
check_build || continue
install_deps
prepare_source
install_headers
((++id))
time {
build_modules
say -n '[-] [%03d] Build time: ' "${id}"
} &
[[ ${BATCH:-} ]] || wait
done
wait
done
popd # ${BASEDIR}
prune_headers
clean_packages
publish_packages
say '=== End ZFS package build job queue.'
}
#----------------------------------------------------------------------#
# MARK: Repo Publish
#----------------------------------------------------------------------#
# Setup repo environment
setup_repository() {
local dist prefix='APT::FTPArchive::Release'
say '[*] Installing publish dependencies.'
qapt update && qapt install -y --no-install-recommends "${REPODEPS[@]}"
gpg -q --import /etc/keys/release.asc
read -r SUITE VERSION < <(
curl -fsS "${MIRROR}/dists/README" |
awk -v name="${CNAME}" '$3 == name {sub(/,/, ""); print $1, $NF}'
)
read -r ARCHES < <(
curl -fsS "${MIRROR}/dists/${CNAME}/InRelease" |
awk -F: '$1 ~ /Architectures/ {print $2}'
)
[[ ${VERSION%.*} =~ ^[0-9]+$ ]] || VERSION=${SUITE} &&
declare -gr ARCHES SUITE VERSION
[[ ${REPUBLISH:-} ]] && {
[[ ${REPUBLISH:-} == all ]] && alist="${ARCHES}"
[[ ! ${SUITE} =~ ^(oldstable|stable|testing)$ ]] ||
TAINTED[${CNAME/%/-backports}]+=r && TAINTED[${CNAME}]+=r
}
for dist in "${!TAINTED[@]}"; do
[[ ! ${REPUBLISH:-} && -f /etc/repo/${dist}.conf ]] ||
printf >"/etc/repo/${dist}.conf" '%s' "
${prefix}::Patterns \"contents-*\";
${prefix}::Origin \"DM4 Productions\";
${prefix}::Label \"Debian ZFS Modules\";
${prefix}::Suite \"${dist//${CNAME}/${SUITE}}\";
${prefix}::Version \"${dist//${CNAME}/${VERSION%.*}}\";
${prefix}::Codename \"${dist}\";
${prefix}::Architectures \"${ARCHES}\";
${prefix}::Components \"${COMPONENT}\";
${prefix}::Description \"Precompiled ZFS module packages for each kernel\";
"
done
}
# Generate source manifest
generate_sources() {
[[ ${TAINTED[$dist]} =~ s|r ]] || return 0 && mkdir -p "${sources%/*}"
say '--- Generating package source manifest.'
apt-ftparchive sources "${ddir}" "${hashes[@]}" >"${sources}" 2>&-
sed -i "s|${ddir}|${POOLDIR}|g" "${sources}"
xz -9fk "${sources}" && gzip -9fk "${sources}" && rm -f "${sources}"
}
# Generate package manifest
generate_packages() {
[[ ${TAINTED[$dist]} =~ b|r ]] || return 0 && mkdir -p "${packages%/*}"
say '--- Generating package manifest for: %s' "${arch}"
apt-ftparchive packages -a "${arch}" "${ddir}" "${hashes[@]}" >"${packages}"
apt-ftparchive contents -a "${arch}" "${ddir}" >"${contents}"
sed -i "s|${ddir}|${POOLDIR}|g" "${packages}"
xz -9fk "${packages}" "${contents}" &&
gzip -9fk "${packages}" "${contents}" &&
rm -f "${packages}" "${contents}"
}
# Generate release file
generate_release() {
[[ ${TAINTED[$dist]} =~ b|s|r ]] || return 0 && mkdir -p "${reldir}"
local files treldir=${reldir/${REPODIR}//tmp/repo}
say '--- Generating and signing release file.'
mkdir -p "${treldir}" && cp -R "${reldir}"/. "${treldir}"
mapfile -t files < <(find "${treldir}" -type f -name '*.gz' -print) &&
((${#files[@]})) && gzip -dfk "${files[@]}"
apt-ftparchive release "${treldir}" "${hashes[@]}" \
-c "/etc/repo/${dist}.conf" | gpg -as --clear-sign --yes -o "${release}"
rm -fr "${treldir}"
ln -fsrn "${reldir}" "${reldir//${CNAME}/${SUITE}}"
}
# Generate extra files manifest
generate_extrafiles() {
local fl ml mf=extrafiles
mapfile -t fl < <(
find . -maxdepth 1 -type f ! -name "${mf}" -printf '%f\n' | sort
)
mapfile -t ml < <(
[[ -f ${mf} ]] &&
awk 'NF == 2 && $1 ~ /[0-9a-f]{32}/ {print $2}' "${mf}"
)
[[ -f ${mf} && ! ${REPUBLISH:-} ]] && sha256sum -c "${mf}" &>/dev/null &&
[[ "${fl[*]}" == "${ml[*]}" ]] || {
[[ "${fl[*]}" ]] || { rm -f "${mf}" && return 0; }
say '[+] Generating top-level extrafiles manifest.'
sha256sum -- "${fl[@]}" 2>&- | gpg -as --clear-sign --yes -o "${mf}"
}
[[ $(stat -c %g "${REPODIR}") -eq 65534 ]] ||
find "${REPODIR}" ! -user 1000 -print0 | xargs -0r chown 1000 2>&-
}
# Run repository publish jobs
sign_repository() {
[[ ${NOPUBLISH:-} ]] && return 0
[[ ${!TAINTED[*]} || ${REPUBLISH:-} || ${EXTRAS:-} ]] || return 0
local arch ddir dist file reldir compdir release contents packages sources
local alist="${HARCH}" hashes=(--sha256 --no-sha512 --no-sha'1' --no-md'5')
say '=== Begin repository publish job queue.'
setup_repository
pushd "${REPODIR}"
for dist in "${!TAINTED[@]}"; do
ddir=${DISTDIR//${CNAME}/${dist}}
reldir=${REPODIR}/dists/${dist}
compdir=${reldir}/${COMPONENT}
release=${reldir}/InRelease
sources=${compdir}/source/Sources
say '[*] Update started for repository: %s' "${dist}"
generate_sources
for arch in ${alist}; do
contents=${compdir}/Contents-${arch}
packages=${compdir}/binary-${arch}/Packages
generate_packages
done
generate_release
say '[+] Update complete for repository: %s' "${dist}"
done
generate_extrafiles
say '=== End repository publish job queue.'
popd # ${REPODIR}
}
#----------------------------------------------------------------------#
# MARK: Build Matrix
#----------------------------------------------------------------------#
# Check workspace structure
check_workspace() {
local exe=${APPDIR}/bin/${_ME}
mkdir -p "${APPDIR}"/{bin,etc/{keys,repo},work} "${PUBDIR}" 2>&- || {
say -e '[!] Failed to prepare workspace, ensure the following exist:'
say -e '--- APPDIR: %s\n--- PUBDIR: %s' "${APPDIR}" "${PUBDIR}"
say -e '--- Or set them to appropriate values.' && _die 1
}
[[ ! ${SETUP:-} && "$0" == "${exe}" ]] && return 0
([[ -f ${exe} && ! -L ${exe} ]] && diff "$0" "${exe}" &>/dev/null) || {
[[ ! ${OVERWRITE:-} && -f ${exe} && ! -L ${exe} ]] || {
say '[*] Installing main script to:\n--- %s' "${exe}"
install -m 0755 "$0" "${exe}"
}
}
}
# Check GPG release key
check_releasekey() {
local ghome=/dev/shm/gpgtmp \
signkey=${APPDIR}/etc/keys/release.asc \
signpub=${APPDIR}/etc/keys/release.pub \
subj='Auto-generated Release Signing Key <release@localhost.local>'
[[ ! ${SETUP:-} && -f ${signkey} && ! -L ${signkey} ]] && return 0
say '[*] Generating new release signing key:'
rm -fr "${ghome}" && mkdir -m 0700 "${ghome}"
gpg -q --homedir "${ghome}" --batch --passphrase '' \
--quick-gen-key "${subj}" ed25519 sign,cert 2y
gpg --homedir "${ghome}" --batch --export-secret-keys -qao "${signkey}"
gpg --homedir "${ghome}" --batch --export -qao "${signpub}"
say '--- SIGNKEY: %s\n--- PUBKEY: %s' "${signkey}" "${signpub}"
gpg --homedir "${ghome}" --batch -qK | sed -n "s/^/--- /; s/$/${_LE}/; 3,5p"
chmod 0600 "${signkey}" && rm -fr "${ghome}"
}
# Check module signing cert
check_signcert() {
local cert=${APPDIR}/etc/keys/db.crt \
key=${APPDIR}/etc/keys/db.key \
subj='Auto-generated Kernel Module Signing Cert' \
config='[sign_cert]
basicConstraints=critical,CA:false
subjectKeyIdentifier=hash
keyUsage=critical,digitalSignature
extendedKeyUsage=codeSigning
'
[[ ! ${SETUP:-} && -f ${cert} && ! -L ${cert} ]] && return 0
say '[*] Generating new module signing certificate:'
openssl req -quiet -batch -x509 -days 730 -sha256 -utf8 -noenc \
-newkey ec -pkeyopt ec_paramgen_curve:secp256k1 \
-keyout "${key}" -out "${cert}" -subj "/CN=${subj}" \
-config <(printf '%s' "$config") -extensions sign_cert 2>&-
say '--- DBCERT: %s\n--- DBKEY: %s' "${cert}" "${key}"
openssl x509 -text -noout -in "${cert}" |
sed -nE "s/\s{8}/--- /; s/$/${_LE}/; 6,11p"
}
# Get release list
get_releases() {
mapfile -t RELS_LIST < <(
curl -fsS "${MIRROR}/dists/README" | awk \
'$1 ~ /^(((old|)old|un|)stable|testing|experimental),/ {print $3}'
)
[[ ! ${XRELS_LIST[*]:-} ]] || mapfile -t RELS_LIST < <(
read -ra XRELS_LIST <<<"${XRELS_LIST}"
for rel in "${RELS_LIST[@]}"; do
for xrel in "${XRELS_LIST[@]}"; do
[[ "${rel}" == "${xrel}" ]] && printf '%s\n' "${rel}"
done
done
) && declare -gr RELS_LIST && unset XRELS_LIST
}
# Get architecture list
get_arches() {
mapfile -t ARCH_LIST < <(
curl -fsS "${MIRROR}/dists/${rel}/InRelease" |
awk '$1 ~ /^Architectures:/ {for (i=2; i <= NF; i++) print $i}'
)
[[ ! ${XARCH_LIST:-} ]] || mapfile -t ARCH_LIST < <(
read -ra XARCH_LIST <<<"${XARCH_LIST}"
for arch in "${ARCH_LIST[@]}"; do
for xarch in "${XARCH_LIST[@]}"; do
[[ "${arch}" == "${xarch}" ]] && printf '%s\n' "${arch}"
done
done
) && declare -g ARCH_LIST
}
# Pull container image
pull_variant() {
[[ ${REPULL:-} ]] ||
! ${ENGINE} image exists "${IMAGE}:${rel}-${carch:-arch}" || return 0
say '[*] Pulling image for: %s | Arch: %s' "${rel}" "${carch:-arch}"
${ENGINE} image pull -q ${USEQEMU:+--platform "${plat}"} \
"${IMAGE}:${rel}" >&- || {
say -e '[!] Failed to pull image:\n--- %s:%s | Arch: %s' \
"${IMAGE}" "${rel}" "${carch:-arch}" && return 1
}
${ENGINE} image tag "${IMAGE}:${rel}" "${IMAGE}:${rel}-${carch:-arch}"
${ENGINE} image untag "${IMAGE}:${rel}" "${IMAGE}:${rel}"
${ENGINE} image prune -f
}
# Run build container
run_build() {
[[ ! ${PULLONLY:-} ]] || return 0
${ENGINE} run --rm --replace --net host \
--memory "${MEMLIMIT}" \
--name "zfs-build-${rel}-${arch}" \
${USEQEMU:+--platform "${plat}"} \
${USEQEMU:+-e CARCH="${_ARCH}"} \
${arch:+-e HARCH="${arch}"} \
${rel:+-e CNAME="${rel}"} \
${RETRY:+-e RETRY} \
${REBUILD:+-e REBUILD} \
${STAMPONLY:+-e STAMPONLY} \
${SRCPULL:+-e SRCPULL} \
${NOBUILD:+-e NOBUILD} \
${BATCH:+-e BATCH} \
${NOPURGE:+-e NOPURGE} \
${NOPUBLISH:+-e NOPUBLISH} \
${REPUBLISH:+-e REPUBLISH} \
${EXTRAS:+-e EXTRAS} \
${QUIET:+-e QUIET} \
${LOGFILE:+-e LOGFILE} \
${PURGEDAYS:+-e PURGEDAYS} \
${MIRROR:+-e MIRROR} \
${XKVER_LIST:+-e XKVER_LIST} \
${XPVER_LIST:+-e XPVER_LIST} \
-v "${APPDIR}/bin:/usr/local/bin:ro" \
-v "${APPDIR}/etc/keys:/etc/keys:ro" \
-v "${APPDIR}/etc/repo:/etc/repo:rw" \
-v "${APPDIR}/work:/srv/zfs:rw" \
-v "${PUBDIR}:/srv/repo:rw" \
"${IMAGE}:${rel}-${carch:-arch}" \
"/usr/local/bin/${_ME}" -b &
sudo -in systemd-inhibit --who="${_ME}:${rel}-${arch}" \
--why="Building ZFS module packages: ${rel}/${arch}" \
pidwait -g "${SYSTEMD_EXEC_PID:-$$}" "${ENGINE}" 2>&- ||
{ say '[-] Unable to set inhibitor, waiting instead.' && wait; }
}
# Run build job matrix
build_matrix() {
local rel arch plat carch
say '=== Begin ZFS package build matrix.'
check_workspace
check_releasekey
check_signcert
pushd "${APPDIR}"
get_releases
for rel in "${RELS_LIST[@]}"; do
get_arches
for arch in "${ARCH_LIST[@]}"; do
plat=''
case ${arch} in
all) continue ;;
armel) plat=linux/arm/v5 ;;
armhf) plat=linux/arm/v7 ;;
loong64) [[ ${USEQEMU:-} ]] && continue ;;
mipsel) [[ ${USEQEMU:-} ]] && continue ;;
mips64el) plat=linux/mips64le ;;
ppc64el) plat=linux/ppc64le ;;
*) : "${plat:=linux/${arch}}" ;;
esac
[[ ${USEQEMU:-} ]] || carch=${_ARCH}
pull_variant || continue
run_build || continue
done
done
popd # ${APPDIR}
say '=== End ZFS package build matrix.'
}
#----------------------------------------------------------------------#
# MARK: End Matter
#----------------------------------------------------------------------#
# Show stack trace on error
_show_trace() {
trap - ERR
local i=0 l s f
say -e '[!] Fatal Error | Exit: %d | Stack Trace:' "${1}"
while read -r l s f < <(caller ${i}); do
((++i)) && say -e '::: at %s:%d in function %s:\n::: %s' \
"${f##*/}" "${l}" "${s}" "$(sed -nE "s/^\s+//; ${l}p" "$0")"
done
exit "$1"
} >&2
# Stop container on termination
_kill_build() {
trap - INT QUIT TERM ERR
local id err=Exit && (($1 > 128)) && err=Signal
id=$(${ENGINE} rm -fit0 "zfs-build-${rel:-*}-${arch:-*}" 2>&-)
say -e '[!] Terminated | %s: %d | Killed container: %s' \
"${err}" "$(($1 % 128))" "${id:-N/A}"
kill -n 15 -$$
} >&2
# Exit without stack trace
_die() { trap - ERR && exit "$1"; }
# Main program entrypoint
_main() {
trap '_show_trace "$?"' ERR
trap '_kill_build "$?"' INT QUIT TERM
declare -rf \
print_usage print_version pushd popd qapt say \
set_globals setup_builder get_versions get_ccmap \
check_usage check_build install_deps prepare_source \
install_headers build_modules prune_headers \
clean_packages publish_packages build_packages \
setup_repository generate_sources generate_packages \
generate_release generate_extrafiles sign_repository \
check_workspace check_releasekey check_signcert \
get_releases get_arches pull_variant run_build \
build_matrix _show_trace _kill_build _die _main
set_globals
case ${1:-} in
-b) build_packages ;&
-s) sign_repository ;;
-y) build_matrix ;;
-h) print_usage ;;
-v) print_version ;;
*) say -e 'Use -y to start build matrix, or -h for usage.' && _die 1 ;;
esac
}
# Only execute if not being sourced
[[ ${BASH_SOURCE[0]} == "$0" ]] || return 0 && _main "$@"
Raw
zfs-module-build.timer
[Unit]
Description=Build binary ZFS modules packages and publish repo
Wants=network.target
[Timer]
OnCalendar=daily
Persistent=1
[Install]
WantedBy=timers.target
@AfroThundr3007730
Copy link
Author

AfroThundr3007730 commented Jan 13, 2026 •
edited
Loading

TODO:

  • Need to dynamically get the version details from package lists
  • Need to make this iterate over K kernel versions and V zfs versions
  • The above will need to be done per S Suites and A arches as well
  • Need to ensure the generated modules are signed before packaging
  • Need to generate an apt repository (signed) with these packages
  • Need to trigger on package updates or poll upstream repos for changes
  • Need to remove stale packages no longer present in upstream repos

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment