raspi alpine lbu

-media-mmcblk0p1-cmdline.txt

modules=loop,squashfs,sd-mod,usb-storage quiet console=tty1

df

Filesystem                 1024-blocks    Used Available Capacity Mounted on
devtmpfs                         10240       0     10240       0% /dev
shm                            1941920       0   1941920       0% /dev/shm
/dev/mmcblk0p1                 2923960 1108200   1815760      38% /media/mmcblk0p1
tmpfs                          1941920  432064   1509856      23% /
tmpfs                           776768     244    776524       1% /run
/dev/loop0                       30592   30592         0     100% /.modloop
/dev/vgdata/lvpers-home         996780   27744    900224       3% /media/persist/home
/dev/vgdata/lvpers-usr         1510792 1177632    238136      84% /media/persist/usr
/dev/vgdata/lvpers-varlog       996780    5756    922212       1% /media/persist/varlog
overlay                         996780   27744    900224       3% /home
overlay                        1510792 1177632    238136      84% /usr
overlay                         996780    5756    922212       1% /var/log
/dev/vgdata/lvvarlibdocker     8154588 3516472   4202304      46% /var/lib/docker

etc-apk-repositories

/media/mmcblk0p1/apks
http://dl-cdn.alpinelinux.org/alpine/v3.20/main
http://dl-cdn.alpinelinux.org/alpine/v3.20/community

etc-apk-world

alpine-base
apk-tools-static
bash
coreutils
docker
docker-compose
dosfstools
e2fsprogs
fail2ban
findutils
git
glances
htop
ifupdown-ng
ifupdown-ng-iproute2
ifupdown-ng-wifi
iotop
iptables
iw
linux-pam
lvm2
musl
openssh
openssl
openvpn
py3-pip
python3
rsync
screen
sysstat
wpa_supplicant

etc-fstab

/dev/cdrom	/media/cdrom	iso9660	noauto,ro 0 0
/dev/usbdisk	/media/usb	vfat	noauto,ro 0 0
/media/mmcblk0p1/boot /boot none defaults,bind 0 0


# persist backing dirs
/dev/vgdata/lvpers-home         /media/persist/home         ext4 rw,relatime,errors=remount-ro 0 0
/dev/vgdata/lvpers-usr          /media/persist/usr          ext4 rw,relatime,errors=remount-ro 0 0
/dev/vgdata/lvpers-varlog       /media/persist/varlog       ext4 rw,relatime,errors=remount-ro 0 0

# persist front dirs
overlay /home           overlay lowerdir=/home,upperdir=/media/persist/home/u,workdir=/media/persist/home/w 0 0
overlay /usr            overlay lowerdir=/usr,upperdir=/media/persist/usr/u,workdir=/media/persist/usr/w 0 0
overlay /var/log        overlay lowerdir=/var/log,upperdir=/media/persist/varlog/u,workdir=/media/persist/varlog/w 0 0

# plain dir
/dev/vgdata/lvvarlibdocker      /var/lib/docker             ext4 rw,noatime,errors=remount-ro 0 0

etc-lbu-lbu.conf

# what cipher to use with -e option
DEFAULT_CIPHER=aes-256-cbc

# Uncomment the row below to encrypt config by default
# ENCRYPTION=$DEFAULT_CIPHER

# Uncomment below to avoid <media> option to 'lbu commit'
# Can also be set to 'floppy'
# LBU_MEDIA=usb
LBU_MEDIA=mmcblk0p1

# Set the LBU_BACKUPDIR variable in case you prefer to save the apkovls
# in a normal directory instead of mounting an external media.
# LBU_BACKUPDIR=/root/config-backups

# Uncomment below to let lbu make up to 3 backups
BACKUP_LIMIT=3

fdisk

vp...~# fdisk -l /dev/mmcblk0
Disk /dev/mmcblk0: 30 GB, 31914983424 bytes, 62333952 sectors
3880 cylinders, 255 heads, 63 sectors/track
Units: sectors of 1 * 512 = 512 bytes

Device       Boot StartCHS    EndCHS        StartLBA     EndLBA    Sectors  Size Id Type
/dev/mmcblk0p1    1023,254,63 1023,254,63       2048    5861423    5859376 2861M  b Win95 FAT32
/dev/mmcblk0p2    1023,254,63 1023,254,63    5863424   62332927   56469504 26.9G 8e Linux LVM

lbu-ls

# had to filter out the etckeeper git repo and the .py files from alerta
# the alerta install is supposed to be in-ram but at the moment it's compiled
# unsatisfying at the very least
# alerta heartbeat is used to detect site blackouts


# lbu ls | grep -v -e site-packages -e etc/.git
etc/apk/commit_hooks.d/
etc/apk/repositories
etc/apk/protected_paths.d/lbu.list
etc/apk/protected_paths.d/ca-certificates.list
etc/apk/arch
etc/apk/keys/[email protected]
etc/apk/keys/[email protected]
etc/apk/keys/[email protected]
etc/apk/keys/[email protected]
etc/apk/keys/[email protected]
etc/apk/cache
etc/apk/world
etc/inittab
etc/zoneinfo/
etc/zoneinfo/UTC
etc/openvpn/openvpn.conf
etc/openvpn/login.conf
etc/lbu/lbu.conf
etc/conf.d/wpa_supplicant
etc/hostname
etc/runlevels/default/openvpn
etc/runlevels/default/fail2ban
etc/runlevels/default/lvm
etc/runlevels/default/ntpd
etc/runlevels/default/sshd
etc/runlevels/default/docker
etc/runlevels/shutdown/savecache
etc/runlevels/shutdown/killprocs
etc/runlevels/shutdown/mount-ro
etc/runlevels/boot/lvm
etc/runlevels/boot/wpa_supplicant
etc/runlevels/boot/syslog
etc/runlevels/boot/bootmisc
etc/runlevels/boot/hostname
etc/runlevels/boot/sysctl
etc/runlevels/boot/modules
etc/runlevels/boot/swclock
etc/runlevels/sysinit/lvm
etc/runlevels/sysinit/modloop
etc/runlevels/sysinit/hwdrivers
etc/runlevels/sysinit/mdev
etc/runlevels/sysinit/dmesg
etc/runlevels/sysinit/devfs
etc/network/interfaces
etc/wpa_supplicant/wpa_supplicant.conf.3285
etc/wpa_supplicant/wpa_supplicant.conf
etc/fstab
etc/lvm/archive/vgdata_00000-1873069622.vg
etc/lvm/backup/vgdata
etc/.etckeeper
etc/30-route.start
etc/70-docker-cmk.start
etc/README
etc/sudoers.d/
etc/sudoers.d/svcmon
etc/sudoers
etc/ssh/ssh_host_rsa_key.pub
etc/ssh/ssh_host_rsa_key
etc/ssh/ssh_host_ecdsa_key.pub
etc/ssh/ssh_host_ecdsa_key
etc/ssh/ssh_host_ed25519_key.pub
etc/ssh/ssh_host_ed25519_key
etc/passwd-
etc/passwd
etc/shadow-
etc/shadow
etc/group-
etc/group
etc/localtime
etc/crontabs/cron.update
etc/crontabs/root
etc/shells
etc/resolv.conf
etc/resolv.conf-tun0.sv
media/persist/
opt/alerta/
opt/alerta/include/
opt/alerta/include/python3.12/
opt/alerta/lib/
opt/alerta/lib/python3.12/
opt/alerta/lib64
opt/alerta/bin/
opt/alerta/bin/python3
opt/alerta/bin/python
opt/alerta/bin/python3.12
opt/alerta/bin/pip
opt/alerta/bin/pip3
opt/alerta/bin/pip3.12
opt/alerta/bin/Activate.ps1
opt/alerta/bin/activate
opt/alerta/bin/activate.csh
opt/alerta/bin/activate.fish
opt/alerta/bin/tabulate
opt/alerta/bin/normalizer
opt/alerta/bin/alerta
opt/alerta/pyvenv.cfg
opt/alerta/.alerta.conf

lvs

  LV             VG     Attr       LSize   Pool Origin Data%  Meta%  Move Log Cpy%Sync Convert
  lvpers-home    vgdata -wi-ao----   1.00g                                                    
  lvpers-usr     vgdata -wi-ao----   1.50g                                                    
  lvpers-varlog  vgdata -wi-ao----   1.00g                                                    
  lvswap         vgdata -wi-a----- 512.00m                                                    
  lvvarlibdocker vgdata -wi-ao----   8.00g   

proc-mounts

sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0
devtmpfs /dev devtmpfs rw,nosuid,noexec,relatime,size=10240k,nr_inodes=476057,mode=755 0 0
proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0
devpts /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0
shm /dev/shm tmpfs rw,nosuid,nodev,noexec,relatime 0 0
/dev/mmcblk0p1 /media/mmcblk0p1 vfat ro,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=ascii,shortname=mixed,errors=remount-ro 0 0
tmpfs / tmpfs rw,relatime,mode=755 0 0
tmpfs /run tmpfs rw,nosuid,nodev,size=776768k,nr_inodes=819200,mode=755 0 0
mqueue /dev/mqueue mqueue rw,nosuid,nodev,noexec,relatime 0 0
/dev/loop0 /.modloop squashfs ro,relatime,errors=continue 0 0
securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0
debugfs /sys/kernel/debug debugfs rw,nosuid,nodev,noexec,relatime 0 0
configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0
pstore /sys/fs/pstore pstore rw,nosuid,nodev,noexec,relatime 0 0
tracefs /sys/kernel/debug/tracing tracefs rw,nosuid,nodev,noexec,relatime 0 0
/dev/mmcblk0p1 /boot vfat ro,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=ascii,shortname=mixed,errors=remount-ro 0 0
/dev/vgdata/lvpers-home /media/persist/home ext4 rw,relatime,errors=remount-ro 0 0
/dev/vgdata/lvpers-usr /media/persist/usr ext4 rw,relatime,errors=remount-ro 0 0
/dev/vgdata/lvpers-varlog /media/persist/varlog ext4 rw,relatime,errors=remount-ro 0 0
overlay /home overlay rw,relatime,lowerdir=/home,upperdir=/media/persist/home/u,workdir=/media/persist/home/w,uuid=null 0 0
overlay /usr overlay rw,relatime,lowerdir=/usr,upperdir=/media/persist/usr/u,workdir=/media/persist/usr/w,uuid=null 0 0
overlay /var/log overlay rw,relatime,lowerdir=/var/log,upperdir=/media/persist/varlog/u,workdir=/media/persist/varlog/w,uuid=null 0 0
/dev/vgdata/lvvarlibdocker /var/lib/docker ext4 rw,noatime,errors=remount-ro 0 0
none /sys/fs/cgroup cgroup2 rw,nosuid,nodev,noexec,relatime 0 0

Alpine Raspberry Bridge Head

Description

LBU mode config, with persistence overlays, with LVM.

i dont know if i'll be able to make nice docs around this, so here's a dump.

it is related to these howtos:
https://www.perrotta.dev/2022/01/alpine-linux-on-raspberry-pi-diskless-mode-with-persistent-storage/
https://alldrops.info/posts/linux-drops/2021-06-21_alpine-linux-on-raspberry-pi-4-headless-persistent-install/
the first one is especially very good.

Backstory

I use my version since around 2019-2020 at a few sites.
It is those howtos on steroids via LVM and a few persistent directories.

the precedessor was an odroid c2 with predefined tooling that could be fetched from git.
That one was created to prep for site maintenance, to hold an offline copy of all firmware, config backups, documentation.
it had a base layout that was easily represented in git, with a multi-tenant structure and an encryption dir for the per-tenant directories.
so you could go onsite, put it on a dhcp server, unlock the directory, access the sensitive stuff and manuals/fw was there too.
i think i wanted to go as far as fetching what you need per site/tenant from gitlab, but didn't figure out how to do the same security on the gitlab end. also the idea was shot down so i gave up on it for years until i was re-integrating it into what we have here.
So, the solution now is to be able to VNF-style just deploy the features you need at a site.

(a part of the old config is another gist somewhere)

Role

Bridge head

The raspis' role is more of a bridge head. they include docker to be able to launch a monitoring site, and that will run in persistent storage; you can use this to provision services, i.e. oxidized or a tftp server.
this is more important while you're still building the env, later you probably want to have those services on a dedicated system.
i've not picked a definite solution for that myself either.

Kiosk

I have a similar setup on an Dell Edge Gateway 5000 (not boot to ram, but same vpn call home etc.) which is a monitoring node and used to run in kiosk mode displaying nagstamon. Now it's on a wall and not displaying anything ;-)
That hardware can run off a lead battery for days (good) and has builtin broadband antennas etc (good) but also has a very shitty atom CPU (very bad) and almost no wwan module works (very bad)
maybe someday i'll ask some re-flow guy to do a cpu swap, and very likely i'll go ahead and make this also boot to ram!
it'll definitely get also its own LoRaWAN interface.
all of that could also be done with a raspi, assuming you find a capable/safe UPS solution.

Heartbeat sender

I'm now working to integrate alerta on all these systems, one of them is the central site.
Alerta can do monitoring - i don't need most of that - and it can do basic heartbeats, which is perfect for my purposes.
So it'll be part of the bridge head.

Summary

A system that can be put anywhere with internet, is identified by its name (drone-customer-site), has its own credentials for vpn, can not reach back but be reached (pushed to), is optimized to always get that connection, has an amount of emergency tooling so it can be used for bootstrapping of better systems. is reasonably resilient. tools can be deployed to it on a whim and will not change the core setup. will report on its state.

Setup

tooling

What I often use there is

• alpine linux
• arpwatch
• alerta
• check_mk 2.1/2.2
• nmap
• oxidized
• tcpdump/tshark
• tftpd-hpa
• unifi cli

I've never deployed netbox or ansible or i-do-it on the small nodes.
it would be a complete antipattern to put them in a non-support function.

HW Config

the hardware is normally a Pi4/4GB with a high-quality casing for passive cooling. There's some metal cases that come with thermal pads etc. and that is what you want.

OS Storage / Config

the rest of the OS is in ram to ensure it can be power cycled on error / sd card hang etc.
The raspis call home using openvpn; all reachable wifis will be used in config of wpa supplicant and eth0 is on dhcp.
so as long as you plug internet somewhere, they will use it.

SD Cards

in the start I had used normal sd cards, today I would only recommend

• Sandisk max endurance (tested)
• Swissbit (not yet tested)

Some SD cards will still just hang every 2 years, so being able to let someone power cycle the thing is important.
I also put a sticker with phone number and device role on each.

Lessons / Experiences / Problems

things that were more tricky than expected:

default route

at times I needed to do policy based routing with it, to be able to track faults in a misconfigured lan i connected in over wifi.
there's no good auto-default-route picker for linux. normally it'll just use the first one period.
I didn't try network namespaces only for a secondary sshd or other crazy stuff.

update breakages

normally everything went fine, but the changes to inittab from /sbin/rc to /sbin/openrc and such before 3.20 caused some grievance.
the chain of failure is somewhere like this

• lbu.conf not correct / apk cache dir wrong
• libary issue
• breaks find (gnu)
• breaks etckeeper
• breaks update
• forgetting lbu ci / not forgetting
• leaves lbu overlay incomplete
• boots old env
• causing library issue
• causing etckeeper to fail

it's an issue that is more easy to run into once the systems are running for a few years and you did a few very easy upgrade

mitigations:

• what you need to do is to ensure your apk repo mirror and cache are up to date.
• find out how to roll back to an older apkvol
• build a system once from a to z a year to maintain awareness
• don't forget you can always re-roll /media/mmcblk0p1 from an alpine raspi tarball and that it would be good to sometimes refresh that
• investigate putting the onerror=panic flag on some of the filesystems and readonly-ro on the others. (basically it should be the old story that you should still be able to ssh in when the boot disk failed. be able to maintain operation even without reboot for a few months till you can go on-site to fix. it's not too hard
• if one can, one might have a lvm mirror between an emmc and sd card for the vgdata partitions. (i think they replaced lvm mirrors with cut-down md raid so of course this is no longer achieving some error compment... but it'll be ok.
• sd+emmc mirror is not on raspi
• please don't use thin pools in this setup. please.
• sd+emmc mirror is beneficial over sd+usb or such since you get away from the usb devloss issues
• usb power options (i think i didn't set them at all here)

usb issues

• I did have issues with usb attached NICs, might be power might a million other things
• same iirc for usb-serial
• you'll might help yourself if you add some gnd/tx/rx wires for the onboard serial (i'm mostly going for esp-link based wireless bridges there)

VLAN

there's an incompatibility between ifupdown-ng and vlan. it sucks. don't know the back story.
I think at some point I had ovs installed and that is probably the best way to go about it.

WiFi

I don't know yet how to have multiple radios that are mostly for ssid monitoring. likely not the best tool for the job or just not knowing how to use the tool for the job.