Last active
February 21, 2023 18:08
-
-
Save smoser/2e622c67e8a679630d6e70e63b0f32d0 to your computer and use it in GitHub Desktop.
[fosdem 2023 talk] oci images in squashfs format with dm-verity
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| # Create application containers from OCI images | |
| # Copyright © 2014 Stéphane Graber <[email protected]> | |
| # Copyright © 2017 Serge Hallyn <[email protected]> | |
| # | |
| # This library is free software; you can redistribute it and/or | |
| # modify it under the terms of the GNU Lesser General Public | |
| # License as published by the Free Software Foundation; either | |
| # version 2.1 of the License, or (at your option) any later version. | |
| # This library is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
| # Lesser General Public License for more details. | |
| # You should have received a copy of the GNU Lesser General Public | |
| # License along with this library; if not, write to the Free Software | |
| # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 | |
| # USA | |
| set -eu | |
| # Make sure the usual locations are in PATH | |
| export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin" | |
| # Check for required binaries | |
| for bin in skopeo umoci jq; do | |
| if ! command -v $bin >/dev/null 2>&1; then | |
| echo "ERROR: Missing required tool: $bin" 1>&2 | |
| exit 1 | |
| fi | |
| done | |
| LXC_OCI_MOUNT="lxc-oci-mount" | |
| ATOMFS="atomfs" | |
| LOCALSTATEDIR=/var | |
| LXC_TEMPLATE_CONFIG=/usr/share/lxc/config | |
| LXC_HOOK_DIR=/usr/share/lxc/hooks | |
| OCI_MOUNT="" | |
| # Some useful functions | |
| cleanup() { | |
| if [ -d "${LXC_ROOTFS}.tmp" ]; then | |
| rm -Rf "${LXC_ROOTFS}.tmp" | |
| fi | |
| if [ -n "$OCI_MOUNT" ]; then | |
| echo "atomfs unmount $OCI_MOUNT" >&2 | |
| "${ATOMFS}" umount "$OCI_MOUNT" | |
| OCI_MOUNT="" | |
| fi | |
| } | |
| in_userns() { | |
| [ -e /proc/self/uid_map ] || { echo no; return; } | |
| while read -r line; do | |
| fields="$(echo "$line" | awk '{ print $1 " " $2 " " $3 }')" | |
| if [ "${fields}" = "0 0 4294967295" ]; then | |
| echo no; | |
| return; | |
| fi | |
| if echo "${fields}" | grep -q " 0 1$"; then | |
| echo userns-root; | |
| return; | |
| fi | |
| done < /proc/self/uid_map | |
| if [ -e /proc/1/uid_map ]; then | |
| if [ "$(cat /proc/self/uid_map)" = "$(cat /proc/1/uid_map)" ]; then | |
| echo userns-root | |
| return | |
| fi | |
| fi | |
| echo yes | |
| } | |
| getmanifestpath() { | |
| local basedir="$1" ref="$2" p="" | |
| # if given 'sha256:<hash>' then return the blobs/sha256/hash | |
| case "$ref" in | |
| sha256:*) | |
| p="$basedir/blobs/sha256/${ref#sha256:}" | |
| [ -f "$p" ] && echo "$p" && return 0 | |
| echo "could not find manifest path to blob $ref. file did not exist: $p" >&2 | |
| return 1 | |
| esac | |
| # find the reference by annotation | |
| local blobref="" hashtype="" hashval="" | |
| blobref=$(jq -c -r --arg q "$ref" '.manifests[] | if .annotations."org.opencontainers.image.ref.name" == $q then .digest else empty end' < "${basedir}/index.json") | |
| # blobref is 'hashtype:hash' | |
| hashtype="${blobref%%:*}" | |
| hashval="${blobref#*:}" | |
| p="$basedir/blobs/$hashtype/$hashval" | |
| [ -f "$p" ] && echo "$p" && return 0 | |
| echo "did not find manifest for $ref. file did not exist: $p" >&2 | |
| return 1 | |
| } | |
| getconfigpath() { | |
| local basedir="$1" mfpath="$2" cdigest="" | |
| # Ok we have the image config digest, now get the config ref from the manifest. | |
| # shellcheck disable=SC2039 | |
| cdigest=$(jq -c -r '.config.digest' < "$mfpath") | |
| if [ -z "${cdigest}" ]; then | |
| echo "container config not found" >&2 | |
| return | |
| fi | |
| # cdigest is 'hashtype:hash' | |
| local ht="${cdigest%%:*}" hv="${cdigest#*:}" p="" | |
| p="$basedir/blobs/$ht/$hv" | |
| if [ ! -f "$p" ]; then | |
| echo "config file did not exist for digest $cdigest" >&2 | |
| return 1 | |
| fi | |
| echo "$p" | |
| } | |
| getlayermediatype() { | |
| jq -c -r '.layers[0].mediaType' <"$1" | |
| } | |
| # Get entrypoint from oci image. Use sh if unspecified | |
| getep() { | |
| if [ "$#" -eq 0 ]; then | |
| echo "/bin/sh" | |
| return | |
| fi | |
| configpath="$1" | |
| ep=$(jq -c '.config.Entrypoint[]?'< "${configpath}" | tr '\n' ' ') | |
| cmd=$(jq -c '.config.Cmd[]?'< "${configpath}" | tr '\n' ' ') | |
| if [ -z "${ep}" ]; then | |
| ep="${cmd}" | |
| if [ -z "${ep}" ]; then | |
| ep="/bin/sh" | |
| fi | |
| elif [ -n "${cmd}" ]; then | |
| ep="${ep} ${cmd}" | |
| fi | |
| echo "${ep}" | |
| return | |
| } | |
| # get environment from oci image. | |
| getenv() { | |
| if [ "$#" -eq 0 ]; then | |
| return | |
| fi | |
| configpath="$1" | |
| env=$(jq -c -r '.config.Env[]'< "${configpath}") | |
| echo "${env}" | |
| return | |
| } | |
| # check var is decimal. | |
| isdecimal() { | |
| var="$1" | |
| if [ "${var}" -eq "${var}" ] 2> /dev/null; then | |
| return 0 | |
| else | |
| return 1 | |
| fi | |
| } | |
| # get uid, gid from oci image. | |
| getuidgid() { | |
| configpath="$1" | |
| rootpath="$2" | |
| passwdpath="${rootpath}/etc/passwd" | |
| grouppath="${rootpath}/etc/group" | |
| usergroup=$(jq -c -r '.config.User' < "${configpath}") | |
| # shellcheck disable=SC2039 | |
| usergroup=(${usergroup//:/ }) | |
| user=${usergroup[0]:-0} | |
| if ! isdecimal "${user}"; then | |
| if [ -f ${passwdpath} ]; then | |
| user=$(grep "^${user}:" "${passwdpath}" | awk -F: '{print $3}') | |
| else | |
| user=0 | |
| fi | |
| fi | |
| group=${usergroup[1]:-} | |
| if [ -z "${group}" ]; then | |
| if [ -f "${passwdpath}" ]; then | |
| group=$(grep "^[^:]*:[^:]*:${user}:" "${passwdpath}" | awk -F: '{print $4}') | |
| else | |
| group=0 | |
| fi | |
| elif ! isdecimal "${group}"; then | |
| if [ -f "${grouppath}" ]; then | |
| group=$(grep "^${group}:" "${grouppath}" | awk -F: '{print $3}') | |
| else | |
| group=0 | |
| fi | |
| fi | |
| echo "${user:-0} ${group:-0}" | |
| return | |
| } | |
| # get cwd from oci image. | |
| getcwd() { | |
| if [ "$#" -eq 0 ]; then | |
| echo "/" | |
| return | |
| fi | |
| configpath="$1" | |
| cwd=$(jq -c -r '.config.WorkingDir // "/"' < "${configpath}") | |
| echo "${cwd}" | |
| return | |
| } | |
| usage() { | |
| cat <<EOF | |
| LXC container template for OCI images | |
| Special arguments: | |
| [ -h | --help ]: Print this help message and exit. | |
| Required arguments: | |
| [ -u | --url <url> ]: The OCI image URL | |
| Optional arguments: | |
| [ --username <username> ]: The username for the registry | |
| [ --password <password> ]: The password for the registry | |
| LXC internal arguments (do not pass manually!): | |
| [ --name <name> ]: The container name | |
| [ --path <path> ]: The path to the container | |
| [ --rootfs <rootfs> ]: The path to the container's rootfs | |
| [ --mapped-uid <map> ]: A uid map (user namespaces) | |
| [ --mapped-gid <map> ]: A gid map (user namespaces) | |
| EOF | |
| return 0 | |
| } | |
| echo "got $#: $*" | |
| if ! options=$(getopt -o u:h -l help,url:,username:,password:,no-cache,dhcp,name:,path:,rootfs:,mapped-uid:,mapped-gid: -- "$@"); then | |
| usage | |
| exit 1 | |
| fi | |
| eval set -- "$options" | |
| OCI_URL="" | |
| OCI_USERNAME= | |
| OCI_PASSWORD= | |
| OCI_USE_CACHE="true" | |
| OCI_USE_DHCP="false" | |
| LXC_MAPPED_GID= | |
| LXC_MAPPED_UID= | |
| LXC_NAME= | |
| LXC_PATH= | |
| LXC_ROOTFS= | |
| while :; do | |
| case "$1" in | |
| -h|--help) usage && exit 1;; | |
| -u|--url) OCI_URL=$2; shift 2;; | |
| --username) OCI_USERNAME=$2; shift 2;; | |
| --password) OCI_PASSWORD=$2; shift 2;; | |
| --no-cache) OCI_USE_CACHE="false"; shift 1;; | |
| --dhcp) OCI_USE_DHCP="true"; shift 1;; | |
| --name) LXC_NAME=$2; shift 2;; | |
| --path) LXC_PATH=$2; shift 2;; | |
| --rootfs) LXC_ROOTFS=$2; shift 2;; | |
| --mapped-uid) LXC_MAPPED_UID=$2; shift 2;; | |
| --mapped-gid) LXC_MAPPED_GID=$2; shift 2;; | |
| *) break;; | |
| esac | |
| done | |
| # Check that we have all variables we need | |
| if [ -z "$LXC_NAME" ] || [ -z "$LXC_PATH" ] || [ -z "$LXC_ROOTFS" ]; then | |
| echo "ERROR: Not running through LXC" 1>&2 | |
| exit 1 | |
| fi | |
| if [ -z "$OCI_URL" ]; then | |
| echo "ERROR: no OCI URL given" | |
| exit 1 | |
| fi | |
| if [ -n "$OCI_PASSWORD" ] && [ -z "$OCI_USERNAME" ]; then | |
| echo "ERROR: password given but no username specified" | |
| exit 1 | |
| fi | |
| if [ "${OCI_USE_CACHE}" = "true" ]; then | |
| if ! skopeo copy --help | grep -q 'dest-shared-blob-dir'; then | |
| echo "INFO: skopeo doesn't support blob caching" | |
| OCI_USE_CACHE="false" | |
| fi | |
| fi | |
| USERNS=$(in_userns) | |
| if [ "$USERNS" = "yes" ]; then | |
| if [ -z "$LXC_MAPPED_UID" ] || [ "$LXC_MAPPED_UID" = "-1" ]; then | |
| echo "ERROR: In a user namespace without a map" 1>&2 | |
| exit 1 | |
| fi | |
| fi | |
| OCI_DIR="$LXC_PATH/oci" | |
| if [ "${OCI_USE_CACHE}" = "true" ]; then | |
| if [ "$USERNS" = "yes" ]; then | |
| DOWNLOAD_BASE="${HOME}/.cache/lxc" | |
| else | |
| DOWNLOAD_BASE="${LOCALSTATEDIR}/cache/lxc" | |
| fi | |
| else | |
| DOWNLOAD_BASE="$OCI_DIR" | |
| fi | |
| mkdir -p "${DOWNLOAD_BASE}" | |
| # Trap all exit signals | |
| trap cleanup EXIT HUP INT TERM | |
| # Download the image | |
| # shellcheck disable=SC2039 | |
| skopeo_args=("--remove-signatures" --insecure-policy) | |
| if [ -n "$OCI_USERNAME" ]; then | |
| CREDENTIALS="${OCI_USERNAME}" | |
| if [ -n "$OCI_PASSWORD" ]; then | |
| CREDENTIALS="${CREDENTIALS}:${OCI_PASSWORD}" | |
| fi | |
| # shellcheck disable=SC2039 | |
| skopeo_args+=(--src-creds "${CREDENTIALS}") | |
| fi | |
| OCI_NAME="$LXC_NAME" | |
| if [ "${OCI_USE_CACHE}" = "true" ]; then | |
| skopeo_args+=(--dest-shared-blob-dir "${DOWNLOAD_BASE}") | |
| mkdir -p "${OCI_DIR}/blobs/" | |
| ln -s "${DOWNLOAD_BASE}/sha256" "${OCI_DIR}/blobs/sha256" | |
| fi | |
| skopeo copy "${skopeo_args[@]}" "${OCI_URL}" "oci:${OCI_DIR}:${OCI_NAME}" | |
| mfpath=$(getmanifestpath "${OCI_DIR}" "${OCI_NAME}") | |
| OCI_CONF_FILE=$(getconfigpath "${OCI_DIR}" "$mfpath") | |
| mediatype=$(getlayermediatype "$mfpath") | |
| echo "mfpath=$mfpath conf=$OCI_CONF_FILE" 1>&2 | |
| echo "mediatype=$mediatype" | |
| echo "DOWNLOAD_BASE=$DOWNLOAD_BASE" >&2 | |
| echo "OCI_DIR=$OCI_DIR" >&2 | |
| case "$mediatype" in | |
| #application/vnd.oci.image.layer.v1.tar+gzip | |
| application/vnd.oci.image.layer.v1.tar*) | |
| echo "Unpacking tar rootfs" 2>&1 | |
| # shellcheck disable=SC2039 | |
| umoci_args=("") | |
| if [ -n "$LXC_MAPPED_UID" ] && [ "$LXC_MAPPED_UID" != "-1" ]; then | |
| # shellcheck disable=SC2039 | |
| umoci_args+=(--rootless) | |
| fi | |
| # shellcheck disable=SC2039 | |
| # shellcheck disable=SC2068 | |
| umoci --log=error unpack ${umoci_args[@]} --image "${OCI_DIR}:${OCI_NAME}" "${LXC_ROOTFS}.tmp" | |
| find "${LXC_ROOTFS}.tmp/rootfs" -mindepth 1 -maxdepth 1 -exec mv '{}' "${LXC_ROOTFS}/" \; | |
| ;; | |
| #application/vnd.stacker.image.layer.squashfs+zstd+verity | |
| application/vnd.*.image.layer.squashfs*) | |
| echo "squashfs type $mediatype" >&2 | |
| echo ${ATOMFS} mount "${OCI_DIR}:${OCI_NAME}" "$LXC_ROOTFS" >&2 | |
| ${ATOMFS} mount "${OCI_DIR}:${OCI_NAME}" "$LXC_ROOTFS" | |
| OCI_MOUNT="$LXC_ROOTFS" | |
| ;; | |
| *) | |
| echo "Unknown media type $mediatype" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| LXC_CONF_FILE="${LXC_PATH}/config" | |
| entrypoint=$(getep "${OCI_CONF_FILE}") | |
| echo "lxc.execute.cmd = '${entrypoint}'" >> "${LXC_CONF_FILE}" | |
| echo "lxc.mount.auto = proc:mixed sys:mixed cgroup:mixed" >> "${LXC_CONF_FILE}" | |
| case "$mediatype" in | |
| application/vnd.*.image.layer.squashfs*) | |
| echo "lxc.hook.pre-mount = ${LXC_OCI_MOUNT}" >> "${LXC_CONF_FILE}";; | |
| esac | |
| environment=$(getenv "${OCI_CONF_FILE}") | |
| # shellcheck disable=SC2039 | |
| while read -r line; do | |
| echo "lxc.environment = ${line}" >> "${LXC_CONF_FILE}" | |
| done <<< "${environment}" | |
| if [ -e "${LXC_TEMPLATE_CONFIG}/common.conf" ]; then | |
| echo "lxc.include = ${LXC_TEMPLATE_CONFIG}/common.conf" >> "${LXC_CONF_FILE}" | |
| fi | |
| if [ -n "$LXC_MAPPED_UID" ] && [ "$LXC_MAPPED_UID" != "-1" ] && [ -e "${LXC_TEMPLATE_CONFIG}/userns.conf" ]; then | |
| echo "lxc.include = ${LXC_TEMPLATE_CONFIG}/userns.conf" >> "${LXC_CONF_FILE}" | |
| fi | |
| if [ -e "${LXC_TEMPLATE_CONFIG}/oci.common.conf" ]; then | |
| echo "lxc.include = ${LXC_TEMPLATE_CONFIG}/oci.common.conf" >> "${LXC_CONF_FILE}" | |
| fi | |
| if [ "${OCI_USE_DHCP}" = "true" ]; then | |
| echo "lxc.hook.start-host = ${LXC_HOOK_DIR}/dhclient" >> "${LXC_CONF_FILE}" | |
| echo "lxc.hook.stop = ${LXC_HOOK_DIR}/dhclient" >> "${LXC_CONF_FILE}" | |
| fi | |
| echo "lxc.uts.name = ${LXC_NAME}" >> "${LXC_CONF_FILE}" | |
| # set the hostname | |
| cat <<EOF > "${LXC_ROOTFS}/etc/hostname" | |
| ${LXC_NAME} | |
| EOF | |
| # set minimal hosts | |
| cat <<EOF > "${LXC_ROOTFS}/etc/hosts" | |
| 127.0.0.1 localhost | |
| 127.0.1.1 ${LXC_NAME} | |
| ::1 localhost ip6-localhost ip6-loopback | |
| fe00::0 ip6-localnet | |
| ff00::0 ip6-mcastprefix | |
| ff02::1 ip6-allnodes | |
| ff02::2 ip6-allrouters | |
| EOF | |
| # shellcheck disable=SC2039 | |
| uidgid=($(getuidgid "${OCI_CONF_FILE}" "${LXC_ROOTFS}" )) | |
| # shellcheck disable=SC2039 | |
| echo "lxc.init.uid = ${uidgid[0]}" >> "${LXC_CONF_FILE}" | |
| # shellcheck disable=SC2039 | |
| echo "lxc.init.gid = ${uidgid[1]}" >> "${LXC_CONF_FILE}" | |
| cwd=$(getcwd "${OCI_CONF_FILE}") | |
| echo "lxc.init.cwd = ${cwd}" >> "${LXC_CONF_FILE}" | |
| if [ -n "$LXC_MAPPED_UID" ] && [ "$LXC_MAPPED_UID" != "-1" ]; then | |
| chown "$LXC_MAPPED_UID" "$LXC_PATH/config" "$LXC_PATH/fstab" > /dev/null 2>&1 || true | |
| fi | |
| if [ -n "$LXC_MAPPED_GID" ] && [ "$LXC_MAPPED_GID" != "-1" ]; then | |
| chgrp "$LXC_MAPPED_GID" "$LXC_PATH/config" "$LXC_PATH/fstab" > /dev/null 2>&1 || true | |
| fi | |
| exit 0 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/sh | |
| show() { | |
| local i=0 | |
| echo "$1" | |
| shift | |
| while [ $# -gt 0 ]; do | |
| printf "%02d %s\n" "$i" "$1" | |
| shift | |
| i=$((i+1)) | |
| done | |
| echo "== env ==" | |
| env | grep LXC | |
| return 0 | |
| } | |
| fail(){ echo "$@" 1>&2; exit 1; } | |
| #exec >>/tmp/my.log 2>&1 | |
| #show "$0" "$@" | |
| #set -x | |
| oci="${LXC_ROOTFS_PATH%/rootfs}/oci" | |
| atomfs mount "$oci:${LXC_NAME}" "${LXC_ROOTFS_PATH}" || | |
| fail "mount failed" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| * Plan is to run a lxd vm of jammy. | |
| * Inside install lxc-utils, skopeo, puzzleos ppa, mdp | |
| * Run a local zot | |
| https://gist.github.com/smoser/5a3623e497f90925ff4d9344a85c50a3 | |
| * install python3-pygments (pygmentize) | |
| $ skopeo copy --src-tls-verify=false --insecure-policy | |
| docker://atom-lab-4:5000/docker-sync/alpine:edge oci:./oci-repo:alpine:edge | |
| $ stacker build \ | |
| --substitute=DOCKER_BASE_INSECURE=true \ | |
| --substitute=DOCKER_BASE=docker://atom-lab-4:5000/docker-sync/ stacker.yaml | |
| TODO: | |
| * read for verity info https://www.starlab.io/blog/dm-verity-in-embedded-device-security | |
| * ociv2 https://hackmd.io/@cyphar/ociv2-brainstorm | |
| * ociv2 https://www.cyphar.com/blog/post/20190121-ociv2-images-i-tar | |
| * Need a picture | |
| * overall change tar -> squash | |
| * https://asciiflow.com/#/ | |
| https://github.com/lewish/asciiflow | |
| * need an lxc template script | |
| * notary or cosign | |
| https://docs.sigstore.dev/cosign/overview/ | |
| https://github.com/notaryproject/notaryproject | |
| cosign by ram: https://aci-github.cisco.com/gist/rchincha/cdf87155fab5365b234f8c7ae3df0cd2#file-sign-stacker-yaml-L62 | |
| mkdir squashfs-oci | |
| cd squashfs-oci | |
| cp ../stacker.yaml . | |
| show stacker.yaml | |
| ## FIXME: in vm, have cert registered, stacker.yaml point a localhost | |
| $ stacker build \ | |
| --substitute=DOCKER_BASE_INSECURE=true \ | |
| --substitute=DOCKER_BASE=docker://atom-lab-4:5000/docker-sync/ stacker.yaml | |
| show oci/index.json | |
| show <sha> | |
| # mediaType | |
| $ lxc-create --name=bbdemo --template=atomfs oci:busybox-custom | |
| $ lxc-execute ---name=bbdemo cat /proc/mounts | |
| root | |
| $ sudo PATH=$PWD/bin:$PATH lxc-create -nfooroot -t$PWD/lxc-oci -- | |
| --url=docker://localhost:5000/smoser/talkrootfs-squashfs:latest | |
| non-root | |
| Both have manifest type | |
| application/vnd.oci.image.manifest.v1+json | |
| - application/vnd.oci.image.layer.v1.tar+gzip | |
| + application/vnd.stacker.image.layer.squashfs+zstd+verity | |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "default": [{"type": "reject"}], | |
| "transports": { | |
| "oci": {"": [{"type": "insecureAcceptAnything"}]}, | |
| "docker": { | |
| "docker.io/library": [{"type": "insecureAcceptAnything"}], | |
| "localhost:5000": [ | |
| { | |
| "keyPath": "/home/smoser/cosign.pub", | |
| "signedIdentity": { | |
| "type": "matchRepository" | |
| }, | |
| "type": "sigstoreSigned" | |
| } | |
| ] | |
| } | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # this is /etc/containers/registries.d/default.yaml | |
| #default-docker: | |
| # use-sigstore-attachments: true | |
| docker: | |
| localhost:5000: | |
| use-sigstore-attachments: true | |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/sh | |
| # shellcheck disable=SC3043 | |
| # start in a lxd vm launched like this: | |
| DEFUSER="chicken1" | |
| PUBKEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICE48uGW69AleaoUtgaJRYffXWAajmQAlCbF6TRJquSm smoser@crabapple" | |
| stderr() { echo "$@" 1>&2; } | |
| run() { | |
| local rc="" | |
| stderr "execute:" "$@" | |
| "$@" | |
| rc="$?" | |
| [ $rc -eq 0 ] || stderr "failed: $rc" | |
| return $rc | |
| } | |
| startvm() { | |
| local name="${1:-fosdemo}" | |
| run lxc init --vm images:ubuntu/22.04/cloud "$name" && | |
| run lxc config set "$name" limits.memory 4096MB && | |
| run lxc start "$name" | |
| return | |
| } | |
| getbin(){ | |
| local name="$1" url="$2" f="" | |
| f="/usr/local/bin/$name" | |
| if [ -f "$f" ]; then | |
| stderr "already have $f" | |
| return 0 | |
| fi | |
| wget -O "$f.tmp" "$url" || return | |
| case "$url" in | |
| *.gz) zcat "$f.tmp" > "$f" || { rm -f "$f"; return 1; };; | |
| *) mv "$f.tmp" "$f" || { rm -f "$f.tmp" ; return 1; };; | |
| esac | |
| chmod 755 "$f" | |
| } | |
| setupvm() { | |
| local user="$1" | |
| local f="/etc/sysctl.d/99-dmesg.conf" | |
| run echo "kernel.dmesg_restrict = 0" > "$f" || return | |
| run add-apt-repository -y ppa:puzzleos/dev || return | |
| run apt-get update || return | |
| run apt-get install --no-install-recommends --assume-yes lxc-utils \ | |
| lxc-templates screen python3-pygments squashfuse umoci libsquashfs1 \ | |
| wget ca-certificates git apache2-utils libgpgme11 uidmap openssh-server \ | |
| squashfs-tools file jq \ | |
| apparmor cryptsetup lxcfs \ | |
| libpam-cgfs dbus-user-session \ | |
| || return | |
| [ -f /etc/fuse.conf ] || { stderr "no /etc/fuse.conf"; return 1; } | |
| if ! grep -q ^user_allow_other$ /etc/fuse.conf; then | |
| echo "user_allow_other" >> /etc/fuse.conf | |
| fi | |
| getbin zot https://github.com/project-zot/zot/releases/download/v1.4.3/zot-linux-amd64 && | |
| getbin cosign https://github.com/sigstore/cosign/releases/download/v2.0.0-rc.1/cosign-linux-amd64 && | |
| getbin stacker https://github.com/project-stacker/stacker/releases/download/v1.0.0-rc2/stacker && | |
| getbin zli https://github.com/project-zot/zot/releases/download/v1.4.3/zli-linux-amd64 && | |
| getbin skopeo http://smoser.brickies.net/fosdemo/skopeo.gz && | |
| getbin atomfs http://smoser.brickies.net/fosdemo/atomfs.gz && | |
| : || return | |
| } | |
| doadduser() { | |
| local user="${1:-$DEFUSER}" | |
| run adduser --disabled-password "--gecos=$user" "$user" | |
| run sh -c \ | |
| 'umask 226 && echo "$1 ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/$1-user' -- "$user" | |
| } | |
| gitclone() { | |
| local url="$1" dir="$2" | |
| if [ -d "$dir/.git" ]; then | |
| stderr "$dir already cloned" | |
| ( cd "$dir" && git fetch ) | |
| return | |
| fi | |
| run git clone "$url" "$dir" | |
| } | |
| # run gitclone https://gist.github.com/2e622c67e8a679630d6e70e63b0f32d0.git talk && | |
| zotsetup() { | |
| run gitclone https://gist.github.com/5a3623e497f90925ff4d9344a85c50a3.git local-zot && | |
| : || return 1 | |
| ( cd local-zot && run ./generate-certs certs && mkdir logs ) | |
| ( cd local-zot && run htpasswd -D htpasswd.txt zot && | |
| run htpasswd -bB htpasswd.txt zot zot ) || return | |
| mkdir -p .docker && pw=$(printf "%s" zot:zot | base64) && | |
| cat > .docker/config.json <<EOF | |
| { | |
| "auths": { | |
| "localhost:5000": {"auth": "$pw"} | |
| } | |
| } | |
| EOF | |
| ( cd local-zot && | |
| run sudo cp certs/ca.pem /usr/local/share/ca-certificates/localhost-zot.crt && | |
| run sudo update-ca-certificates | |
| ) || return | |
| run cp ~/talk/zot-sync local-zot/zot-sync && | |
| run cp ~/talk/zot-start local-zot/zot-start && | |
| run ./local-zot/zot-start || return | |
| run zli config add localhost https://127.0.0.1:5000/ | |
| } | |
| zotsync() { | |
| cd ~/local-zot/ && run ./zot-sync | |
| } | |
| usersetup() { | |
| run gitclone https://gist.github.com/2e622c67e8a679630d6e70e63b0f32d0.git talk && | |
| : || return | |
| [ -d .ssh ] || run mkdir --mode 700 .ssh || return | |
| if [ ! -f .ssh/authorized_keys ]; then | |
| : > .ssh/authorized_keys && chmod 600 .ssh/authorized_keys || return | |
| fi | |
| grep -q "$PUBKEY" .ssh/authorized_keys || | |
| printf "%s\n" "$PUBKEY" >>.ssh/authorized_keys || | |
| return | |
| ( cd talk && | |
| sudo mkdir -p /etc/containers/registries.d && | |
| run sudo cp -v registry-default.yaml /etc/containers/registries.d/default.yaml && | |
| run sudo cp -v policy.json /etc/containers/ | |
| ) || return | |
| local d="/usr/share/lxc/templates" tmpl="" | |
| tmpl="$d/lxc-oci" | |
| if [ -f "$tmpl" ] && [ ! -f "$tmpl.dist" ]; then | |
| run sudo cp -v "$tmpl" "$tmpl.dist" || return | |
| fi | |
| ( cd talk && | |
| run sudo cp lxc-oci "$tmpl" && | |
| run sudo chmod 755 "$tmpl" && | |
| run sudo cp show lxc-oci-mount /usr/local/bin && | |
| run sudo chmod ugo+x /usr/local/bin/* ) || | |
| return | |
| cat >.screenrc <<EOF | |
| hardstatus alwayslastline "%-Lw%{= BW}%50>%n%f* %t%{-}%+Lw%< %= %H" | |
| defscrollback 8096 | |
| EOF | |
| zotsetup || return | |
| zotsync || return | |
| lxcsetup || return | |
| sudo stacker unpriv-setup | |
| } | |
| lxcsetup() { | |
| local user=${1:-${DEFUSER}} | |
| mkdir -p .config/lxc | |
| cat > .config/lxc/default.conf <<EOF | |
| lxc.include = /etc/lxc/default.conf | |
| lxc.idmap = u 0 165536 65536 | |
| lxc.idmap = g 0 165536 65536 | |
| EOF | |
| [ $? -eq 0 ] || return | |
| local lpath="/lxc/$user" | |
| cat > .config/lxc/lxc.conf <<EOF | |
| lxc.lxcpath = $lpath | |
| EOF | |
| [ $? -eq 0 ] || return | |
| run sudo mkdir -p "$lpath" && | |
| run sudo chown "$user:$user" "$lpath" || | |
| return | |
| local f=/etc/lxc/lxc-usernet | |
| if [ -f "$f" ]; then | |
| sudo touch "$f" | |
| fi | |
| if ! grep -q "$user" "$f"; then | |
| echo "$user veth lxcbr0 32" | run sudo tee "$f" | |
| fi | |
| # You need to edit | |
| # /etc/apparmor.d/abstractions/lxc/start-container | |
| local aaprof="/etc/apparmor.d/abstractions/lxc/start-container" | |
| if [ ! -f "$aaprof" ]; then | |
| echo no "$aaprof file" | |
| return 1 | |
| fi | |
| if ! grep -q 'fstype=fuse[*]' "$aaprof"; then | |
| if [ ! -f "$aaprof.dist" ]; then | |
| run cp "$aaprof" "$aaprof.dist" | |
| fi | |
| cat >>"$aaprof" <<EOF | |
| # add 'mount fstype=fuse*' | |
| # without this here (and with the '*'), we get: | |
| # dmesg errors like fstype="fuse.squashfuse_ll" srcname="squashfuse_ll" | |
| mount fstype=fuse*, | |
| EOF | |
| [ $? -eq 0 ] || return | |
| fi | |
| # FIXME: you also may need to add 'fstype=fuse' to | |
| # /etc/apparmor.d/lxc/lxc-default-cgns | |
| } | |
| case "$1" in | |
| startvm|setupvm|doadduser|usersetup) | |
| n="$1" | |
| shift | |
| $n "$@" | |
| exit;; | |
| main) | |
| setupvm && doadduser && | |
| sudo -Hu "$DEFUSER" --login -- "$0" usersetup | |
| ;; | |
| *) stderr "unknown command $1"; exit 1;; | |
| esac |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/sh | |
| # shellcheck disable=SC2166 | |
| Usage() { | |
| cat <<EOF | |
| ${0##*/} [type] file | |
| show file of type 'type'. if type not given, then try to figure out. | |
| EOF | |
| } | |
| fail(){ echo "$@" 1>&2; exit 1; } | |
| [ $# -eq 1 -o $# -eq 2 ] || { Usage 1>&2; exit 1; } | |
| [ "$1" = "-h" -o "$1" = "--help" ] && { Usage; exit 0; } | |
| [ $# -eq 2 ] && ftype="$1" && shift | |
| file="$1" | |
| if [ ! -f "$file" ]; then | |
| cand=$(echo "$file" | sed 's,sha256:,sha256/,') | |
| if [ -f "$cand" ]; then | |
| file="$cand" | |
| elif [ -f "oci/blobs/$cand" ]; then | |
| file="oci/blobs/$cand" | |
| else | |
| fail "$file: not a file" | |
| fi | |
| fi | |
| decomp="" | |
| if [ -z "$ftype" -o "$ftype" = "auto" ]; then | |
| fout=$(file --uncompress --dereference "$file") || | |
| fail "failed to get type of $file" | |
| case "$fout" in | |
| *gzip\ compressed*) | |
| decomp="zcat";; | |
| esac | |
| fmt="cat" | |
| ftype="" | |
| case "$fout" in | |
| *JSON*) | |
| fmt="python3 -m json.tool" | |
| ftype="json";; | |
| *tar\ archive*) | |
| fmt="tar -tvf -" | |
| ftype="fstar";; | |
| *shell\ script*) | |
| ftype="shell";; | |
| *) | |
| case "$file" in | |
| *.yaml) ftype="yaml";; | |
| esac | |
| esac | |
| fi | |
| [ -n "$ftype" ] && targ="-l$ftype" || targ="-g" | |
| set -- pygmentize "$targ" -O style=solarized-dark,linenos=1 | |
| echo "$file" | |
| if [ -n "$decomp" ]; then | |
| echo "fmt=$fmt" | |
| $decomp "$file" | $fmt | "$@" | |
| elif [ "$fmt" = "cat" ]; then | |
| # if we were just going to cat the file, better auto with filename | |
| "$@" "$file" | |
| else | |
| $fmt "$file" | "$@" | |
| fi |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| minbase: | |
| build_only: true | |
| from: | |
| type: docker | |
| url: ${{DOCKER_BASE:docker://docker.io/library/}}ubuntu:jammy | |
| minboot: | |
| build_only: true | |
| from: | |
| type: built | |
| tag: minbase | |
| run: | | |
| apt-get update -q | |
| apt-get install --yes --no-install-recommends systemd-sysv | |
| talkrootfs: | |
| from: | |
| type: built | |
| tag: minboot | |
| run: | | |
| # password 'none' | |
| hashpass='$6$cpHPicKJ$O3lnesUxTQYSfuFtllAQZEf4xPjqSQMjV.hOCwATxUBm/oPVSA7zlTZSHffkXIDpIsW7/ec7chroIjo.7sV6i/' | |
| sed -i 's@^root:[^:]*:\(.*$\)@root:'"$hashpass"':\1@' /etc/shadow | |
| cat > /etc/rc.local <<"EOF" | |
| read up idle </proc/uptime | |
| echo "rc.local was run at $up seconds" > /run/rc.local.out | |
| EOF |
%title: Quick starting secure container storage using squashfs, overlay and dm-verity %author: Scott Moser %date: 2023-02-05
-> # Me
- Cisco
- Project-machine https://github.com/project-machine
- [email protected]
-> # Intro
- Goal is simple: Replace use of OCI images in tar+gz images with squasfs. ^
- Path there:
^
- Comparision of types stored in a registry ^
- Comparision of runtime use. ^
- Sales Pitch^H^H^H^H^H Demo
^
- Build --> Stacker ^
- Sign --> cosign ^
- Publish --> Zot ^
- Run --> lxc
-> # Registry Comparison ┌─────────────────────────────────────────────────────────────────────┐ │ │ │ Image Registry │ │ │ │ TAR.GZ │ SQUASHFS+VERITY │ │ ────────────────────────────┼─────────────────────────── │ │ - Registry stores OCI Images │ - Registry stores OCI Images │ │ │ │ │ - Images list layers │ - Images list layers │ │ │ │ │ - Layers are mediaType │ - Layers are mediaType │ │ oci.image. │ stacker.image. │ │ + layer.v1.tar+gzip │ + layer.squashfs+zstd+verity │ │ │ │ │ - signed checksum of tarball │ - signed checksum of image │ │ │ │ │ │ - signed dm-verity hash │ │ │ │ └─────────────────────────────────────────────────────────────────────┘
-> # Runtime Comparison ┌─────────────────────────────────────────────────────────────────────┐ │ │ │ Container Use / Runtime │ │ │ │ TAR.GZ │ SQUASHFS+VERITY │ │ ────────────────────────────┼─────────────────────────── │ │ - layers uncompress and untar │ - layer (can be) used from oci or │ │ │ copied per-container │ │ │ │ │ - Compare to original tar │ - Compare via sha256sum │ │ file-by-file │ │ │ │ │ │ │ - Privileged mounts get dm-verity │ │ │ │ │ │ - Unpriv mount via squashfuse │ │ │ │ │ │ - FS does not implement write │ │ │ │ │ │ - less easily readable │ └─────────────────────────────────────────────────────────────────────┘
-> # Write support
Overlayfs
- View a series of images overlayed into a single view ^
- Support for whiteouts and copy-ups just like oci format. ^
- Overlay bugs still present but largely handled ^
- Changes to the write layer can be easily seen
-> # dm-verity
^
- transparent integrity checking of block devices
^
- root hash is signed as part of image metadata
^
- block device read-time validation rather than up front checksum.
^
- Requires root - no unprivileged use of device-mapper.
-> # Demo
^
-
Warning: Users sensitive to invented-here-software might want to divert their eyes. (It is good software though.) ^
-
Build (stacker) - https://github.com/project-stacker/stacker ^
-
Sign (cosign) ^
-
Publish (zot) - https://github.com/project-zot/zot ^
-
Run (lxc)
-> # Stacker
- OCI Image build tool ^
- Unprivileged Builds ^
- Can be used to provide a container to build your software in. ^
- 5 years development. ^
- Going through CNCF inclusion process
-> # Sign
- Sign the image with cosign. The dm-verity information is present
-> # Zot ^
- Single Binary, easy to develop against ^
- Unprivileged ^
- Just like docker.io ^
- CNCF Sandbox project
-> # Thanks / Questions
- God, Team, Family, Cisco, You
- https://github.com/project-machine
- Questions
- Scott Moser [email protected]
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| http: | |
| # set this to literal 'address: ""' to listen on all interfaces. | |
| # https://github.com/project-zot/zot/issues/1063 | |
| address: "127.0.0.1" | |
| port: 5000 | |
| realm: "localhost-zot" | |
| tls: | |
| cert: certs/server-cert.pem | |
| key: certs/server-key.pem | |
| auth: | |
| htpasswd: | |
| path: "htpasswd.txt" | |
| accessControl: | |
| # by default, anonymous read. the zot user can read/write | |
| "**": | |
| anonymousPolicy: [read] | |
| defaultPolicy: [read] | |
| policies: | |
| - users: [zot] | |
| actions: [read, create, update, delete, detectManifestCollision] | |
| log: | |
| level: "debug" | |
| audit: "logs/audit.log" | |
| output: "logs/zot.log" | |
| storage: | |
| dedupe: true | |
| rootdirectory: "storage" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/sh | |
| Usage() { | |
| cat <<EOF | |
| ${0##*/} [dir] | |
| run ran server serving dir on port. | |
| EOF | |
| } | |
| fail() { echo "$@" 1>&2; exit 1; } | |
| dir=${2:-$HOME/local-zot} | |
| [ "$1" = "-h" -o "$1" = "--help" ] && { Usage; exit 0; } | |
| dir=$( cd "$dir" && pwd) || fail "failed cd $dir" | |
| cd "$dir" || fail "failed cd $dir" | |
| log_d="$HOME/logs" | |
| log="${log_d}/zot.log" | |
| pidf="$log.pid" | |
| mkdir -p "$log_d" || fail "failed mkdir $log_d" | |
| if [ -f "$pidf" ]; then | |
| if read p <"$pidf" && [ -n "$p" ] && [ -d "/proc/$p" ]; then | |
| echo "killing existing pid $p" | |
| kill $p | |
| sleep 1 | |
| fi | |
| fi | |
| config="config.yaml" | |
| zot="./bin/zot" | |
| if [ ! -x "$zot" ]; then | |
| zot=$(command -v "zot") 2>/dev/null || fail "no zot found" | |
| fi | |
| mkdir -p logs storage || fail "failed to make dirs in $PWD" | |
| [ -f "$config" ] || fail "no config '$config' in $PWD" | |
| [ -x "$zot" ] || fail "no zot bin '$zot' in $PWD" | |
| set -- $zot serve "$config" | |
| echo "$(date -R):" "writing to $log: $*" 1>&2 | |
| sh -ec ' | |
| pidf=$1; shift; echo $$ > "$pidf"; | |
| echo "# running in $PWD: $*" | |
| exec "$@"' -- \ | |
| "$pidf" "$@" </dev/null > "$log" 2>&1 & | |
| newpid=$! | |
| sleep 1 | |
| [ -d "/proc/$newpid" ] || { cat "$log" 1>&2; fail "failed to start $*"; } | |
| cat "$log" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| # sync some docker images to zot | |
| # for these docker urls, then you can save yourself from | |
| # the docker bandwidth limit by just referencing | |
| # * docker://localhost:5000/docker-sync/ubuntu:latest | |
| # | |
| # Run this with some frequency, as up to date images are good to have. | |
| DOCKER_URLS=( | |
| "docker://ubuntu:jammy" | |
| "docker://busybox:latest" | |
| ) | |
| # Sync to a oci local directory to use less bandwidth from docker | |
| # and also to resume more safely. | |
| SYNC_OCI="oci:/tmp/sync-oci.d" | |
| ZOT_BASE="docker://localhost:5000/docker-sync" | |
| fail() { echo "$@" 1>&2; exit 1; } | |
| msg() { echo "$(date -R)" "$@"; } | |
| mkdir -p "${SYNC_OCI#oci:}" || | |
| fail "failed to create ${SYNC_OCI} dir for local sync" | |
| for dockurl in "${DOCKER_URLS[@]}"; do | |
| base=${dockurl##*/} | |
| ociurl="${SYNC_OCI}:$base" | |
| msg "sync $dockurl -> $ociurl" | |
| skopeo copy "$dockurl" "$ociurl" || | |
| fail "failed to sync $dockurl -> $ociurl" | |
| done | |
| for dockurl in "${DOCKER_URLS[@]}"; do | |
| base=${dockurl##*/} | |
| zoturl="${ZOT_BASE%/}/${base}" | |
| ociurl="${SYNC_OCI}:$base" | |
| msg "publish $ociurl -> $zoturl" | |
| skopeo copy --dest-tls-verify "$ociurl" "$zoturl" || | |
| fail "failed to publish $ociurl -> $zoturl" | |
| done | |
| msg "done" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment