[fosdem 2023 talk] oci images in squashfs format with dm-verity

README.md

Fosdem 2023 Talk on use of squashfs in oci images

lxc-oci

#!/bin/bash
# Create application containers from OCI images

# Copyright © 2014 Stéphane Graber <[email protected]>
# Copyright © 2017 Serge Hallyn <[email protected]>
#
#  This library is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public
#  License as published by the Free Software Foundation; either
#  version 2.1 of the License, or (at your option) any later version.

#  This library is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  Lesser General Public License for more details.

#  You should have received a copy of the GNU Lesser General Public
#  License along with this library; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#  USA

set -eu

# Make sure the usual locations are in PATH
export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin"

# Check for required binaries
for bin in skopeo umoci jq; do
  if ! command -v $bin >/dev/null 2>&1; then
    echo "ERROR: Missing required tool: $bin" 1>&2
    exit 1
  fi
done

LXC_OCI_MOUNT="lxc-oci-mount"
ATOMFS="atomfs"
LOCALSTATEDIR=/var
LXC_TEMPLATE_CONFIG=/usr/share/lxc/config
LXC_HOOK_DIR=/usr/share/lxc/hooks
OCI_MOUNT=""

# Some useful functions
cleanup() {
  if [ -d "${LXC_ROOTFS}.tmp" ]; then
    rm -Rf "${LXC_ROOTFS}.tmp"
  fi
  if [ -n "$OCI_MOUNT" ]; then
    echo "atomfs unmount $OCI_MOUNT" >&2
    "${ATOMFS}" umount "$OCI_MOUNT"
    OCI_MOUNT=""
  fi
}

in_userns() {
  [ -e /proc/self/uid_map ] || { echo no; return; }
  while read -r line; do
    fields="$(echo "$line" | awk '{ print $1 " " $2 " " $3 }')"
    if [ "${fields}" = "0 0 4294967295" ]; then
      echo no;
      return;
    fi
    if echo "${fields}" | grep -q " 0 1$"; then
      echo userns-root;
      return;
    fi
  done < /proc/self/uid_map

  if [ -e /proc/1/uid_map ]; then
    if [ "$(cat /proc/self/uid_map)" = "$(cat /proc/1/uid_map)" ]; then
      echo userns-root
      return
    fi
  fi
  echo yes
}

getmanifestpath() {
  local basedir="$1" ref="$2" p=""
  # if given 'sha256:<hash>' then return the blobs/sha256/hash
  case "$ref" in
    sha256:*)
      p="$basedir/blobs/sha256/${ref#sha256:}"
      [ -f "$p" ] && echo "$p" && return 0
      echo "could not find manifest path to blob $ref. file did not exist: $p" >&2
      return 1
  esac
  # find the reference by annotation
  local blobref="" hashtype="" hashval=""
  blobref=$(jq -c -r --arg q "$ref" '.manifests[] | if .annotations."org.opencontainers.image.ref.name" == $q then .digest else empty end' < "${basedir}/index.json")
  # blobref is 'hashtype:hash'
  hashtype="${blobref%%:*}"
  hashval="${blobref#*:}"
  p="$basedir/blobs/$hashtype/$hashval"
  [ -f "$p" ] && echo "$p" && return 0
  echo "did not find manifest for $ref. file did not exist: $p" >&2
  return 1
}

getconfigpath() {
  local basedir="$1" mfpath="$2" cdigest=""
  # Ok we have the image config digest, now get the config ref from the manifest.
  # shellcheck disable=SC2039
  cdigest=$(jq -c -r '.config.digest' < "$mfpath")
  if [ -z "${cdigest}" ]; then
    echo "container config not found" >&2
    return
  fi

  # cdigest is 'hashtype:hash'
  local ht="${cdigest%%:*}" hv="${cdigest#*:}" p=""
  p="$basedir/blobs/$ht/$hv"
  if [ ! -f "$p" ]; then
    echo "config file did not exist for digest $cdigest" >&2
    return 1
  fi
  echo "$p"
}

getlayermediatype() {
  jq -c -r '.layers[0].mediaType' <"$1"
}

# Get entrypoint from oci image. Use sh if unspecified
getep() {
  if [ "$#" -eq 0 ]; then
    echo "/bin/sh"
    return
  fi

  configpath="$1"

  ep=$(jq -c '.config.Entrypoint[]?'< "${configpath}" | tr '\n' ' ')
  cmd=$(jq -c '.config.Cmd[]?'< "${configpath}" | tr '\n' ' ')
  if [ -z "${ep}" ]; then
    ep="${cmd}"
    if [ -z "${ep}" ]; then
      ep="/bin/sh"
    fi
  elif [ -n "${cmd}" ]; then
    ep="${ep} ${cmd}"
  fi

  echo "${ep}"
  return
}

# get environment from oci image.
getenv() {
  if [ "$#" -eq 0 ]; then
    return
  fi

  configpath="$1"

  env=$(jq -c -r '.config.Env[]'< "${configpath}")

  echo "${env}"
  return
}

# check var is decimal.
isdecimal() {
  var="$1"
  if [ "${var}" -eq "${var}" ] 2> /dev/null; then
    return 0
  else 
    return 1
  fi
}

# get uid, gid from oci image.
getuidgid() {
  configpath="$1"
  rootpath="$2"
  passwdpath="${rootpath}/etc/passwd"
  grouppath="${rootpath}/etc/group"

  usergroup=$(jq -c -r '.config.User' < "${configpath}")
  # shellcheck disable=SC2039
  usergroup=(${usergroup//:/ })

  user=${usergroup[0]:-0}
  if ! isdecimal "${user}"; then
    if [ -f ${passwdpath} ]; then
      user=$(grep "^${user}:" "${passwdpath}" | awk -F: '{print $3}')
    else
      user=0
    fi
  fi

  group=${usergroup[1]:-}
  if [ -z "${group}" ]; then
    if [ -f "${passwdpath}" ]; then
      group=$(grep "^[^:]*:[^:]*:${user}:" "${passwdpath}" | awk -F: '{print $4}')
    else
      group=0
    fi
  elif ! isdecimal "${group}"; then
    if [ -f "${grouppath}" ]; then
      group=$(grep "^${group}:" "${grouppath}" | awk -F: '{print $3}')
    else
      group=0
    fi
  fi

  echo "${user:-0} ${group:-0}"
  return
}

# get cwd from oci image.
getcwd() {
  if [ "$#" -eq 0 ]; then
    echo "/"
    return
  fi

  configpath="$1"

  cwd=$(jq -c -r '.config.WorkingDir // "/"' < "${configpath}")

  echo "${cwd}"
  return
}

usage() {
  cat <<EOF
LXC container template for OCI images

Special arguments:
[ -h | --help ]: Print this help message and exit.

Required arguments:
[ -u | --url <url> ]: The OCI image URL

Optional arguments:
[ --username <username> ]: The username for the registry
[ --password <password> ]: The password for the registry

LXC internal arguments (do not pass manually!):
[ --name <name> ]: The container name
[ --path <path> ]: The path to the container
[ --rootfs <rootfs> ]: The path to the container's rootfs
[ --mapped-uid <map> ]: A uid map (user namespaces)
[ --mapped-gid <map> ]: A gid map (user namespaces)
EOF
  return 0
}
echo "got $#: $*"

if ! options=$(getopt -o u:h -l help,url:,username:,password:,no-cache,dhcp,name:,path:,rootfs:,mapped-uid:,mapped-gid: -- "$@"); then
    usage
    exit 1
fi
eval set -- "$options"

OCI_URL=""
OCI_USERNAME=
OCI_PASSWORD=
OCI_USE_CACHE="true"
OCI_USE_DHCP="false"

LXC_MAPPED_GID=
LXC_MAPPED_UID=
LXC_NAME=
LXC_PATH=
LXC_ROOTFS=

while :; do
  case "$1" in
    -h|--help)    usage && exit 1;;
    -u|--url)     OCI_URL=$2; shift 2;;
    --username)   OCI_USERNAME=$2; shift 2;;
    --password)   OCI_PASSWORD=$2; shift 2;;
    --no-cache)   OCI_USE_CACHE="false"; shift 1;;
    --dhcp)       OCI_USE_DHCP="true"; shift 1;;
    --name)       LXC_NAME=$2; shift 2;;
    --path)       LXC_PATH=$2; shift 2;;
    --rootfs)     LXC_ROOTFS=$2; shift 2;;
    --mapped-uid) LXC_MAPPED_UID=$2; shift 2;;
    --mapped-gid) LXC_MAPPED_GID=$2; shift 2;;
    *)            break;;
  esac
done

# Check that we have all variables we need
if [ -z "$LXC_NAME" ] || [ -z "$LXC_PATH" ] || [ -z "$LXC_ROOTFS" ]; then
  echo "ERROR: Not running through LXC" 1>&2
  exit 1
fi

if [ -z "$OCI_URL" ]; then
  echo "ERROR: no OCI URL given"
  exit 1
fi

if [ -n "$OCI_PASSWORD" ] && [ -z "$OCI_USERNAME" ]; then
  echo "ERROR: password given but no username specified"
  exit 1
fi

if [ "${OCI_USE_CACHE}" = "true" ]; then
  if ! skopeo copy --help | grep -q 'dest-shared-blob-dir'; then
    echo "INFO: skopeo doesn't support blob caching"
    OCI_USE_CACHE="false"
  fi
fi

USERNS=$(in_userns)

if [ "$USERNS" = "yes" ]; then
  if [ -z "$LXC_MAPPED_UID" ] || [ "$LXC_MAPPED_UID" = "-1" ]; then
    echo "ERROR: In a user namespace without a map" 1>&2
    exit 1
  fi
fi

OCI_DIR="$LXC_PATH/oci"
if [ "${OCI_USE_CACHE}" = "true" ]; then
  if [ "$USERNS" = "yes" ]; then
    DOWNLOAD_BASE="${HOME}/.cache/lxc"
  else
    DOWNLOAD_BASE="${LOCALSTATEDIR}/cache/lxc"
  fi
else
  DOWNLOAD_BASE="$OCI_DIR"
fi
mkdir -p "${DOWNLOAD_BASE}"

# Trap all exit signals
trap cleanup EXIT HUP INT TERM

# Download the image
# shellcheck disable=SC2039
skopeo_args=("--remove-signatures" --insecure-policy)
if [ -n "$OCI_USERNAME" ]; then
  CREDENTIALS="${OCI_USERNAME}"

  if [ -n "$OCI_PASSWORD" ]; then
    CREDENTIALS="${CREDENTIALS}:${OCI_PASSWORD}"
  fi

  # shellcheck disable=SC2039
  skopeo_args+=(--src-creds "${CREDENTIALS}")
fi

OCI_NAME="$LXC_NAME"
if [ "${OCI_USE_CACHE}" = "true" ]; then
  skopeo_args+=(--dest-shared-blob-dir "${DOWNLOAD_BASE}")
  mkdir -p "${OCI_DIR}/blobs/"
  ln -s "${DOWNLOAD_BASE}/sha256" "${OCI_DIR}/blobs/sha256"
fi

skopeo copy "${skopeo_args[@]}" "${OCI_URL}" "oci:${OCI_DIR}:${OCI_NAME}"

mfpath=$(getmanifestpath "${OCI_DIR}" "${OCI_NAME}")
OCI_CONF_FILE=$(getconfigpath "${OCI_DIR}" "$mfpath")
mediatype=$(getlayermediatype "$mfpath")
echo "mfpath=$mfpath conf=$OCI_CONF_FILE" 1>&2
echo "mediatype=$mediatype"
echo "DOWNLOAD_BASE=$DOWNLOAD_BASE" >&2
echo "OCI_DIR=$OCI_DIR" >&2

case "$mediatype" in
  #application/vnd.oci.image.layer.v1.tar+gzip
  application/vnd.oci.image.layer.v1.tar*) 
    echo "Unpacking tar rootfs" 2>&1
    # shellcheck disable=SC2039
    umoci_args=("")
    if [ -n "$LXC_MAPPED_UID" ] && [ "$LXC_MAPPED_UID" != "-1" ]; then
      # shellcheck disable=SC2039
      umoci_args+=(--rootless)
    fi
    # shellcheck disable=SC2039
    # shellcheck disable=SC2068
    umoci --log=error unpack ${umoci_args[@]} --image "${OCI_DIR}:${OCI_NAME}" "${LXC_ROOTFS}.tmp"
    find "${LXC_ROOTFS}.tmp/rootfs" -mindepth 1 -maxdepth 1 -exec mv '{}' "${LXC_ROOTFS}/" \;
    ;;
  #application/vnd.stacker.image.layer.squashfs+zstd+verity
  application/vnd.*.image.layer.squashfs*)
    echo "squashfs type $mediatype" >&2
    echo ${ATOMFS} mount "${OCI_DIR}:${OCI_NAME}" "$LXC_ROOTFS" >&2
    ${ATOMFS} mount "${OCI_DIR}:${OCI_NAME}" "$LXC_ROOTFS"
    OCI_MOUNT="$LXC_ROOTFS"
    ;;
  *)
    echo "Unknown media type $mediatype" >&2
    exit 1
    ;;
esac

LXC_CONF_FILE="${LXC_PATH}/config"
entrypoint=$(getep "${OCI_CONF_FILE}")
echo "lxc.execute.cmd = '${entrypoint}'" >> "${LXC_CONF_FILE}"
echo "lxc.mount.auto = proc:mixed sys:mixed cgroup:mixed" >> "${LXC_CONF_FILE}"

case "$mediatype" in
  application/vnd.*.image.layer.squashfs*)
    echo "lxc.hook.pre-mount = ${LXC_OCI_MOUNT}" >> "${LXC_CONF_FILE}";;
esac

environment=$(getenv "${OCI_CONF_FILE}")
# shellcheck disable=SC2039
while read -r line; do
  echo "lxc.environment = ${line}" >> "${LXC_CONF_FILE}"
done <<< "${environment}"

if [ -e "${LXC_TEMPLATE_CONFIG}/common.conf" ]; then
  echo "lxc.include = ${LXC_TEMPLATE_CONFIG}/common.conf" >> "${LXC_CONF_FILE}"
fi

if [ -n "$LXC_MAPPED_UID" ] && [ "$LXC_MAPPED_UID" != "-1" ] && [ -e "${LXC_TEMPLATE_CONFIG}/userns.conf" ]; then
  echo "lxc.include = ${LXC_TEMPLATE_CONFIG}/userns.conf" >> "${LXC_CONF_FILE}"
fi

if [ -e "${LXC_TEMPLATE_CONFIG}/oci.common.conf" ]; then
  echo "lxc.include = ${LXC_TEMPLATE_CONFIG}/oci.common.conf" >> "${LXC_CONF_FILE}"
fi

if [ "${OCI_USE_DHCP}" = "true" ]; then
  echo "lxc.hook.start-host = ${LXC_HOOK_DIR}/dhclient" >> "${LXC_CONF_FILE}"
  echo "lxc.hook.stop = ${LXC_HOOK_DIR}/dhclient" >> "${LXC_CONF_FILE}"
fi

echo "lxc.uts.name = ${LXC_NAME}" >> "${LXC_CONF_FILE}"
# set the hostname
cat <<EOF > "${LXC_ROOTFS}/etc/hostname"
${LXC_NAME}
EOF

# set minimal hosts
cat <<EOF > "${LXC_ROOTFS}/etc/hosts"
127.0.0.1   localhost
127.0.1.1   ${LXC_NAME}
::1     localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
EOF

# shellcheck disable=SC2039
uidgid=($(getuidgid "${OCI_CONF_FILE}" "${LXC_ROOTFS}" ))
# shellcheck disable=SC2039
echo "lxc.init.uid = ${uidgid[0]}" >> "${LXC_CONF_FILE}"
# shellcheck disable=SC2039
echo "lxc.init.gid = ${uidgid[1]}" >> "${LXC_CONF_FILE}"

cwd=$(getcwd "${OCI_CONF_FILE}")
echo "lxc.init.cwd = ${cwd}" >> "${LXC_CONF_FILE}"

if [ -n "$LXC_MAPPED_UID" ] && [ "$LXC_MAPPED_UID" != "-1" ]; then
  chown "$LXC_MAPPED_UID" "$LXC_PATH/config" "$LXC_PATH/fstab" > /dev/null 2>&1 || true
fi
if [ -n "$LXC_MAPPED_GID" ] && [ "$LXC_MAPPED_GID" != "-1" ]; then
  chgrp "$LXC_MAPPED_GID" "$LXC_PATH/config" "$LXC_PATH/fstab" > /dev/null 2>&1 || true
fi

exit 0

lxc-oci-mount

#!/bin/sh
show() {
    local i=0
    echo "$1"
    shift
    while [ $# -gt 0 ]; do
      printf "%02d %s\n" "$i" "$1"
      shift
      i=$((i+1))
    done
    echo "== env =="
    env | grep LXC
    return 0
}
fail(){ echo "$@" 1>&2; exit 1; }

#exec >>/tmp/my.log 2>&1
#show "$0" "$@"
#set -x
oci="${LXC_ROOTFS_PATH%/rootfs}/oci"
atomfs mount "$oci:${LXC_NAME}" "${LXC_ROOTFS_PATH}" ||
    fail "mount failed"

notes.txt

* Plan is to run a lxd vm of jammy.
* Inside install lxc-utils, skopeo, puzzleos ppa, mdp
* Run a local zot
  https://gist.github.com/smoser/5a3623e497f90925ff4d9344a85c50a3
* install python3-pygments (pygmentize)

$ skopeo copy --src-tls-verify=false --insecure-policy 
   docker://atom-lab-4:5000/docker-sync/alpine:edge oci:./oci-repo:alpine:edge

$ stacker build \
   --substitute=DOCKER_BASE_INSECURE=true \
   --substitute=DOCKER_BASE=docker://atom-lab-4:5000/docker-sync/ stacker.yaml

TODO:
 * read for verity info https://www.starlab.io/blog/dm-verity-in-embedded-device-security
 * ociv2 https://hackmd.io/@cyphar/ociv2-brainstorm 
 * ociv2 https://www.cyphar.com/blog/post/20190121-ociv2-images-i-tar 
 * Need a picture 
   * overall change tar -> squash
   * https://asciiflow.com/#/
     https://github.com/lewish/asciiflow
 * need an lxc template script
 * notary or cosign
   https://docs.sigstore.dev/cosign/overview/
   https://github.com/notaryproject/notaryproject
   cosign by ram: https://aci-github.cisco.com/gist/rchincha/cdf87155fab5365b234f8c7ae3df0cd2#file-sign-stacker-yaml-L62

mkdir squashfs-oci
cd squashfs-oci
cp ../stacker.yaml .
show stacker.yaml

## FIXME: in vm, have cert registered, stacker.yaml point a localhost
$ stacker build \
   --substitute=DOCKER_BASE_INSECURE=true \
   --substitute=DOCKER_BASE=docker://atom-lab-4:5000/docker-sync/ stacker.yaml

show oci/index.json
show <sha>
# mediaType

$ lxc-create --name=bbdemo --template=atomfs oci:busybox-custom
$ lxc-execute ---name=bbdemo cat /proc/mounts

root
 $ sudo PATH=$PWD/bin:$PATH lxc-create -nfooroot  -t$PWD/lxc-oci --
      --url=docker://localhost:5000/smoser/talkrootfs-squashfs:latest
non-root



Both have manifest type 
application/vnd.oci.image.manifest.v1+json
 - application/vnd.oci.image.layer.v1.tar+gzip
 + application/vnd.stacker.image.layer.squashfs+zstd+verity

policy.json

{
 "default": [{"type": "reject"}],
 "transports": {
  "oci": {"": [{"type": "insecureAcceptAnything"}]},
  "docker": {
   "docker.io/library": [{"type": "insecureAcceptAnything"}],
   "localhost:5000": [
    {
     "keyPath": "/home/smoser/cosign.pub",
     "signedIdentity": {
      "type": "matchRepository"
     },
     "type": "sigstoreSigned"
    }
   ]
  }
 }
}

registry-default.yaml

# this is /etc/containers/registries.d/default.yaml
#default-docker:
#  use-sigstore-attachments: true
docker:
  localhost:5000:
    use-sigstore-attachments: true

setup

#!/bin/sh
# shellcheck disable=SC3043
# start in a lxd vm launched like this:
DEFUSER="chicken1"
PUBKEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICE48uGW69AleaoUtgaJRYffXWAajmQAlCbF6TRJquSm smoser@crabapple"

stderr() { echo "$@" 1>&2; }
run() {
   local rc=""
   stderr "execute:" "$@"
   "$@"
   rc="$?"
   [ $rc -eq 0 ] || stderr "failed: $rc"
   return $rc
}

startvm() {
    local name="${1:-fosdemo}"
    run lxc init --vm images:ubuntu/22.04/cloud "$name" &&
        run lxc config set "$name" limits.memory 4096MB &&
        run lxc start "$name"
    return
}

getbin(){
    local name="$1" url="$2" f=""
    f="/usr/local/bin/$name"
    if [ -f "$f" ]; then
        stderr "already have $f"
        return 0
    fi
    wget -O "$f.tmp" "$url" || return
    case "$url" in
        *.gz) zcat "$f.tmp" > "$f" || { rm -f "$f"; return 1; };;
        *) mv "$f.tmp" "$f" || { rm -f "$f.tmp" ; return 1; };;
    esac
    chmod 755 "$f"
}

setupvm() {
    local user="$1"
    local f="/etc/sysctl.d/99-dmesg.conf"
    run echo "kernel.dmesg_restrict = 0" > "$f" || return
    run add-apt-repository -y ppa:puzzleos/dev || return
    run apt-get update || return
    run apt-get install --no-install-recommends --assume-yes lxc-utils \
       lxc-templates screen python3-pygments squashfuse umoci libsquashfs1 \
       wget ca-certificates git apache2-utils libgpgme11 uidmap openssh-server \
       squashfs-tools file jq \
       apparmor cryptsetup lxcfs \
       libpam-cgfs dbus-user-session \
       || return

    [ -f /etc/fuse.conf ] || { stderr "no /etc/fuse.conf"; return 1; }
    if ! grep -q ^user_allow_other$ /etc/fuse.conf; then
        echo "user_allow_other" >> /etc/fuse.conf
    fi

    getbin zot https://github.com/project-zot/zot/releases/download/v1.4.3/zot-linux-amd64 &&
        getbin cosign https://github.com/sigstore/cosign/releases/download/v2.0.0-rc.1/cosign-linux-amd64 &&
        getbin stacker https://github.com/project-stacker/stacker/releases/download/v1.0.0-rc2/stacker &&
        getbin zli https://github.com/project-zot/zot/releases/download/v1.4.3/zli-linux-amd64 &&
        getbin skopeo http://smoser.brickies.net/fosdemo/skopeo.gz &&
        getbin atomfs http://smoser.brickies.net/fosdemo/atomfs.gz &&
        : || return
}

doadduser() {
    local user="${1:-$DEFUSER}"
    run adduser --disabled-password "--gecos=$user" "$user"
    run sh -c \
        'umask 226 && echo "$1 ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/$1-user' -- "$user"
}

gitclone() {
    local url="$1" dir="$2"
    if [ -d "$dir/.git" ]; then
        stderr "$dir already cloned"
        ( cd "$dir" && git fetch )
        return
    fi
    run git clone "$url" "$dir"
}

#    run gitclone https://gist.github.com/2e622c67e8a679630d6e70e63b0f32d0.git talk &&
zotsetup() {

    run gitclone https://gist.github.com/5a3623e497f90925ff4d9344a85c50a3.git local-zot &&
        : || return 1

    ( cd local-zot && run ./generate-certs certs && mkdir logs )
    ( cd local-zot && run htpasswd -D htpasswd.txt zot &&
      run htpasswd -bB htpasswd.txt zot zot ) || return

    mkdir -p .docker && pw=$(printf "%s" zot:zot | base64) &&
        cat > .docker/config.json <<EOF
{
  "auths": {
    "localhost:5000": {"auth": "$pw"}
  }
}
EOF

    ( cd local-zot &&
      run sudo cp certs/ca.pem /usr/local/share/ca-certificates/localhost-zot.crt &&
        run sudo update-ca-certificates
    ) || return

    run cp ~/talk/zot-sync local-zot/zot-sync &&
    run cp ~/talk/zot-start local-zot/zot-start &&
        run ./local-zot/zot-start || return

    run zli config add localhost https://127.0.0.1:5000/
}


zotsync() {
    cd ~/local-zot/ && run ./zot-sync
}

usersetup() {
    run gitclone https://gist.github.com/2e622c67e8a679630d6e70e63b0f32d0.git talk &&
        : || return

    [ -d .ssh ] || run mkdir --mode 700 .ssh || return
    if [ ! -f .ssh/authorized_keys ]; then
        : > .ssh/authorized_keys && chmod 600 .ssh/authorized_keys || return
    fi
    grep -q "$PUBKEY" .ssh/authorized_keys ||
        printf "%s\n" "$PUBKEY" >>.ssh/authorized_keys ||
        return

    ( cd talk &&
      sudo mkdir -p /etc/containers/registries.d &&
      run sudo cp -v registry-default.yaml /etc/containers/registries.d/default.yaml &&
      run sudo cp -v policy.json /etc/containers/
    ) || return

    local d="/usr/share/lxc/templates" tmpl=""
    tmpl="$d/lxc-oci"
    if [ -f "$tmpl" ] && [ ! -f "$tmpl.dist" ]; then
        run sudo cp -v "$tmpl" "$tmpl.dist" || return
    fi

    ( cd talk &&
        run sudo cp lxc-oci "$tmpl" &&
        run sudo chmod 755 "$tmpl" &&
        run sudo cp show lxc-oci-mount /usr/local/bin &&
        run sudo chmod ugo+x /usr/local/bin/* ) ||
        return

    cat >.screenrc <<EOF
hardstatus alwayslastline "%-Lw%{= BW}%50>%n%f* %t%{-}%+Lw%< %= %H"
defscrollback 8096
EOF

    zotsetup || return
    zotsync || return
    lxcsetup || return

    sudo stacker unpriv-setup
}

lxcsetup() {
    local user=${1:-${DEFUSER}}
    mkdir -p .config/lxc
    cat > .config/lxc/default.conf <<EOF
lxc.include = /etc/lxc/default.conf
lxc.idmap = u 0 165536 65536
lxc.idmap = g 0 165536 65536
EOF
    [ $? -eq 0 ] || return

    local lpath="/lxc/$user"
    cat > .config/lxc/lxc.conf <<EOF
lxc.lxcpath = $lpath
EOF
    [ $? -eq 0 ] || return

    run sudo mkdir -p "$lpath" &&
        run sudo chown "$user:$user" "$lpath" ||
        return

    local f=/etc/lxc/lxc-usernet
    if [ -f "$f" ]; then
        sudo touch "$f"
    fi
    if ! grep -q "$user" "$f"; then
        echo "$user veth lxcbr0 32" | run sudo tee "$f"
    fi

    # You need to edit
    # /etc/apparmor.d/abstractions/lxc/start-container
    local aaprof="/etc/apparmor.d/abstractions/lxc/start-container"
    if [ ! -f "$aaprof" ]; then
        echo no "$aaprof file"
        return 1
    fi
    if ! grep -q 'fstype=fuse[*]' "$aaprof"; then
        if [ ! -f "$aaprof.dist" ]; then
            run cp "$aaprof" "$aaprof.dist"
        fi
        cat >>"$aaprof" <<EOF
# add 'mount fstype=fuse*'
# without this here (and with the '*'), we get:
# dmesg errors like fstype="fuse.squashfuse_ll" srcname="squashfuse_ll"
mount fstype=fuse*,
EOF
        [ $? -eq 0 ] || return
    fi

    # FIXME: you also may need to add 'fstype=fuse' to
    # /etc/apparmor.d/lxc/lxc-default-cgns
}


case "$1" in
    startvm|setupvm|doadduser|usersetup)
        n="$1"
        shift
        $n "$@"
        exit;;
    main)
        setupvm && doadduser &&
            sudo -Hu "$DEFUSER" --login -- "$0" usersetup
        ;;
    *) stderr "unknown command $1"; exit 1;;
esac

show

#!/bin/sh
# shellcheck disable=SC2166
Usage() {
cat <<EOF
    ${0##*/} [type] file

show file of type 'type'. if type not given, then try to figure out.
EOF
}

fail(){ echo "$@" 1>&2; exit 1; }
[ $# -eq 1 -o $# -eq 2 ] || { Usage 1>&2; exit 1; }
[ "$1" = "-h" -o "$1" = "--help" ] && { Usage; exit 0; }

[ $# -eq 2 ] && ftype="$1" && shift
file="$1"
if [ ! -f "$file" ]; then
    cand=$(echo "$file" | sed 's,sha256:,sha256/,')
    if [ -f "$cand" ]; then
        file="$cand"
    elif [ -f "oci/blobs/$cand" ]; then
        file="oci/blobs/$cand"
    else
        fail "$file: not a file"
    fi
fi

decomp=""
if [ -z "$ftype" -o "$ftype" = "auto" ]; then
    fout=$(file --uncompress --dereference "$file") ||
        fail "failed to get type of $file"
    case "$fout" in
        *gzip\ compressed*)
            decomp="zcat";;
    esac
    fmt="cat"
    ftype=""
    case "$fout" in
        *JSON*)
            fmt="python3 -m json.tool"
            ftype="json";;
        *tar\ archive*)
            fmt="tar -tvf -"
            ftype="fstar";;
        *shell\ script*)
            ftype="shell";;
        *)
            case "$file" in
                *.yaml) ftype="yaml";;
            esac
    esac
fi

[ -n "$ftype" ] && targ="-l$ftype" || targ="-g"

set -- pygmentize "$targ" -O style=solarized-dark,linenos=1

echo "$file"
if [ -n "$decomp" ]; then
    echo "fmt=$fmt"
    $decomp "$file" | $fmt | "$@"
elif [ "$fmt" = "cat" ]; then
    # if we were just going to cat the file, better auto with filename
    "$@" "$file"
else
    $fmt "$file" | "$@"
fi

stacker.yaml

minbase:
  build_only: true
  from:
    type: docker
    url: ${{DOCKER_BASE:docker://docker.io/library/}}ubuntu:jammy

minboot:
  build_only: true
  from:
    type: built
    tag: minbase
  run: |
    apt-get update -q
    apt-get install --yes --no-install-recommends systemd-sysv

talkrootfs:
  from:
    type: built
    tag: minboot
  run: |
    # password 'none'
    hashpass='$6$cpHPicKJ$O3lnesUxTQYSfuFtllAQZEf4xPjqSQMjV.hOCwATxUBm/oPVSA7zlTZSHffkXIDpIsW7/ec7chroIjo.7sV6i/'
    sed -i 's@^root:[^:]*:\(.*$\)@root:'"$hashpass"':\1@' /etc/shadow

    cat > /etc/rc.local <<"EOF"
    read up idle </proc/uptime
    echo "rc.local was run at $up seconds" > /run/rc.local.out
    EOF

talk.md

%title: Quick starting secure container storage using squashfs, overlay and dm-verity %author: Scott Moser %date: 2023-02-05

-> # Me

• Cisco
• Project-machine https://github.com/project-machine
• [email protected]

---

-> # Intro

• Goal is simple: Replace use of OCI images in tar+gz images with squasfs. ^
• Path there: ^
  • Comparision of types stored in a registry ^
  • Comparision of runtime use. ^
  • Sales Pitch^H^H^H^H^H Demo ^
    • Build --> Stacker ^
    • Sign --> cosign ^
    • Publish --> Zot ^
    • Run --> lxc

---

-> # Registry Comparison ┌─────────────────────────────────────────────────────────────────────┐ │ │ │ Image Registry │ │ │ │ TAR.GZ │ SQUASHFS+VERITY │ │ ────────────────────────────┼─────────────────────────── │ │ - Registry stores OCI Images │ - Registry stores OCI Images │ │ │ │ │ - Images list layers │ - Images list layers │ │ │ │ │ - Layers are mediaType │ - Layers are mediaType │ │ oci.image. │ stacker.image. │ │ + layer.v1.tar+gzip │ + layer.squashfs+zstd+verity │ │ │ │ │ - signed checksum of tarball │ - signed checksum of image │ │ │ │ │ │ - signed dm-verity hash │ │ │ │ └─────────────────────────────────────────────────────────────────────┘

---

-> # Runtime Comparison ┌─────────────────────────────────────────────────────────────────────┐ │ │ │ Container Use / Runtime │ │ │ │ TAR.GZ │ SQUASHFS+VERITY │ │ ────────────────────────────┼─────────────────────────── │ │ - layers uncompress and untar │ - layer (can be) used from oci or │ │ │ copied per-container │ │ │ │ │ - Compare to original tar │ - Compare via sha256sum │ │ file-by-file │ │ │ │ │ │ │ - Privileged mounts get dm-verity │ │ │ │ │ │ - Unpriv mount via squashfuse │ │ │ │ │ │ - FS does not implement write │ │ │ │ │ │ - less easily readable │ └─────────────────────────────────────────────────────────────────────┘

---

-> # Write support

Overlayfs

• View a series of images overlayed into a single view ^
• Support for whiteouts and copy-ups just like oci format. ^
• Overlay bugs still present but largely handled ^
• Changes to the write layer can be easily seen

---

-> # dm-verity

^

• transparent integrity checking of block devices

^

• root hash is signed as part of image metadata

^

• block device read-time validation rather than up front checksum.

^

• Requires root - no unprivileged use of device-mapper.

---

-> # Demo

^

• Warning: Users sensitive to invented-here-software might want to divert their eyes. (It is good software though.) ^

• Build (stacker) - https://github.com/project-stacker/stacker ^

• Sign (cosign) ^

• Publish (zot) - https://github.com/project-zot/zot ^

• Run (lxc)

---

-> # Stacker

• OCI Image build tool ^
• Unprivileged Builds ^
• Can be used to provide a container to build your software in. ^
• 5 years development. ^
• Going through CNCF inclusion process

---

-> # Sign

• Sign the image with cosign. The dm-verity information is present

---

-> # Zot ^

• Single Binary, easy to develop against ^
• Unprivileged ^
• Just like docker.io ^
• CNCF Sandbox project

---

-> # Thanks / Questions

• God, Team, Family, Cisco, You
• https://github.com/project-machine
• Questions
• Scott Moser [email protected]

zot-config.yaml

http:
  # set this to literal 'address: ""' to listen on all interfaces.
  # https://github.com/project-zot/zot/issues/1063
  address: "127.0.0.1"
  port: 5000
  realm: "localhost-zot"
  tls:
    cert: certs/server-cert.pem
    key: certs/server-key.pem
  auth:
    htpasswd:
      path: "htpasswd.txt"
  accessControl:
    # by default, anonymous read. the zot user can read/write
    "**":
      anonymousPolicy: [read]
      defaultPolicy: [read]
      policies:
        - users: [zot]
          actions: [read, create, update, delete, detectManifestCollision]
log:
  level: "debug"
  audit: "logs/audit.log"
  output: "logs/zot.log"
storage:
  dedupe: true
  rootdirectory: "storage"

zot-start

#!/bin/sh
Usage() {
    cat <<EOF
${0##*/} [dir]

run ran server serving dir on port.

EOF
}
fail() { echo "$@" 1>&2; exit 1; }
dir=${2:-$HOME/local-zot}
[ "$1" = "-h" -o "$1" = "--help" ] && { Usage; exit 0; }
dir=$( cd "$dir" && pwd) || fail "failed cd $dir"
cd "$dir" || fail "failed cd $dir"
log_d="$HOME/logs"
log="${log_d}/zot.log"
pidf="$log.pid"

mkdir -p "$log_d" || fail "failed mkdir $log_d"

if [ -f "$pidf" ]; then
    if read p <"$pidf" && [ -n "$p" ] && [ -d "/proc/$p" ]; then
        echo "killing existing pid $p"
        kill $p
        sleep 1
    fi
fi

config="config.yaml"
zot="./bin/zot"
if [ ! -x "$zot" ]; then
    zot=$(command -v "zot") 2>/dev/null || fail "no zot found"
fi
mkdir -p logs storage || fail "failed to make dirs in $PWD"
[ -f "$config" ]  || fail "no config '$config' in $PWD"
[ -x "$zot" ] || fail "no zot bin '$zot' in $PWD"
set -- $zot serve "$config"
echo "$(date -R):" "writing to $log: $*" 1>&2
sh -ec '
    pidf=$1; shift; echo $$ > "$pidf";
    echo "# running in $PWD: $*"
    exec "$@"' -- \
    "$pidf" "$@" </dev/null > "$log" 2>&1 &
newpid=$!
sleep 1
[ -d "/proc/$newpid" ] || { cat "$log" 1>&2; fail "failed to start $*"; }
cat "$log"

zot-sync

#!/bin/bash
# sync some docker images to zot
# for these docker urls, then you can save yourself from
# the docker bandwidth limit by just referencing
#  * docker://localhost:5000/docker-sync/ubuntu:latest
#
# Run this with some frequency, as up to date images are good to have.
DOCKER_URLS=(
 "docker://ubuntu:jammy"
 "docker://busybox:latest"
)

# Sync to a oci local directory to use less bandwidth from docker
# and also to resume more safely.
SYNC_OCI="oci:/tmp/sync-oci.d"
ZOT_BASE="docker://localhost:5000/docker-sync"

fail() { echo "$@" 1>&2; exit 1; }
msg() { echo "$(date -R)" "$@"; }

mkdir -p "${SYNC_OCI#oci:}" ||
    fail "failed to create ${SYNC_OCI} dir for local sync"

for dockurl in "${DOCKER_URLS[@]}"; do
    base=${dockurl##*/}
    ociurl="${SYNC_OCI}:$base"
    msg "sync $dockurl -> $ociurl"
    skopeo copy "$dockurl" "$ociurl" ||
        fail "failed to sync $dockurl -> $ociurl"
done

for dockurl in "${DOCKER_URLS[@]}"; do
    base=${dockurl##*/}
    zoturl="${ZOT_BASE%/}/${base}"
    ociurl="${SYNC_OCI}:$base"
    msg "publish $ociurl -> $zoturl"
    skopeo copy --dest-tls-verify "$ociurl" "$zoturl" ||
        fail "failed to publish $ociurl -> $zoturl"
done

msg "done"