wolfi get file list and apk info

README.md

Random wolfi and tools.

• get-archive-info - get a tar tvf output and the .APKINFO for every file in the archive.

• build-stage - throw a bunch of files and see which build. they do not depend on each other (each only builds with the wolfi repo)

  I used this to help create batches of things when changing lots of files.

• test-installable - its like the c-i test that checks that all packages that were built are installable.

• docker-run-action - it is like the docker run action

build-stage

#!/bin/bash
# shellcheck disable=SC2015,SC2039,SC2166,SC3043
#
# the goal of this script was to ultimately be given a long
# list of packages and to sort out batches of packages that
# depended on another so that they could be submitted.  wolfictl's
# dependency resolution / build-order solving was having problems
# and queing up N batches that could land in order was nice.
#
# Other thing that this does well is give nice per-package logs.
VERBOSITY=0
TEMP_D=""

stderr() { echo "$@" 1>&2; }
fail() { local r=$?;  [ $r -eq 0 ] && r=1; failrc "$r" "$@"; }
failrc() { local r="$1"; shift; [ $# -eq 0 ] || stderr "$@"; exit "$r"; }

Usage() {
    cat <<EOF
Usage: ${0##*/} [ options ] output-dir/ package1 package2 ....

   build packages one by one.

   each package builds with access to wolfi repo only.
   successful builds populate output-dir/packages/
   logs for each build are put in output-dir/logs/<package>.log
   summary log written to output-dir/build.log

   options:
      -h | --help           show usage
      -v | --verbose        increase verbosity
      -j | --jobs N         use N processes
           --skip-test      do not 'melange test'
           --skip-install   do not test install of output.
EOF

}

bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || stderr "$@"; return 1; }
cleanup() {
    [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}

debug() {
    local level="$1"
    shift
    [ "${level}" -gt "${VERBOSITY}" ] && return
    stderr "${@}"
}

do_build_Usage() {
    cat <<EOF
${0##*/} do-build [options] log-dir packages-dir package ...

options:
 --summary  file    write short log to file
EOF
}

sumlog() {
    local f="$SUMLOG"
    if [ -n "$f" ]; then
        echo "$@" >> "$f"
    else
        echo "$@" 1>&2
    fi
}

vrun() {
    local rc=0
    stderr "running:" "$@"
    "$@" || rc=$?
    [ $rc -eq 0 ] && stderr "success" || stderr "fail: $rc"
    stderr
    return $rc
}

dothing() {
    local msg="$1" log="$2" rc=0 rs="SUCCESS" opstart="$SECONDS"
    shift 2
    sumlog "START $msg [${log##*/}]"
    vrun "$@" >>"$log" 2>&1 ||
        { rc=$? ; rs=FAIL; }
    sumlog "END   $msg $rs [$((SECONDS-opstart))s]"
    return $rc
}

build_index() {
    # FIXME: this should probably lock
    local pdir="$1" key="" d=""
    shift
    [ $# -ge 1 ] && key=$1 && shift
    for arch in x86_64 aarch64; do
        [ -d "$pdir/$arch" ] || continue
        ( cd "$pdir/$arch" && set -- *.apk;
            [ -f "$1" ] || {
                stderr "no .apk files in $PWD ($1 not a file)";
                exit 1;
            }
            melange index ${key:+"--signing-key=$key"} "$@"
        ) || { 
            stderr "failed to generate index in $pdir/$arch"
            return 1
        }
    done
    return 0
}

publish() {
    local src="${1%/}" dest="$2" key="$3"
    rsync -a --include="*.apk" --include='*/' \
        --exclude='*' "$src/" "$dest"
    build_index "$dest" ${key:+"$key"} ||
        fail "failed to build index in $dest"
}


# shellcheck disable=SC2115
empty_dir() {
    local d="$1"
    [ "$d" = "/" -o "$d" = "" ] &&
        { stderr "empty_dir '$d' - not doing that"; return 1; }
    [ -d "$d" ] || return 0
    (
      # all gets done in a subprocess do i don't have to change dir back.
      shopt -s nullglob
      cd "$d" || { stderr "could not cd '$d'"; exit 1; }
      set -- * .?*
      [ $# -eq 0 ] && exit 0
      lf=$(mktemp) || exit 1
      trap 'rm -Rf "$lf"' EXIT
      # don't want to spew errors
      rm -Rf "$@" >"$lf" 2>&1 && exit 0
      rrc0=$? crc=0 rrc1=0
      chmod -R u+w "$@" >> "$lf" 2>&1
      crc=$?
      if rm -Rf "$@" >"$lf" 2>&1; then
        stderr "empty_dir($d) required chmod u+w (rrc0=$rrc0 crc=$crc)"
        exit 0
      fi

      rrc1=$?
      stderr "empty_dir($d) failed $rrc1 (rrc0=$rrc0 crc=$crc)"
      tail -n 10 "$lf" 1>&2
      exit $rrc1
    )
}

do_build() {
    local short_opts="hv"
    local long_opts="help,skip-build,skip-test,skip-install,summary:,verbose"
    local getopt_out=""
    getopt_out=$(getopt --name "${0##*/}" \
        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
        eval set -- "${getopt_out}" ||
        { bad_Usage; return; }

    local summary="" dbuild=true dtest=true dinstall=true

    while [ $# -ne 0 ]; do
        { cur="$1"; next="$2"; }
        case "$cur" in
            -h|--help) Usage ; exit 0;;
            --summary) summary="$next"; shift;;
            --skip-build) dbuild=false;;
            --skip-test) dtest=false;;
            --skip-install) dinstall=false;;
            -v|--verbose) VERBOSITY=$((VERBOSITY+1));;
            --) shift; break;;
        esac
        shift;
    done

    local logd="$1" trepo="$2"
    shift 2

    SUMLOG="$summary"
    mkdir -p "$logd" "$trepo" || fail "failed making dirs"

    TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/build-XXXXXX") ||
        fail "failed to make tempdir"
    trap cleanup EXIT

    local lkey=""
    [ -f "local-melange.rsa" ] && lkey="$PWD/local-melange.rsa"

    local merepos="" ierepos="" r="" stages=""
    # a 'stage' here would be the output of another build-stage
    # where we include it's packages for use.
    # shellcheck disable=SC2206
    stages=( ${MY_STAGES} )
    for r in "${stages[@]}"; do
        merepos="$merepos --repository-append=$r/packages"
        ierepos="$ierepos --repo=$r/packages"
    done
    merepos=${merepos# }
    ierepos=${ierepos# }

    local pkg="" pstart="" log="" rs="" rc="" pfail=""
    for pkg in "$@"; do
        pstart=$SECONDS
        pkg=${pkg%.yaml}
        log="$logd/$pkg.log"
        : > "$log" || fail "failed writing to $log"
        echo " ${#stages[@]} stages - ${stages[*]}" >> "$log"
        echo "start $pkg $log"

        pdir="$TEMP_D/$pkg"
        prepo="$pdir/repo"
        tdir="$TEMP_D/$pkg.tmp"
        mkdir -p "$prepo" "$tdir" || fail "failed create prepo dir"

        pfail=""
        if [ $dbuild = true ]; then
            dothing "$pkg build" "$log" \
                env TMPDIR="$tdir" \
                    make "package/$pkg" "REPO=$prepo" \
                        MELANGE_EXTRA_OPTS="--out-dir=$prepo $merepos" ||
                pfail=build
            empty_dir "$tdir"
        fi

        if [ -z "$pfail" ] && [ $dtest = true ]; then
            dothing "$pkg test" "$log" \
                env TMPDIR="$tdir" \
                    make "test/$pkg" "REPO=$prepo" \
                        "MELANGE_EXTRA_OPTS=$merepos" ||
                pfail="test"
            empty_dir "$tdir"
        fi

        if [ -z "$pfail" ] && [ $dinstall = true ]; then
            # shellcheck disable=SC2086
            dothing "$pkg test-install" "$log" \
                test-installable "--repo=$prepo" $ierepos ||
                    pfail="install"
            empty_dir "$tdir"
        fi


        if [ -z "$pfail" ]; then
            ## FIXME: need a local signing key here. just using local-signing
            dothing "$pkg publish" "$log" \
                publish "$prepo" "$trepo" ${lkey:+"$lkey"} ||
                pfail="publish"
        fi

        if [ -n "$pfail" ]; then
            rs="FAIL/$pfail"
            fails="$fails $pkg"
        else
            rs="PASS"
         fi

        rm -Rf "$pdir"
        echo "finish $pkg $rs $log [$((SECONDS-pstart))s]"
        sumlog "RESULT $pkg $rs [$((SECONDS-pstart))s] ${log##*/}"
    done

    if [ -n "$fails" ]; then
        return 1
    fi
    return 0
}

main() {
    local short_opts="hj:v"
    local long_opts="help,jobs:,skip-build,skip-test,skip-install,verbose"
    local getopt_out=""
    getopt_out=$(getopt --name "${0##*/}" \
        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
        eval set -- "${getopt_out}" ||
        { bad_Usage; return; }

    local cur="" next="" pt="" jobs=1
    pt=( )

    while [ $# -ne 0 ]; do
        # shellcheck disable=SC2034
        { cur="$1"; next="$2"; }
        case "$cur" in
            -h|--help) Usage ; exit 0;;
            -j|--jobs) jobs=$next; shift;;
            -v|--verbose) VERBOSITY=$((VERBOSITY+1));;
               --skip-build|--skip-test|--skip-install) pt[${#pt[@]}]="$cur";;
            --) shift; break;;
        esac
        shift;
    done

    mkdir -p ~/.parallel;
    : > ~/.parallel/will-cite

    [ $# -ge 2 ] || {
        bad_Usage "must provide arguments. got $# ($*) expect 2+";
        return;
    }

    local outd="$1" outd_in="$1"
    shift
    mkdir -p "$outd_in" || fail "failed create out-dir '$outd_in'"
    outd=$(cd "$outd_in" && pwd) || fail "failed to get real path to $outd_in"
    mkdir -p "$outd/packages" "$outd/logs" ||
        fail "failed mkdir $outd/packages $outd/logs"

    TEMP_D=$(mktemp -d "$outd/.build-XXXXXX") ||
        fail "failed to make tempdir"
    trap cleanup EXIT

    local workd="$TEMP_D"
    local pkg="" sumlog="$outd/logs/short.log" fails=""
    local failn=0 passn=0
    : > "$sumlog"

    echo "building $# packages in $jobs jobs. output in $outd."
    TMPDIR="$workd" 
        parallel \
        "--jobs=$jobs" --line-buffer -- \
            "$0" do-build "${pt[@]}" "--summary=$sumlog" \
                "$outd/logs" "$outd/packages" \
            ::: "$@"
    rc=$?

    awk '$1 == "RESULT" && $3 ~ "^FAIL" { print $2 }' "$sumlog" > "$outd/logs/fails"
    awk '$1 == "RESULT" && $3 ~ "PASS" { print $2 }' "$sumlog" > "$outd/logs/passes"
    sort "$outd/logs/fails" "$outd/logs/passes" > "$outd/logs/attempts"
    failn=$(wc -l < "$outd/logs/fails")
    passn=$(wc -l < "$outd/logs/passes")
    echo "$((failn+passn)) attempts. $failn fails. $passn pass. took $((SECONDS))s"
    [ "$failn" -eq 0 ]
}

if [ "$1" = "do-build" ]; then
    shift
    do_build "$@"
    exit
elif [ "$1" = "publish" ]; then
    shift
    publish "$@"
    exit
fi
main "$@"
# vi: ts=4 expandtab

build-stages

#!/bin/bash
# 
# needs usage and stuff, but just commit it for now.
#
# Basically this calls 'build-stage' until it doesn't
# accomplish anything. so you will end up with 'stageXX'
# directories each having built more things.
#
# When I did this with the 184 ruby 3.2 files changing to
# ruby 3.3, it took 6 stages.  All packages are built in
# isolation in a stage, only given access to previous
# stages and archive.
set -o pipefail

vrun() {
    local out="$1"
    local start=$SECONDS
    shift
    {
        echo "execute: $*"
        time "$@"
        r=$?
        echo "r=$r [$((SECONDS-start))s]"
    }  2>&1 | tee "$out"
}

njobs=16
n=1
max=20

n=0
to_build="$*"
to_buildn=$#
lto_buildn=0
lstage=""
MY_STAGES=""
while n=$((n+1)) && [ $n -lt $max ]; do
    if [ -n "$lstage" ]; then
        to_build=$(cat "$lstage/logs/fails")
        to_buildn=$(wc -l < "$lstage/logs/fails")
    fi
    if [ "$to_buildn" -eq 0 ]; then
        echo "stage '$lstage' had no failures"
        exit 0
    fi
    if [ "$to_buildn" -eq "$lto_buildn" ]; then
        echo "Quitting, stage $lstage did not make progress ($to_buildn fails)"
        exit 1
    fi
    stage=$(printf "stage%02d" "$n")
    echo "===== stage $stage to_buildn=$to_buildn (lstage=$lstage) ====="
    MY_STAGES="${MY_STAGES} $PWD/$lstage"
    # shellcheck disable=SC2086
    set -- $to_build
    vrun "log-$stage.txt" \
        env MY_STAGES="$MY_STAGES" \
            build-stage "-j$njobs" "$stage" "$@"
    lstage=$stage
    lto_buildn=$to_buildn
    fails=$(wc -l < "$stage/logs/fails")
    passes=$(wc -l < "$stage/logs/passes")
    echo "==== stage $stage attempts=$to_buildn fails=$fails passes=$passes ==="
done
echo "Quitting n=$n"

create-multiple-prs

#!/bin/sh
# shellcheck disable=SC3043,SC2059,SC2015,SC2162
#
# this is here as demonstration of 'gh' usage and cutting up a mega-pr
# into smaller prs.
# I had made https://github.com/wolfi-dev/os/pull/45833 which changed 555
# packages one commit each (using a python program) and then
# needed those split up to get them landed.
#
# To do so:
# git log --no-decorate --no-abbrev-commit --format=oneline \
#     upstream/main..test/tw/ldd-check-the-things > list.txt
# create-multiple-prs < list.txt

fail() { echo "$@" 1>&2; exit 1; }
vr() { echo "$ $*" 1>&2; "$@"; return; }

finish() {
    local branch="$1" num="$2" numpad="" githubname="smoser" out=""
    cat >pr-body.txt <<EOF
This cleans up use of ldd-check, replacing
test/ldd-check with test/tw/ldd-check and dropping
the defaults where applicable.
EOF
    numpad=$(printf "%02d" "$num")
    echo "finish $branch [$num]"
    vr git push $githubname HEAD || fail "failed push $githubname HEAD"
    out=$(vr gh pr create \
        --label="automated pr" \
        --label="smoser/tw-ldd-check" \
        --base=wolfi-dev/os/main --body-file=pr-body.txt \
        --title="tw/ldd-check cleanup batch $numpad" \
        --repo=wolfi-dev/os --base=main \
        "--head=$githubname:$branch" </dev/null) || fail "bad - $branch/$num"
    echo "$branch - $out" | tee -a prs.txt
#gh pr create --label="automated pr" --draft --body-file=pr-body.txt --title="tw/ldd-check cleanup batch 1" --base=main --repo=wolfi-dev/os --head=smoser:tw/ldd-check-cleanup-01
}

cutup() {
    local branchfmt="tw/ldd-check-cleanup-%02d"
    local branchnum=1 batchsize=15
    while read commit _desc; do
        fullnum=$((fullnum+1))
        if [ $((fullnum%batchsize)) -eq 1 ]; then
            branchname=$(printf "$branchfmt" "$(((fullnum/batchsize)+1))")
            vr git checkout -b "$branchname" upstream/main &&
               vr git reset --hard upstream/main ||
               fail "failed branch create for $branchname"
        fi

        vr git cherry-pick "$commit" || fail "cherry-pick $commit failed"

        if [ $((fullnum%batchsize)) -eq 0 ]; then
            finish "$branchname" "$((fullnum/batchsize))" || fail
            branchnum=$((branchnum+1))
            [ $branchnum -gt 40 ] && { echo "skipping out"; exit 0; }
        fi
    done
    if [ "$fullnum" -eq 0 ]; then
        fail "didn't get any input?"
    fi
    if [ $((fullnum%batchsize)) -ne 0 ]; then
        finish "$branchname" $((fullnum/batchsize)) || fail
    fi
}


# run this with input of 'git log upstream/main..HEAD
# git log --no-decorate --no-abbrev-commit --format=oneline upstream/main..HEAD | go-cut.sh
cutup

docker-run-action

#!/bin/bash
# shellcheck disable=SC2015,SC2039,SC2166,SC3043

IMAGE="ghcr.io/wolfi-dev/sdk:latest"
VERBOSITY=0
TEMP_D=""

stderr() { echo "$@" 1>&2; }
fail() { local r=$?;  [ $r -eq 0 ] && r=1; failrc "$r" "$@"; }
failrc() { local r="$1"; shift; [ $# -eq 0 ] || stderr "$@"; exit "$r"; }

Usage() {
    cat <<EOF
Usage: ${0##*/} [ options ] command args

   Taken from .github/actions/docker-run/action.yaml
   Run stuff in a container.

   options:
      -h | --help              help
      -v | --verbose           increase verbosity
      -i | --image  IMAGE      image to run [default=$IMAGE]
      -w | --workdir DIR       default is current working dir
      -e | --entrypoint        entrypoint - default is /bin/bash
      -t | --tty               use a tty - default is to give tty if /dev/stdin is tty
           --no-tty            do not use --tty

           --pt      OPT       pass through to docker run

   Example:

      ${0##*/} --repo=packages packages/*.apk
EOF

}

bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || stderr "$@"; return 1; }
cleanup() {
    [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}

debug() {
    local level="$1"
    shift
    [ "${level}" -gt "${VERBOSITY}" ] && return
    stderr "${@}"
}

main() {
    local short_opts="hi:w:e:tr:v"
    local long_opts="help,entrypoint:,image:,no-tty,no-priv,tty,pt:,volume:,workdir:,verbose"
    local getopt_out=""
    getopt_out=$(getopt --name "${0##*/}" \
        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
        eval set -- "${getopt_out}" ||
        { bad_Usage; return; }

    local cur="" next="" pt="" priv=true
    local tty="" image="$IMAGE" workdir="$PWD" entrypoint="/bin/bash"
    pt=( )

    while [ $# -ne 0 ]; do
        { cur="$1"; next="$2"; }
        case "$cur" in
            -h|--help) Usage ; exit 0;;
            -i|--image) image=$next; shift;;
            -w|--workdir) workdir=$next; shift;;
            -e|--entrypoint) entrypoint=$next; shift;;
            -t|--tty) tty=true;;
               --no-tty) tty=false;;
               --pt) pt[${#pt[@]}]="$next"; shift;;
            -r|--repo) repo="$next"; shift;;
               --volume) volumes[${#volumes[@]}]="--volume=$next"; shift;;
            -v|--verbose) VERBOSITY=$((VERBOSITY+1));;
               --no-priv) priv=false;;
            --) shift; break;;
        esac
        shift;
    done

    local privops="" ttyflag=""
    if [ -z "$tty" ] ; then
        [ -t 0 ] && tty="true" || tty=false
    fi
    [ $tty = "true" ] && ttyflag="--tty"

    local home="${HOME:-/home/user}"

    case "$image" in
        *.*/*) :;;
        *) image="cgr.dev/chainguard-private/$image";;
    esac
    case "$entrypoint" in
        none) entrypoint="";;
    esac

    privops=(
        --privileged
        --security-opt="seccomp=unconfined"
        --security-opt="apparmor:unconfined" \
    )
    [ "$priv" = "true" ] || privops=( )

    docker run \
      "${privops[@]}" \
      --volume="$workdir:$workdir" \
      --volume="$home:$home" \
      "${volumes[@]}" \
      --workdir="$workdir" \
      ${entrypoint:+"--entrypoint=${entrypoint}"} \
      --env "HOME=$home" \
      --interactive ${ttyflag:+"${ttyflag}"} \
      "${pt[@]}" \
      "$image" "$@"
}

main "$@"
# vi: ts=4 expandtab

get-archive-info

#!/bin/bash
# shellcheck disable=SC2015,SC2039,SC2166,SC3043
#
# the goal of this script was to ultimately be given a long
# list of packages and to sort out batches of packages that
# depended on another so that they could be submitted.  wolfictl's
# dependency resolution / build-order solving was having problems
# and queing up N batches that could land in order was nice.
#
# Other thing that this does well is give nice per-package logs.
VERBOSITY=0
TEMP_D=""

WOLFI_ARCHIVE="https://packages.wolfi.dev/os"
WOLFI_ARCHIVE="https://apk.cgr.dev/chainguard"
ENTERPRISE_ARCHIVE="https://apk.cgr.dev/chainguard-private"
EXTRAS_ARCHIVE="https://packages.cgr.dev/extras"
DEF_ARCHIVE="$WOLFI_ARCHIVE"

stderr() { echo "$@" 1>&2; }
fail() { local r=$?;  [ $r -eq 0 ] && r=1; failrc "$r" "$@"; }
failrc() { local r="$1"; shift; [ $# -eq 0 ] || stderr "$@"; exit "$r"; }

Usage() {
    cat <<EOF
Usage: ${0##*/} [ options ] output-dir/ [package1 package2] ....

   Grab information about packages in archive.

   options:
      -h | --help           show usage
      -v | --verbose        increase verbosity
      -j | --jobs N         use N processes - default to 1 per cpu
      -a | --archive A      use archive rather than default (wolfi)
           --arch    ARCH   use arch rather than current arch
EOF
}

bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || stderr "$@"; return 1; }
cleanup() {
    [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}

debug() {
    local level="$1"
    shift
    [ "${level}" -gt "${VERBOSITY}" ] && return
    stderr "${@}"
}

apk_info_Usage() {
    cat <<EOF
${0##*/} apk-info [options] url output-base

EOF
}

# shellcheck disable=SC2120
get_arch() {
    local archin="$1" unamem=""
    [ -n "$archin" ] && { echo "$archin"; return 0; }
    unamem=$(uname -m) || { stderr "failed to get 'uname -m'"; return 1; }
    case "$unamem" in
        x86_64|aarch64) : ;;
        *) stderr "do not know what to do with uname -m output '$unamem'"
           return 1;;
    esac
    echo "$unamem"
}

dl() {
    local url="$1" dest="$2" out="" rc=""
    [ $# -lt 2 ] && { stderr "dl got $# args ($*) needed 2"; return 1; }
    shift 2
    set -- ${HTTP_AUTH:+--user "user:${HTTP_AUTH}"} --fail --location "$@"
    if [ -z "$dest" -o "$dest" = "-" ]; then
        curl --quiet "$@" "$url"
        return
    fi
    out=$(curl "$@" --output "$dest" "$url" 2>&1) &&
        return 0

    rc=$?
    echo "$out" 1>&2
    echo "failed $rc dl $url to $dest" 1>&2
    return $rc
}

apk_info() {
    local short_opts="hv"
    local long_opts="help,verbose"
    local getopt_out=""
    getopt_out=$(getopt --name "${0##*/}" \
        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
        eval set -- "${getopt_out}" ||
        { bad_Usage; return; }

    local cur="" next=""
    local outbase="" url="" keep_apk=false
    while [ $# -ne 0 ]; do
        { cur="$1"; next="$2"; }
        case "$cur" in
            -h|--help) Usage ; exit 0;;
            -v|--verbose) VERBOSITY=$((VERBOSITY+1));;
            --) shift; break;;
        esac
        shift;
    done

    local url="$1" outbase="$2" tmpd="" out=""
    shift 2

    TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/apk-info-XXXXXX") ||
        fail "failed to make tempdir"
    trap cleanup EXIT

    tmpd="${TEMP_D}"
    bn=${url##*/}

    case "$url" in
        http*) dl "$url" "$tmpd/$bn" || return;;
        *) cp "$url" "$tmpd/$bn" || fail "failed copy $url";;
    esac

    tar --warning=none --to-stdout -xf "$tmpd/$bn" --occurrence=1 .PKGINFO > "$tmpd/info" &&
        mv "$tmpd/info" "${outbase}info" ||
            fail "failed reading .PKGINFO"

    tar --warning=none -tvf "$tmpd/$bn" > "$tmpd/list" &&
        mv "$tmpd/list" "${outbase}flist" ||
            fail "$url: could not get list"

    if [ "$keep_apk" = "true" ]; then
        mv "$tmpd/$bn" "${outbase}apk" ||
            fail "$url: failed renaming apk"
    fi

    echo "${outbase##*/} ${SECONDS}s"

    return 0
}

main() {
    local short_opts="a:hj:v"
    local long_opts="arch:,archive:,help,jobs:,verbose"
    local getopt_out=""
    getopt_out=$(getopt --name "${0##*/}" \
        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
        eval set -- "${getopt_out}" ||
        { bad_Usage; return; }

    local cur="" next="" pt="" jobs=0 archives="" arch=""
    pt=( )
    archives=( )

    while [ $# -ne 0 ]; do
        # shellcheck disable=SC2034
        { cur="$1"; next="$2"; }
        case "$cur" in
            -h|--help) Usage ; exit 0;;
            -j|--jobs) jobs=$next; shift;;
            -v|--verbose) VERBOSITY=$((VERBOSITY+1));;
            -a|--archive)
                case "$next" in
                    enterprise) archives[${#archives[@]}]="$ENTERPRISE_ARCHIVE" ;;
                    extras) archives[${#archives[@]}]="$EXTRAS_ARCHIVE";;
                    wolfi) archives[${#archives[@]}]="$WOLFI_ARCHIVE";;
                    *) archives[${#archives[@]}]="$next";;
                esac
                shift;;
               --arch) arch=$next; shift;;
            --) shift; break;;
        esac
        shift;
    done

    mkdir -p ~/.parallel;
    : > ~/.parallel/will-cite

    if [ "$jobs" = "0" ]; then
        local out=""
        [ -f /proc/cpuinfo ] &&
            out=$(grep -c processor /proc/cpuinfo) && jobs=$((out*2))
    fi

    [ $# -ge 1 ] || {
        bad_Usage "must provide arguments. got $# ($*) expect 1+";
        return;
    }

    local outd="$1" outd_in="$1" packages="" archive=""
    shift
    packages=( "$@" )
    mkdir -p "$outd_in" || fail "failed create out-dir '$outd_in'"
    outd=$(cd "$outd_in" && pwd) || fail "failed to get real path to $outd_in"

    if [ "${#archives[@]}" = "0" ]; then
        archives=( "$DEF_ARCHIVE" )
    elif [ "${#archives[@]}" -gt 1 ]; then
        # TODO : more than one archive
        fail "cannot have more than one archive, sorry"
    fi
    archive=${archives[0]}
    archive=${archive%/}

    if [ "$archive" = "$ENTERPRISE_ARCHIVE" ]; then
        if [ -z "$HTTP_AUTH" ]; then
            HTTP_AUTH=$(chainctl auth token --audience=apk.cgr.dev) ||
                { stderr "failed to get token for apk.cgr.dev with chainctl"; return 1; }
        fi
    fi

    if [ -z "$arch" ]; then
        arch=$(get_arch) || fail "failed to get arch"
    fi

    TEMP_D=$(mktemp -d "$outd/.build-XXXXXX") ||
        fail "failed to make tempdir"
    trap cleanup EXIT

    local workd="$TEMP_D"
    local jsonl="$workd/archive.json" fullflat="$workd/archive.flat" flat=""

    dl "$archive/$arch/APKINDEX.tar.gz" "$workd/apkindex.tar.gz" || {
        stderr "failed to get index for $archive/$arch"
        return 1
    }

    apkrane ls --latest --json "$workd/apkindex.tar.gz" > "$jsonl" || {
        stderr "failed apkrane ls --latest --json $archive/$arch/APKINDEX.tar.gz";
        return 1;
    }

    # get space delimited origin, name, version, filename
    local rmold=true
    local jqr='.Origin + " " + .Version + " " + .Name + " " + .Name + "-" + .Version + ".apk"'
    jq -r  "$jqr" "$jsonl" > "$fullflat.raw"
    sort <"$fullflat.raw" > "$fullflat"

    local numorigins="" origins="$workd/origins.list"
    local fullcurflat="$workd/archive-cur.flat"
    ## there will be only one package that has the same name as its origin
    ## so we assume that that package is latest.
    ## that way we can filter binaries with other versions
    awk '$1 == $3 { printf("%s %s\n", $1, $2); }' "$fullflat" | sort > "$origins"
    numorigins=$(wc -l < "$origins")

    # fullcurflat gets only those packages that have the same version as the origin
    grep --fixed-strings --file="$origins" "$fullflat" > "$fullcurflat"

    if [ ${#packages[@]} -eq 0 ]; then
        flat="$fullcurflat"
        rmold=true
    else
        rmold=false
        flat="$workd/packages.flat"
        {
            for p in "${packages[@]}"; do
                awk '$1 == n { print $0 }' "n=$p" "$fullcurflat"
            done
        } > "$flat"
    fi

    local numbins="" pinput="$workd/p-input"
    numbins=$(wc -l < "$flat")
    # remove any .flist/.info files that are not in fullflat
    local archivebins="$workd/archive-bins.list"
    local dirbins="$workd/dir-bins.list"
    local oldbins="$workd/old-bins.list"
    local newbins="$workd/new-bins.list"
    local existbins="$workd/existing-bins.list"

    awk '{ printf("%s-%s\n", $3, $2); }' "$flat" > "$archivebins"
    ( cd "$outd" &&
        find . -maxdepth 1 -name "*.flist" -type f |
        sed -e 's,^[.]/,,' -e 's,.flist$,,' ) > "$dirbins"
    sort "$archivebins" "$archivebins" "$dirbins" | uniq -u > "$oldbins"
    sort "$archivebins" "$dirbins" "$dirbins" | uniq -u > "$newbins"
    sort "$archivebins" "$dirbins" | uniq -d > "$existbins"
    existed=$(wc -l < "$existbins")

    awk '{ printf("%s/%s.apk\n%s/%s.\n", burl, $1, outd, $1) }' \
        burl="$archive/$arch" "outd=$outd" "$newbins" >"$pinput" ||
        fail "failed to read '$newbins' into '$pinput'"

    numnew=$(wc -l <"$newbins")
    numold=$(wc -l <"$oldbins")
    echo "getting info for $numorigins origins, $numbins apks ($existed existing, $numnew new) apks in $jobs jobs (old=$numold)"

    # get output-base url
    env TMPDIR="$workd" ${HTTP_AUTH:+HTTP_AUTH="${HTTP_AUTH}"} \
        parallel \
        "--jobs=$jobs" --line-buffer --max-replace-args=2 -- \
            "$0" apk-info "${pt[@]}" < "$pinput"
    rc=$?

    if [ "$rmold" = "true" ] && [ "$numold" -ge 1 ]; then
        echo "cleaning local data for $numold local binaries not in archive"
        ( cd "$outd" && while read bin ; do rm -f "$bin".*; done ) < "$oldbins"
    fi
    echo "finished $numbins apks ${SECONDS}s. Exiting $rc"
    exit $rc
}

if [ "$1" = "apk-info" ]; then
    shift
    apk_info "$@"
    exit
fi
main "$@"
# vi: ts=4 expandtab

test-installable

#!/bin/sh
# shellcheck disable=SC2015,SC2039,SC2166,SC3043

VERBOSITY=0
TEMP_D=""

stderr() { echo "$@" 1>&2; }
fail() { local r=$?;  [ $r -eq 0 ] && r=1; failrc "$r" "$@"; }
failrc() { local r="$1"; shift; [ $# -eq 0 ] || stderr "$@"; exit "$r"; }

Usage() {
    cat <<EOF
Usage: ${0##*/} [ options ] packages

   Taken partially from wolfi-dev/os/.github/workflows/ci-build.yaml
   "Check that packages can be installed with apk add"

   options:
      -h | --help    help
      -v | --verbose increase verbosity

   Example:

      ${0##*/} --repo=packages packages/*.apk
EOF

}

bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || stderr "$@"; return 1; }
cleanup() {
    [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}

debug() {
    local level="$1"
    shift
    [ "${level}" -gt "${VERBOSITY}" ] && return
    stderr "${@}"
}

inside_Usage() {
    cat <<EOF
Usage: ${0##*/} inside repo [ repo ... ] -- packages
EOF
}

vr() { 
  "$@" && return 0;
  fail "FAILED[$?]: $*";
}

runv() {
    local v="$1"
    shift
    debug "$v" "execute:" "$@"
    "$@"
}

inside() {
    local short_opts="hr:v"
    local long_opts="help,repo:,verbose"
    local getopt_out=""
    getopt_out=$(getopt --name "${0##*/}" \
        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
        eval set -- "${getopt_out}" ||
        { bad_Usage; return; }

    local cur="" next=""
    while [ $# -ne 0 ]; do
        # shellcheck disable=SC2034
        { cur="$1"; next="$2"; }
        case "$cur" in
            -h|--help) Usage ; exit 0;;
            -v|--verbose) VERBOSITY=$((VERBOSITY+1));;
            --) shift; break;;
        esac
        shift;
    done

    for r in /repos/repo*; do
        [ -d "$r" ] || continue
        rflags="$rflags --repository=$r"
    done
    rflags=${rflags# }
    ls /repos/

    if [ $# -eq 0 ]; then
        [ -d /repos/repo0 ] ||
            fail "sorry, no /repos/repo0 and no packages"
        set -- $(find /repos/repo0 -name "*.apk" -type f)
    fi

    eroot="/tmp/emptyroot"
    vr mkdir -p "$eroot/etc/apk"
    vr cp -r "/etc/apk"/* "$eroot/etc/apk"
    : > "$eroot/etc/apk/world"

    vr mkdir -p "$eroot/lib/apk/db"
    vr touch "$eroot/lib/apk/db"/{installed,lock,scripts.tar,triggers}

    vr mkdir -p "$eroot" "$eroot/var/cache/apk"
    vr apk update "--root=$eroot"

    fails=""
    n=$#
    for f in "$@"; do
        echo "== $f =="
        vr tar -Oxf "$f" .PKGINFO
        apk add --allow-untrusted --simulate \
            --root="$eroot" ${rflags} \
            "$f"
        [ $? -eq 0 ] || fails="$fails $f"
        echo
    done
    echo "tested $# apk installs"
    [ -z "$fails" ] && exit 0

    set +f
    set -- $fails
    echo "$# failed install: $fails"
    exit 1
}

fullpath() {
    local p="$1"
    ( cd "$p" && pwd )
}

main() {
    local short_opts="hr:v"
    local long_opts="help,repo:,verbose"
    local getopt_out=""
    getopt_out=$(getopt --name "${0##*/}" \
        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
        eval set -- "${getopt_out}" ||
        { bad_Usage; return; }

    local cur="" next="" repos="" r=""

    while [ $# -ne 0 ]; do
        # shellcheck disable=SC2034
        { cur="$1"; next="$2"; }
        case "$cur" in
            -h|--help) Usage ; exit 0;;
            -r|--repo)
                r=$(fullpath "$next") ||
                    fail "failed to get full path to $next"
                repos="$repos $r"; shift;;
            -v|--verbose) VERBOSITY=$((VERBOSITY+1));;
            --) shift; break;;
        esac
        shift;
    done

    repos=${repos# }
    if [ -z "$repos" ]; then
        [ -d ./packages ] ||
            fail "no --repo given and default (./packages) is not a dir"
        repos="$PWD/packages"
    fi
    debug 1 "repos: $repos"

    set +f
    local volumes="" r="" n=0
    volumes=""
    for r in ${repos}; do
        volumes="$volumes --volume=$r:/repos/repo$n"
        n=$((n+1))
    done
    volumes=${volumes# }

    runv 1 docker-run-action \
        $volumes \
        --image=cgr.dev/chainguard/wolfi-base:latest \
        --entrypoint=/bin/sh -- -s "inside" "$@" <"$0"
}

if [ "$1" = "inside" ]; then
    shift
    inside "$@"
fi

main "$@"
# vi: ts=4 expandtab

usrmerge-data

#!/bin/bash
#
# This was built to grab a list of packages that needed to be updated
# for usrmerge.  It outputs 3 different files, each with
# a list of repo, pkgname, origin, version, B, S, US, LIB
#   where B=number of files in bin/
#   S=number of files in sbin/
#   US=number of files in usr/sbin/
#   LIB=number of files in lib/
#
# As of 2025-04-01 there are no files in wolfi, enterprise or extra
# in bin/, sbin/, or usr/sbin/.
set -o pipefail

repo_info() {
    local name="$1" infod="$2" gitd="$3"
    shift 3
    (
    cd "$infod" || exit 1
    if [ $# -eq 0 ]; then
        set -- *.info
    fi
    for info in "$@"; do
       flist=${info%.info}.flist
       out=$(awk '
           $1 == "origin" { o=$3; };
           $1 == "pkgname" { p=$3; };
           $1 == "pkgver" { v=$3 };
           END { printf("%s\t%s\t%s\n", o, v, p); }' "$info") || { echo "failed"; exit 1; }
       set -- $out
       origin=$1
       pkg=$2
       ver=$3
       if [ ! -f "$gitd/$origin.yaml" ]; then
           echo "[$name] skipping $origin/$pkg [obsolete/not in git]" 1>&2
           continue
       fi
       out=$(awk '
          $6 ~ mus { us=us+1; }
          $6 ~ mb { b=b+1; }
          $6 ~ ms { s=s+1; }
          $6 ~ mlib { lib=lib+1; }
          END { printf("%d %d %d %d\n", b, s, us, lib); }' \
            mus="^usr/sbin/" mb="^bin/" ms="^sbin/" mlib="^lib/" \
            "$flist") || exit 1
       set -- $out
       b=$1
       s=$2
       us=$3
       lib=$4
       [ "$b" = "0" ] && [ "$s" = "0" ] && [ "$us" = "0" ] && [ "$lib" = "0" ] && continue
       printf "%s\t%s\t%s\t%s\t%d\t%d\t%d\t%d\n" \
           "$name" "$origin" "$ver" "$pkg" "$b" "$s" "$us" "$lib"
    done
    )
}

outd=${1:-/tmp}
mkdir -p "$outd" || exit 1
repos="extras enterprise wolfi"
for repo in $repos; do
    case $repo in
        enterprise) upstream="[email protected]:chainguard-dev/enterprise-packages";;
        extras) upstream="[email protected]:chainguard-dev/extra-packages";;
        wolfi) upstream="[email protected]:wolfi-dev/os";;
    esac
    if [ ! -d "$repo.git" ]; then
        git clone -o upstream "$upstream" "$repo.git" || { rm -Rf $repo.git; exit 1; }
    else
        ( cd "$repo.git" && git fetch upstream && git checkout upstream/main ) ||
            { echo "failed update $repo.git"; exit 1; }
    fi
done

for repo in $repos; do
    get-archive-info "--archive=$repo" "$PWD/$repo" ||
        { echo "failed get-archive-info --archive=$repo $PWD/$repo"; exit 1; }
    repo_info "$repo" "$PWD/$repo" "$PWD/$repo.git" > "$outd/$repo-usrmerge.txt" ||
        exit 1
done

sort -k2 $outd/*-usrmerge.txt > $outd/all.txt

wolfi-base

#!/bin/sh
fail() { echo "$@" 1>&2; exit 1; }

wait_for() {
    local name="$1" id="" rc="" n=0
    while n=$((n+1)); do
        id=$(docker ps --quiet "--filter=name=$name")
        rc=$?
        [ $rc -eq 0 ] && [ -n "$id" ] && return 0
        if [ $rc -ne 0 ]; then
            echo "rc=$rc - giving up"
            return 1
        fi
        [ $n -eq 30 ] && {
            echo "giving up on $name. seems not present" 1>&2
            return 1
        }
        sleep .1
    done
}

add_extras() {
    local name="$1"
    wait_for "$name" || fail "$name didn't exist"
    wolfi-setup add-extras "$name"
}

add_enterprise() {
    local name="$1" pubkey="$2" repod="$3" pkname="$4"
    wait_for "$name" || fail "$name didn't exist"
    sleep .1
    docker exec "$name" sh -exc '
        pubkey="$1"
        repod="$2"
        pkname=${3:-${pubkey##*/}}

        cp "$pubkey" "/etc/apk/keys/$pkname"
        echo "$repod" >> /etc/apk/repositories
        apk update
    ' -- "$pubkey" "$repod" "$pkname"
}

case "$1" in
    add_extras|add_enterprise)
        n="$1"
        shift
        $n "$@"
        exit
esac

enterprise=false
[ "$1" = "-E" -o "$1" = "--enterprise" ] &&
    { shift; enterprise=true; }

[ "$1" = "-e" -o "$1" = "--extra" ] &&
    { shift; extra=true; } || extra=false

name=$(petname)
echo "$name" 1>&2

set --
if [ "$enterprise" = "true" ]; then
    osdir="/gcsfuse/chainguard-enterprise-registry-destination/os"
    pubkey="$osdir/chainguard-enterprise.rsa.pub"
    [ -d "$osdir" ] || fail "$osdir not a dir"
    [ -f "$pubkey" ] || fail "no ${pubkey##*/} in $osdir"
    pkgpath="/packages-enterprise"
    set -- "--mount=type=bind,source=$osdir,destination=$pkgpath"
    # the key has to be named the same as the data in APKINDEX
    # so we have to keep 'chainguard-enterprise.rsa.pub'
    "$0" add_enterprise "$name" "$pkgpath/${pubkey##*/}" \
        "$pkgpath" "${pubkey##*/}" &
fi

if [ "$extra" = "true" ]; then
    "$0" add_extras "$name" &
fi

exec docker run --rm -it --name="$name" \
    $DOCKER_RUN_OPTS \
    "$@" \
    cgr.dev/chainguard/wolfi-base:latest