OSX pkgsrc release procedure

OSX Bulk Builds

This is my log of how to perform a quarterly branch pkgsrc bulk build on my local OSX/SmartOS setup.

Git

On gromit.local

BRANCH=2015Q1
cd /nfs
git clone --depth 1 \
    --branch joyent/release/${BRANCH} \
    https://github.com/joyent/pkgsrc.git \
    pkgsrc-${BRANCH}

mk.conf / pbulk.conf

On gromit.local

cd /nfs/mk
PREV=2014Q4
cp mk-${PREV}.conf mk-${BRANCH}.conf
cp mk-${PREV}-include.conf mk-${BRANCH}-include.conf
vi mk-${BRANCH}*

cd /nfs
cp pbulk-${PREV}.conf pbulk-${BRANCH}.conf
# Q1->Q2, Q4->Q1, etc.
vi pbulk-${BRANCH}.conf

pbulk bootstrap

On OSX. Start a screen session with variables set.

export ABI=32 ARCH=i386    # 32-bit
export ABI=64 ARCH=x86_64  # 64-bit
export BRANCH=2015Q1

screen -S pbulk-${BRANCH}

sudo /net/gromit/nfs/scripts/mksandbox-osx /content/chroot-build-pbulk
sudo chroot /content/chroot-build-pbulk login -fq root env ABI=$ABI ARCH=$ARCH BRANCH=$BRANCH bash
cd /net/gromit/nfs/pkgsrc-${BRANCH}/bootstrap
rm -rf /usr/pbulk
./bootstrap \
    --abi=${ABI} \
    --prefix=/usr/pbulk \
    --pkgdbdir=/usr/pbulk/.pkg \
    --prefer-pkgsrc=yes \
    --gzip-binary-kit=/net/gromit/nfs/bootstrap/Darwin/bootstrap-pbulk-${BRANCH}-${ARCH}.tar.gz \
    --mk-fragment=/net/gromit/nfs/mk/mk-pbulk-include.conf \
    --workdir=/tmp/bootstrap-pkgsrc
rm -rf /tmp/bootstrap-pkgsrc
PATH=/usr/pbulk/sbin:/usr/pbulk/bin:$PATH
cd ..
for dir in pkgtools/mksandbox pkgtools/pbulk pkgtools/pkgdiff security/gnupg2 sysutils/user_darwin
do
    (cd ${dir}; bmake DEPENDS_TARGET=package-install package-install)
done
exit
sudo /net/gromit/nfs/scripts/rmsandbox-osx /content/chroot-build-pbulk

Outside chroot:

sudo rm -rf /usr/pbulk
tar -zxf /net/gromit/nfs/bootstrap/Darwin/bootstrap-pbulk-${BRANCH}-${ARCH}.tar.gz -C /
env PKG_PATH=/net/gromit/nfs/packages/Darwin/pbulk-${BRANCH}/${ARCH}/All /usr/pbulk/sbin/pkg_add \
    gnupg2 mksandbox pbulk pkgdiff

bootstrap

On OSX

sudo /net/gromit/nfs/scripts/mksandbox-osx /content/chroot-bootstrap
sudo chroot /content/chroot-bootstrap login -fq root env ABI=$ABI ARCH=$ARCH BRANCH=$BRANCH bash

cd /net/gromit/nfs/pkgsrc-${BRANCH}/bootstrap
./bootstrap \
    --abi=${ABI} \
    --make-jobs=3 \
    --prefix=/usr/pkg \
    --pkgdbdir=/usr/pkg/.pkgdb \
    --varbase=/var \
    --mk-fragment=/net/gromit/nfs/mk/mk-${BRANCH}-include.conf \
    --workdir=/tmp/bootstrap-pkgsrc
mkdir -p /usr/pkg/etc/gnupg
/usr/pbulk/bin/gpg2 --export DE817B8E >/usr/pkg/etc/gnupg/pkgsrc.gpg
cat >/usr/pkg/etc/pkg_install.conf <<EOF
GPG=/usr/pbulk/bin/gpg2
GPG_SIGN_AS=DE817B8E
GPG_KEYRING_VERIFY=/usr/pkg/etc/gnupg/pkgsrc.gpg
EOF
(cd /; tar -zcf /net/gromit/nfs/bootstrap/Darwin/bootstrap-${BRANCH}-${ARCH}-pbulk.tar.gz ./usr/pkg)
exit

sudo /net/gromit/nfs/scripts/rmsandbox-osx /content/chroot-bootstrap

bulkbuild

Perform an initial bulk build. The remote package directories should not be created until the builds look good and the packages have been signed.

sudo /net/gromit/nfs/scripts/mksandbox-osx /content/chroot-bulkbuild
sudo chroot /content/chroot-bulkbuild login -fq root env BRANCH=$BRANCH bash
postfix start

/usr/pbulk/bin/bulkbuild /net/gromit/nfs/pbulk-${BRANCH}.conf

You'll need to stop postfix before deleting the chroot.

postfix stop
exit
sudo /net/gromit/nfs/scripts/rmsandbox-osx /content/chroot-bulkbuild

Sign Packages

/data/pbulk/scripts/mksandbox-osx /data/chroot/sign-osx-${BRANCH}-${ARCH}
chroot /data/chroot/sign-osx-${BRANCH}-${ARCH} login -fq root env BRANCH=$BRANCH ARCH=$ARCH /bin/bash

tar zxf /nfs/packages/bootstrap-pbulk/bootstrap-${BRANCH}-${ARCH}.tar.gz -C /
PATH=/opt/pkg/sbin:/opt/pkg/bin:$PATH

# Cache gpg-agent password.
touch file
/opt/pbulk/bin/gpg2 --sign file
rm file file.gpg

# For updates, do this on gromit first
cd /nfs/packages/Darwin/${BRANCH}/${ARCH}/All
find . -name "*.tgz" | xargs -n1 basename | xargs file | awk -F: '/gzip compressed/ {print $1}' >tosign

# NOTE: Always perform a test file first before running this, otherwise you nuke packages.
cd /nfs/packages/Darwin/${BRANCH}/${ARCH}/All
for f in $(find . -name "*.tgz"); do
    pkg=$(basename ${f})
    echo ${pkg}
    pkg_admin gpg-sign-package ${pkg} signed-${pkg}
    touch -r ${pkg} signed-${pkg}
    mv signed-${pkg} ${pkg}
done

# If doing updates
cat tosign | while read pkg; do
    echo ${pkg}
    pkg_admin gpg-sign-package ${pkg} signed-${pkg}
    touch -r ${pkg} signed-${pkg}
    mv signed-${pkg} ${pkg}
done

exit

/data/pbulk/scripts/rmsandbox-osx /data/chroot/sign-osx-${BRANCH}-${ARCH}

Release

On mini0.local

sudo /net/gromit/nfs/scripts/mksandbox-osx /content/chroot-release
sudo chroot /content/chroot-release login -fq root env BRANCH=$BRANCH ARCH=$ARCH bash

tar zxf /net/gromit/nfs/bootstrap/Darwin/bootstrap-${BRANCH}-${ARCH}-pbulk.tar.gz -C /

ed /usr/pkg/etc/pkg_install.conf >/dev/null <<EOF
/^GPG=/d
/^GPG_SIGN_AS=/d
\$
a
GPG_KEYRING_PKGVULN=/usr/pkg/share/gnupg/pkgsrc-security.gpg
PKG_PATH=http://pkgsrc.joyent.com/packages/Darwin/${BRANCH}/${ARCH}/All
VERIFIED_INSTALLATION=trusted
.
w
q
EOF

env PKG_PATH=/net/gromit/nfs/packages/Darwin/${BRANCH}/${ARCH}/All \
    /usr/pkg/sbin/pkg_add mozilla-rootcerts pkg_alternatives pkgin pkgsrc-gnupg-keys
printf "#\nhttp://pkgsrc.joyent.com/packages/Darwin/${BRANCH}/${ARCH}/All\n" >>/usr/pkg/etc/pkgin/repositories.conf
/usr/pkg/sbin/mozilla-rootcerts install
# Pull in anything necessary from mk-generic.conf and tidy
vi /usr/pkg/etc/mk.conf
cd /; gnutar zcf /net/gromit/nfs/bootstrap/Darwin/bootstrap-${BRANCH}-${ARCH}.tar.gz ./usr/pkg
/usr/pbulk/bin/gpg2 --armor --sign --detach /net/gromit/nfs/bootstrap/Darwin/bootstrap-${BRANCH}-${ARCH}.tar.gz
rsync -av --progress /net/gromit/nfs/bootstrap/Darwin/bootstrap-${BRANCH}-${ARCH}.tar.gz* \
  remoteurl:.../packages/Darwin/bootstrap/

Fixing Packages

sudo /net/gromit/nfs/scripts/mksandbox-osx /content/chroot-fix
sudo chroot /content/chroot-fix login -fq root env BRANCH=$BRANCH ARCH=$ARCH bash

tar zxf /net/gromit/nfs/bootstrap/Darwin/bootstrap-${BRANCH}-${ARCH}-pbulk.tar.gz -C /
export DEPENDS_TARGET=bin-install
PATH=/usr/pkg/sbin:/usr/pkg/bin:$PATH

Hack, build, sign...

exit
sudo /net/gromit/nfs/scripts/rmsandbox-osx /content/chroot-fix

nosami/osx-pkgsrc-release.md