WAL-E vendoring compile

compile

#!/bin/bash
set -ue

cache_key() {
    # Generate an md5sum from one literal input and a series of files.
    #
    # This is to allow for nested content hash derivation, useful when
    # the caller of a function may need to trigger a cache-bust in a
    # subroutine.
    cat <(echo "$1") "${@:2}" | md5sum | cut -f1 -d ' '
}

vendor_wal_e() {
    # Vendor WAL-E.
    #
    # Rebuild it only if necessary, and copy it from the cache
    # otherwise.
    cache_base_hash=$1
    build_dir=$2
    cache_dir=$3/wal-e-vendor-cache
    content_key_path=$cache_dir/cachecontentkey

    if ! [ -d "$cache_dir" ]
    then
	mkdir "$cache_dir"
	echo 'init' > "$content_key_path"
    fi

    export PYTHONPATH=$build_dir/vendor/virtualenv-1.9.1
    python -m virtualenv --quiet --no-site-packages "/app/wal-e-env"

    source_archives=( $(find "$build_dir/vendor/python-sdists" -type f |
	    sort | xargs) )
    new_key=$(cache_key "$cache_base_hash" "$(which python)" \
	"${source_archives[@]}")
    stored_key=$(cat "$cache_dir/cachecontentkey")

    if [ "$new_key" = "$stored_key" ]
    then
	# Cache hit, copy from the cache into the build directory.
	cp -a "$cache_dir/wal-e-env" "$build_dir"
    else
	# Cache miss, do all the compilation work.
	export PATH=/app/wal-e-env/bin:$PATH

	echo 'WAL-E vendor cache miss, re-building, this is a > 60s wait...'
	pip install --no-index --find-links "file://$build_dir/vendor/python-sdists" wal-e

	cp -a /app/wal-e-env "$build_dir"

	# Re-populate the cache with fresh content.
	rm -rf "$cache_dir/wal-e-env"
	cp -a /app/wal-e-env "$cache_dir"
	echo "$new_key" > "$content_key_path"
    fi
}

# Bust the cache when this program text is changed.
self_hash=$(cache_key "" "$0")

vendor_wal_e "$self_hash" "$1" "$2"