Remove orphan layers left by the "file" storage backend of the docker registry, heavily inspired by https://gist.github.com/shepmaster/53939af82a51e3aa0cd6

remove-orphan-images.sh

#!/bin/bash

set -eu
shopt -s nullglob

readonly base_dir=/var/lib/docker/registry
readonly output_dir=$(mktemp -d -t trace-images-XXXX)
readonly jq=/usr/bin/jq

readonly repository_dir=$base_dir/repositories
readonly image_dir=$base_dir/images

readonly all_images=$output_dir/all
readonly used_images=$output_dir/used
readonly unused_images=$output_dir/unused

function info() {
    echo -e "\nArtifacts available in $output_dir"
}
trap info EXIT ERR INT

function image_history() {
    local readonly image_hash=$1
    $jq '.[]' $image_dir/$image_hash/ancestry | tr -d  '"'
}

echo "Collecting orphan images"
for library in $repository_dir/*; do
    echo "Library $(basename $library)" >&2

    for repo in $library/*; do
        echo " Repo $(basename $repo)" >&2

        for tag in $repo/tag_*; do
            echo "  Tag $(basename $tag)" >&2

            tagged_image=$(cat $tag)
            image_history $tagged_image
        done
    done
done | sort | uniq > $used_images

ls $image_dir > $all_images

grep -v -F -f $used_images $all_images > $unused_images

readonly all_image_count=$(wc -l $all_images | awk '{print $1}')
readonly used_image_count=$(wc -l $used_images | awk '{print $1}')
readonly unused_image_count=$(wc -l $unused_images | awk '{print $1}')
readonly unused_image_size=$(cd $image_dir; du -hc $(cat $unused_images) | tail -n1 | cut -f1)

echo "${all_image_count} images, ${used_image_count} used, ${unused_image_count} unused"
echo "Unused images consume ${unused_image_size}"

echo -e "\nTrimming _index_images..."
readonly unused_images_quoted=$output_dir/unused.flatten
cat $unused_images | sed -e 's/\(.*\)/\"\1\" /' > $unused_images_quoted

for library in $repository_dir/*; do
    echo "Library $(basename $library)" >&2

    for repo in $library/*; do
        echo " Repo $(basename $repo)" >&2
        mkdir -p "$output_dir/$(basename $repo)"
        jq '.' "$repo/_index_images" > "$output_dir/$(basename $repo)/_index_images.old"
        jq -s '.[0] - [ .[1:][] | {id: .} ]' "$repo/_index_images" $unused_images_quoted > "$output_dir/$(basename $repo)/_index_images"
        cp "$output_dir/$(basename $repo)/_index_images" "$repo/_index_images"
    done
done

echo -e "\nRemoving images"
cat $unused_images | xargs -I{} rm -rf $image_dir/{}

Why the [1:]? Doesn't this drop the first item in unused_images_flatten?

Ah I see, slurp option takes all objects from inputs and puts them into an array. The second file is a number of string objects, so you can refer to them with .[1:]. Very clever.

Seems to have issues with jq1.5:

jq: error (at /tmp/trace-images-DWtl/unused.flatten:0): Cannot iterate over string ("db4e537798...)
jq: error (at /tmp/trace-images-DWtl/unused.flatten:0): Cannot iterate over string ("dbdb0a509f...)
jq: error (at /tmp/trace-images-DWtl/unused.flatten:0): Cannot iterate over string ("ddda88bcb8...)
jq: error (at /tmp/trace-images-DWtl/unused.flatten:0): Cannot iterate over string ("de1bc6e387...)
jq: error (at /tmp/trace-images-DWtl/unused.flatten:0): Cannot iterate over string ("df266e249a...)
jq: error (at /tmp/trace-images-DWtl/unused.flatten:0): Cannot iterate over string ("e0bda8871e...)

@andrewmichaelsmith indeed, it seems that jq changed its behavior when slurping single line... I updated the script to put one image id per line, a colleague of mine has verified it was working on jq 1.5

@bjaglin does this work with a v2 registry?

@bjaglin Well I'm a little late but this worked perfectly. Thanks!

compatible with Docker Repository v2

https://gist.github.com/qoomon/7c7f16939630cafafceeb83d254194e4