Docker pruner. Deletes docker's overlay2 leftovers that survive 'docker system prune -af --volumes'

docker-pruner.sh

#!/usr/bin/env bash

set -eEuo pipefail

MARKER_FILE_NAME="${DOCKER_PRUNER_MARKER:-DOCKER-PRUNER-MARKER-FILE}"
DOCKER_PATH="${DOCKER_PATH:-/var/lib/docker/overlay2}"

function _used_dirs() {
    for docker_obj in $(docker ps -aq) $(docker image ls -aq); do
        lowerdir="$(docker inspect "$docker_obj" | jq '.[].GraphDriver.Data.LowerDir' -r)"
        for dir in ${lowerdir//:/ }; do
            dirname "$dir"
        done
        dirname "$(docker inspect "$docker_obj" | jq '.[].GraphDriver.Data.MergedDir' -r)"
    done
}

function used_dirs() {
    _used_dirs | sort
}

function all_dirs() {
    find "$DOCKER_PATH"/ -maxdepth 1 -type d | grep -Ev '^'"$DOCKER_PATH"'/?l?$' | sort
}

function unused_dirs() {
    grep -v -xF -f <(used_dirs) <(all_dirs)
}

function set_marker() {
    touch "$1"/merged/"$MARKER_FILE_NAME" 2> /dev/null || \
        touch "$1"/diff/"$MARKER_FILE_NAME"
}

function _check() {
    for container in $(docker ps -aq); do
        docker exec "$container" ls "/$MARKER_FILE_NAME" && echo "container: $container" || true
    done
    for image in $(docker image ls -aq); do
        docker run --rm -it --entrypoint ls "$image" "/$MARKER_FILE_NAME" && echo "image: $image" || true
    done
}

function check() {
    output="$(_check 2>&1 | grep -Ev /"$MARKER_FILE_NAME'?"': No such file or directory')"
    if [[ -n "$output" ]]; then
        echo "Problem detected !"
        echo "$output"
        exit 1
    fi
}

function usage() {
    echo -e "Usage:
    \t$0 list    -- List all directories that needs to be removed
    \t$0 marker  -- Put a marker on each directory, to check if the marker in found in a running container (detecting an issue with $0)
    \t$0 check   -- Check if a marker is found. Must have run $0 marker first to make sense
    \t$0 clear   -- Remove the directories"
}

if [[ $# == 0 ]]; then
    usage ; exit 0
fi

if [ "$EUID" -ne 0 ]
  then echo "Please run as root"
  exit 1
fi


if [[ "$1" == "list" ]]; then
    unused_dirs
elif [[ "$1" == "marker" ]]; then
    export -f set_marker
    export MARKER_FILE_NAME="$MARKER_FILE_NAME"
    unused_dirs | xargs -I {} bash -c "set_marker {}"
elif [[ "$1" == "check" ]]; then
    check
elif [[ "$1" == "clear" ]]; then
    unused_dirs | xargs -I {} find {} -delete
elif [[ "$1" == "fn" ]]; then
    $2 "$@"
fi

Hi @fayak , after I use this tool to prune the overlay2, when I re-build a new image, I met below issue, do you have meet similar issue before?

#0 building with "default" instance using docker driver

#1 [internal] load .dockerignore
#1 transferring context: 2B done
#1 DONE 0.0s

#2 [internal] load build definition from Dockerfile
#2 transferring dockerfile: 4.15kB done
#2 DONE 0.0s

#3 [internal] load metadata for docker.io/library/ubuntu:22.04
#3 ...

#4 [internal] load metadata for docker.io/pytorch/llvm:9.0.1
#4 DONE 0.4s

#3 [internal] load metadata for docker.io/library/ubuntu:22.04
#3 DONE 0.4s

#5 FROM docker.io/pytorch/llvm:9.0.1@sha256:946fe9e12ad235bc41a008b4a3033e5043008abc67ea600fc8acf45520ff9051
#5 CACHED

#6 [stage-0  1/49] FROM docker.io/library/ubuntu:22.04@sha256:0e5e4a57c2499249aafc3b40fcd541e9a456aab7296681a3994d631587203f97
#6 CACHED

#7 [internal] load build context
#7 ERROR: stat /var/lib/docker/overlay2/y9i4ol147j3ih95mzagm24hps: no such file or directory
------
 > [internal] load build context:
------
ERROR: failed to solve: stat /var/lib/docker/overlay2/y9i4ol147j3ih95mzagm24hps: no such file or directory

Resolved by add one more line docker system prune -af. Thanks

Resolved by add one more line docker system prune -af. Thanks

@fayak
Should be added to the script, or is it possible to exclude build cache?

awesome script :)

Edit: Note that docker system prune will also remove networks, images etc.
As we only want to prune the build cache, use docker builder prune -f instead.

Just add it between line 80 and 81.

Hi @lug-gh do you know how to exclude build cache? I still struggle with the above issue...

Hi @lug-gh do you know how to exclude build cache? I still struggle with the above issue...

Unfortunately not, I had tried a few approaches but discarded them.
You can use the command docker buildx du to display the IDs of the build cache, unfortunately these do not always match the Overlay2 folder names. You probably have to dig a little deeper here.

this script seems to not work for me

./docker-pruner.sh clear
./docker-pruner.sh: line 10: jq: command not found
find: cannot delete ‘/var/lib/docker/overlay2/2f2919cde9e463006e4e970b245008a44124df68a43bfb28feca266de611e4e5/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/2f2919cde9e463006e4e970b245008a44124df68a43bfb28feca266de611e4e5’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/4583e0a79463bb97ae72bca38138c18b6c3bdb781cf709bc2715bb080eaed521/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/4583e0a79463bb97ae72bca38138c18b6c3bdb781cf709bc2715bb080eaed521’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/4d570c33ffc8b23f561ca904d42cdda1f4543a223c9ecc5a8674e673a5eeccae/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/4d570c33ffc8b23f561ca904d42cdda1f4543a223c9ecc5a8674e673a5eeccae’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/6ad22e6f1f2fedba21b2ebad2ae4510715df41ce33d8a02fc883640b62e0e1db/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/6ad22e6f1f2fedba21b2ebad2ae4510715df41ce33d8a02fc883640b62e0e1db’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/751c12fbece9dad3685c718e345c2ae8e75f7e54a0917a2f3ebef75628af3bfa/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/751c12fbece9dad3685c718e345c2ae8e75f7e54a0917a2f3ebef75628af3bfa’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/7b752178086b6576dd0e4a4663f7360c5b2c7aa7d667e07e1cca712b743baeef/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/7b752178086b6576dd0e4a4663f7360c5b2c7aa7d667e07e1cca712b743baeef’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/7e37d168ea897414693595f01f461360af3d6cb3dfe586d13c6a1b8c8ea9dfd4/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/7e37d168ea897414693595f01f461360af3d6cb3dfe586d13c6a1b8c8ea9dfd4’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/883939a7b4eeb09beee6c9a239a03e9a5b2d67b3c183a153f50dd6503cd3c102/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/883939a7b4eeb09beee6c9a239a03e9a5b2d67b3c183a153f50dd6503cd3c102’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/a6df26d9e0d46e764fd33af05a54edbdc0986cc0e0eecff765a2a5aa49043fb7/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/a6df26d9e0d46e764fd33af05a54edbdc0986cc0e0eecff765a2a5aa49043fb7’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/c3e15389ed01ff570ed36f62fcc3f8fc1e1e728a79cd3968cc239bfa6e2368f8/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/c3e15389ed01ff570ed36f62fcc3f8fc1e1e728a79cd3968cc239bfa6e2368f8’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/c84d8696bbc1b26d8dd472ef9097cb671e3da33177cc46f857c72961d493ab22/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/c84d8696bbc1b26d8dd472ef9097cb671e3da33177cc46f857c72961d493ab22’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/d6d1d823115b2b97739e81b464a46390d74ed90784e8bd4f24596b90df5a0d14/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/d6d1d823115b2b97739e81b464a46390d74ed90784e8bd4f24596b90df5a0d14’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/ed2db5caef85ac12ffe19a8aa557c629ffc1c62852a9107e3fe42db187293857/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/ed2db5caef85ac12ffe19a8aa557c629ffc1c62852a9107e3fe42db187293857’: Directory not empty

./docker-pruner.sh marker
./docker-pruner.sh: line 10: jq: command not found
touch: cannot touch '/var/lib/docker/overlay2/2f2919cde9e463006e4e970b245008a44124df68a43bfb28feca266de611e4e5/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/4583e0a79463bb97ae72bca38138c18b6c3bdb781cf709bc2715bb080eaed521/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/4d570c33ffc8b23f561ca904d42cdda1f4543a223c9ecc5a8674e673a5eeccae/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/6ad22e6f1f2fedba21b2ebad2ae4510715df41ce33d8a02fc883640b62e0e1db/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/751c12fbece9dad3685c718e345c2ae8e75f7e54a0917a2f3ebef75628af3bfa/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/7b752178086b6576dd0e4a4663f7360c5b2c7aa7d667e07e1cca712b743baeef/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/7e37d168ea897414693595f01f461360af3d6cb3dfe586d13c6a1b8c8ea9dfd4/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/883939a7b4eeb09beee6c9a239a03e9a5b2d67b3c183a153f50dd6503cd3c102/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/a6df26d9e0d46e764fd33af05a54edbdc0986cc0e0eecff765a2a5aa49043fb7/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/c3e15389ed01ff570ed36f62fcc3f8fc1e1e728a79cd3968cc239bfa6e2368f8/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/c84d8696bbc1b26d8dd472ef9097cb671e3da33177cc46f857c72961d493ab22/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/d6d1d823115b2b97739e81b464a46390d74ed90784e8bd4f24596b90df5a0d14/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/ed2db5caef85ac12ffe19a8aa557c629ffc1c62852a9107e3fe42db187293857/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory

 ./docker-pruner.sh list
./docker-pruner.sh: line 10: jq: command not found
/var/lib/docker/overlay2/2f2919cde9e463006e4e970b245008a44124df68a43bfb28feca266de611e4e5
/var/lib/docker/overlay2/4583e0a79463bb97ae72bca38138c18b6c3bdb781cf709bc2715bb080eaed521
/var/lib/docker/overlay2/4d570c33ffc8b23f561ca904d42cdda1f4543a223c9ecc5a8674e673a5eeccae
/var/lib/docker/overlay2/6ad22e6f1f2fedba21b2ebad2ae4510715df41ce33d8a02fc883640b62e0e1db
/var/lib/docker/overlay2/751c12fbece9dad3685c718e345c2ae8e75f7e54a0917a2f3ebef75628af3bfa
/var/lib/docker/overlay2/7b752178086b6576dd0e4a4663f7360c5b2c7aa7d667e07e1cca712b743baeef
/var/lib/docker/overlay2/7e37d168ea897414693595f01f461360af3d6cb3dfe586d13c6a1b8c8ea9dfd4
/var/lib/docker/overlay2/883939a7b4eeb09beee6c9a239a03e9a5b2d67b3c183a153f50dd6503cd3c102
/var/lib/docker/overlay2/a6df26d9e0d46e764fd33af05a54edbdc0986cc0e0eecff765a2a5aa49043fb7
/var/lib/docker/overlay2/c3e15389ed01ff570ed36f62fcc3f8fc1e1e728a79cd3968cc239bfa6e2368f8
/var/lib/docker/overlay2/c84d8696bbc1b26d8dd472ef9097cb671e3da33177cc46f857c72961d493ab22
/var/lib/docker/overlay2/d6d1d823115b2b97739e81b464a46390d74ed90784e8bd4f24596b90df5a0d14
/var/lib/docker/overlay2/ed2db5caef85ac12ffe19a8aa557c629ffc1c62852a9107e3fe42db187293857

Docker version 28.2.2, build e6534b4

Am i doing something wrong?

./docker-pruner.sh: line 10: jq: command not found

Just install jq on your host