Docker pruner. Deletes docker's overlay2 leftovers that survive 'docker system prune -af --volumes'

docker-pruner.sh

#!/usr/bin/env bash

set -eEuo pipefail

MARKER_FILE_NAME="${DOCKER_PRUNER_MARKER:-DOCKER-PRUNER-MARKER-FILE}"
DOCKER_PATH="${DOCKER_PATH:-/var/lib/docker/overlay2}"

function _used_dirs() {
    for docker_obj in $(docker ps -aq) $(docker image ls -aq); do
        lowerdir="$(docker inspect "$docker_obj" | jq '.[].GraphDriver.Data.LowerDir' -r)"
        for dir in ${lowerdir//:/ }; do
            dirname "$dir"
        done
        dirname "$(docker inspect "$docker_obj" | jq '.[].GraphDriver.Data.MergedDir' -r)"
    done
}

function used_dirs() {
    _used_dirs | sort
}

function all_dirs() {
    find "$DOCKER_PATH"/ -maxdepth 1 -type d | grep -Ev '^'"$DOCKER_PATH"'/?l?$' | sort
}

function unused_dirs() {
    grep -v -xF -f <(used_dirs) <(all_dirs)
}

function set_marker() {
    touch "$1"/merged/"$MARKER_FILE_NAME" 2> /dev/null || \
        touch "$1"/diff/"$MARKER_FILE_NAME"
}

function _check() {
    for container in $(docker ps -aq); do
        docker exec "$container" ls "/$MARKER_FILE_NAME" && echo "container: $container" || true
    done
    for image in $(docker image ls -aq); do
        docker run --rm -it --entrypoint ls "$image" "/$MARKER_FILE_NAME" && echo "image: $image" || true
    done
}

function check() {
    output="$(_check 2>&1 | grep -Ev /"$MARKER_FILE_NAME'?"': No such file or directory')"
    if [[ -n "$output" ]]; then
        echo "Problem detected !"
        echo "$output"
        exit 1
    fi
}

function usage() {
    echo -e "Usage:
    \t$0 list    -- List all directories that needs to be removed
    \t$0 marker  -- Put a marker on each directory, to check if the marker in found in a running container (detecting an issue with $0)
    \t$0 check   -- Check if a marker is found. Must have run $0 marker first to make sense
    \t$0 clear   -- Remove the directories"
}

if [[ $# == 0 ]]; then
    usage ; exit 0
fi

if [ "$EUID" -ne 0 ]
  then echo "Please run as root"
  exit 1
fi


if [[ "$1" == "list" ]]; then
    unused_dirs
elif [[ "$1" == "marker" ]]; then
    export -f set_marker
    export MARKER_FILE_NAME="$MARKER_FILE_NAME"
    unused_dirs | xargs -I {} bash -c "set_marker {}"
elif [[ "$1" == "check" ]]; then
    check
elif [[ "$1" == "clear" ]]; then
    unused_dirs | xargs -I {} find {} -delete
elif [[ "$1" == "fn" ]]; then
    $2 "$@"
fi

Hi @lug-gh do you know how to exclude build cache? I still struggle with the above issue...

Unfortunately not, I had tried a few approaches but discarded them.
You can use the command docker buildx du to display the IDs of the build cache, unfortunately these do not always match the Overlay2 folder names. You probably have to dig a little deeper here.

this script seems to not work for me

./docker-pruner.sh clear
./docker-pruner.sh: line 10: jq: command not found
find: cannot delete ‘/var/lib/docker/overlay2/2f2919cde9e463006e4e970b245008a44124df68a43bfb28feca266de611e4e5/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/2f2919cde9e463006e4e970b245008a44124df68a43bfb28feca266de611e4e5’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/4583e0a79463bb97ae72bca38138c18b6c3bdb781cf709bc2715bb080eaed521/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/4583e0a79463bb97ae72bca38138c18b6c3bdb781cf709bc2715bb080eaed521’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/4d570c33ffc8b23f561ca904d42cdda1f4543a223c9ecc5a8674e673a5eeccae/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/4d570c33ffc8b23f561ca904d42cdda1f4543a223c9ecc5a8674e673a5eeccae’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/6ad22e6f1f2fedba21b2ebad2ae4510715df41ce33d8a02fc883640b62e0e1db/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/6ad22e6f1f2fedba21b2ebad2ae4510715df41ce33d8a02fc883640b62e0e1db’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/751c12fbece9dad3685c718e345c2ae8e75f7e54a0917a2f3ebef75628af3bfa/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/751c12fbece9dad3685c718e345c2ae8e75f7e54a0917a2f3ebef75628af3bfa’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/7b752178086b6576dd0e4a4663f7360c5b2c7aa7d667e07e1cca712b743baeef/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/7b752178086b6576dd0e4a4663f7360c5b2c7aa7d667e07e1cca712b743baeef’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/7e37d168ea897414693595f01f461360af3d6cb3dfe586d13c6a1b8c8ea9dfd4/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/7e37d168ea897414693595f01f461360af3d6cb3dfe586d13c6a1b8c8ea9dfd4’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/883939a7b4eeb09beee6c9a239a03e9a5b2d67b3c183a153f50dd6503cd3c102/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/883939a7b4eeb09beee6c9a239a03e9a5b2d67b3c183a153f50dd6503cd3c102’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/a6df26d9e0d46e764fd33af05a54edbdc0986cc0e0eecff765a2a5aa49043fb7/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/a6df26d9e0d46e764fd33af05a54edbdc0986cc0e0eecff765a2a5aa49043fb7’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/c3e15389ed01ff570ed36f62fcc3f8fc1e1e728a79cd3968cc239bfa6e2368f8/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/c3e15389ed01ff570ed36f62fcc3f8fc1e1e728a79cd3968cc239bfa6e2368f8’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/c84d8696bbc1b26d8dd472ef9097cb671e3da33177cc46f857c72961d493ab22/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/c84d8696bbc1b26d8dd472ef9097cb671e3da33177cc46f857c72961d493ab22’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/d6d1d823115b2b97739e81b464a46390d74ed90784e8bd4f24596b90df5a0d14/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/d6d1d823115b2b97739e81b464a46390d74ed90784e8bd4f24596b90df5a0d14’: Directory not empty
find: cannot delete ‘/var/lib/docker/overlay2/ed2db5caef85ac12ffe19a8aa557c629ffc1c62852a9107e3fe42db187293857/merged’: Device or resource busy
find: cannot delete ‘/var/lib/docker/overlay2/ed2db5caef85ac12ffe19a8aa557c629ffc1c62852a9107e3fe42db187293857’: Directory not empty

./docker-pruner.sh marker
./docker-pruner.sh: line 10: jq: command not found
touch: cannot touch '/var/lib/docker/overlay2/2f2919cde9e463006e4e970b245008a44124df68a43bfb28feca266de611e4e5/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/4583e0a79463bb97ae72bca38138c18b6c3bdb781cf709bc2715bb080eaed521/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/4d570c33ffc8b23f561ca904d42cdda1f4543a223c9ecc5a8674e673a5eeccae/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/6ad22e6f1f2fedba21b2ebad2ae4510715df41ce33d8a02fc883640b62e0e1db/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/751c12fbece9dad3685c718e345c2ae8e75f7e54a0917a2f3ebef75628af3bfa/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/7b752178086b6576dd0e4a4663f7360c5b2c7aa7d667e07e1cca712b743baeef/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/7e37d168ea897414693595f01f461360af3d6cb3dfe586d13c6a1b8c8ea9dfd4/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/883939a7b4eeb09beee6c9a239a03e9a5b2d67b3c183a153f50dd6503cd3c102/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/a6df26d9e0d46e764fd33af05a54edbdc0986cc0e0eecff765a2a5aa49043fb7/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/c3e15389ed01ff570ed36f62fcc3f8fc1e1e728a79cd3968cc239bfa6e2368f8/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/c84d8696bbc1b26d8dd472ef9097cb671e3da33177cc46f857c72961d493ab22/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/d6d1d823115b2b97739e81b464a46390d74ed90784e8bd4f24596b90df5a0d14/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory
touch: cannot touch '/var/lib/docker/overlay2/ed2db5caef85ac12ffe19a8aa557c629ffc1c62852a9107e3fe42db187293857/diff/DOCKER-PRUNER-MARKER-FILE': No such file or directory

 ./docker-pruner.sh list
./docker-pruner.sh: line 10: jq: command not found
/var/lib/docker/overlay2/2f2919cde9e463006e4e970b245008a44124df68a43bfb28feca266de611e4e5
/var/lib/docker/overlay2/4583e0a79463bb97ae72bca38138c18b6c3bdb781cf709bc2715bb080eaed521
/var/lib/docker/overlay2/4d570c33ffc8b23f561ca904d42cdda1f4543a223c9ecc5a8674e673a5eeccae
/var/lib/docker/overlay2/6ad22e6f1f2fedba21b2ebad2ae4510715df41ce33d8a02fc883640b62e0e1db
/var/lib/docker/overlay2/751c12fbece9dad3685c718e345c2ae8e75f7e54a0917a2f3ebef75628af3bfa
/var/lib/docker/overlay2/7b752178086b6576dd0e4a4663f7360c5b2c7aa7d667e07e1cca712b743baeef
/var/lib/docker/overlay2/7e37d168ea897414693595f01f461360af3d6cb3dfe586d13c6a1b8c8ea9dfd4
/var/lib/docker/overlay2/883939a7b4eeb09beee6c9a239a03e9a5b2d67b3c183a153f50dd6503cd3c102
/var/lib/docker/overlay2/a6df26d9e0d46e764fd33af05a54edbdc0986cc0e0eecff765a2a5aa49043fb7
/var/lib/docker/overlay2/c3e15389ed01ff570ed36f62fcc3f8fc1e1e728a79cd3968cc239bfa6e2368f8
/var/lib/docker/overlay2/c84d8696bbc1b26d8dd472ef9097cb671e3da33177cc46f857c72961d493ab22
/var/lib/docker/overlay2/d6d1d823115b2b97739e81b464a46390d74ed90784e8bd4f24596b90df5a0d14
/var/lib/docker/overlay2/ed2db5caef85ac12ffe19a8aa557c629ffc1c62852a9107e3fe42db187293857

Docker version 28.2.2, build e6534b4

Am i doing something wrong?

./docker-pruner.sh: line 10: jq: command not found

Just install jq on your host