Docker tricks

docker.sh

# compact docker ps
docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Status}}\t{{.Names}}"

# *** START: MAINTENANCE ******************************

# remove non-running containers
docker rm $(docker ps -a -q -f status=exited -f status=dead)

# remove untagged images, selects column {2} (zero based) from lines starting with <none>
docker images | grep '^<none>' | sed -nE 's/^(\S+\s+){2}(\S+).*/\2/; p' | xargs docker image rm

# delete dangling volumes
# https://stackoverflow.com/a/65157753/1072212
docker volume ls -qf dangling=true | egrep '^[a-z0-9]{64}$' | \
  xargs --no-run-if-empty docker volume rm

# prune networks
docker network prune
  
# *** END: MAINTENANCE ******************************

# Fix ERROR: readlink /data/docker/overlay2/l/7M5EN7M5OUETAW42LC63NZBNJA: no such file or directory errors
docker image ls | while read row; do
  image_id=$(echo $row | sed -E 's/^(\S+\s+){2}(\S+).*/\2/')
  docker inspect --format='{{.GraphDriver.Data.MergedDir}}' $image_id || \
      (echo -e "Broken image:\n$row"; echo docker image rm $image_id)
done

# find images containing files being reported by security scans etc. as
# /var/lib/docker/overlay2/e94cd857503582e69d7c8554b49f14e54d157f823da7211a18a3d447d867e9a6/merged/usr/.../gems//rack-3.1.12
docker image ls | grep -v '<none>' | sed -En '/REPOSITORY/ n; s/ +/:/; s/ .*//; p' | \
  sort | xargs -IF bash -c "echo; echo F; docker run --rm -t --entrypoint sh F -c \
  'find / -name rexml-3.2* -o -name rack-[123]* 2>/dev/null'"

# This might be a lot quicker / lighter:
docker image inspect $(docker image ls -q) --format json \
  | jq ".[] | {id: .Id, tags: .RepoTags, root_layers: .RootFS.Layers, gd_data: .GraphDriver.Data}" \
  | sed -En '/"id"/,/],/ H; /"id"/ h; /FRAGMENT_OF_HASH_YOURE_LOOKING_FOR/ {x;p;x;p}'

# stop all containers
docker stop $(docker ps -a -q)

# command to get paths for named volumes, with no parameter it lists
# named volumes, with a volume name it lists the host path, so
#   cp setup.dat $(dv myapp_data)/tmp/
# for example would copy setup.dat into the volume 'myapp_data'
dv () {
  if [ "$1" ]; then
    # docker volume inspect $1 | jq -r '.[] | .Mountpoint'
    docker volume inspect $1 | sed -nr '/^\s*"Mountpoint": "/ {s/^\s*"Mountpoint": "//; s/",\s*$//; p}'
  else
    docker volume ls | grep '\s\w\{,60\}$'
    D=$(docker volume ls | grep '\s\w\{64\}$' | wc -l)
    echo "$D anonymous volumes omitted"
    D=$(docker volume ls -q -f dangling=true | wc -l)
    echo "docker volume ls -f dangling=true lists $D volumes"
  fi
}

# list volumes for all containers
docker ps --format '{{.Names}}' \
  | xargs -IF bash -c "echo F; docker inspect -f '{{ .Mounts }}' F | sed 's/}/}\n/g'"

# show runtime settings
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock assaflavie/runlike \
  CONTAINER_NAME | sed 's/ --/\n  --/g'

# misc. ways to get env. var. and other container internal stuff
#!/bin/bash
if [ -f /.dockerenv ]; then
    # use env. vars. to generate bootstrap script
    cat << EOT
if [ ! "\$(ls $CKAN_HOME_VOL)" ]; then
    docker run -v "$CKAN_HOME_VOL":/out ubuntu bash -c 'cp -r /usr/lib/ckan/* /out/'
fi
EOT
else
    # invoke Docker to access env. vars.
    # docker run --env-file .env -v $(realpath $0):/script.sh ubuntu bash /script.sh
    # docker-compose run --entrypoint bash ckan /usr/lib/ckan/venv/src/ckan/ccte_local/build.sh
    CKAN_HOME_VOL=$(docker run --env-file .env ubuntu bash -c 'echo $CKAN_HOME_VOL')
    echo It is $CKAN_HOME_VOL
    if [ ! "\$(ls $CKAN_HOME_VOL)" ]; then
        docker run -v "$CKAN_HOME_VOL":/out ubuntu bash -c 'cp -r /usr/lib/ckan/* /out/'
    fi
fi

docker_data.py

"""
docker_data.py - list *all* host paths used by Docker for storage

Download (wget, curl) from:
https://gist.githubusercontent.com/tbnorth/fbe6b109aba0c90c0b2d18a6cedb4014/raw/docker_data.py

usage:
    sudo python3 docker_data.py [--container] [--color]
    --container: sort by container name instead of volume path
    --color: highlight volume names / container names

Terry N. Brown [email protected] Sun 28 Jun 2020 11:31:40 AM CDT
"""
import argparse
import json
import os
import re
import subprocess


def make_parser():

    parser = argparse.ArgumentParser(
        description="""Report on docker storage""",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )

    parser.add_argument(
        "--container",
        action='store_true',
        help="Sort by container name instead of filesystem path",
    )
    parser.add_argument(
        "--color", action='store_true', help="Color highlighting of output"
    )

    return parser


def get_options(args=None):
    """
    get_options - use argparse to parse args, and return a
    argparse.Namespace, possibly with some changes / expansions /
    validations.

    Client code should call this method with args as per sys.argv[1:],
    rather than calling make_parser() directly.

    Args:
        args ([str]): arguments to parse

    Returns:
        argparse.Namespace: options with modifications / validations
    """
    opt = make_parser().parse_args(args)

    # modifications / validations go here

    return opt


opt = get_options()


def text_from_cmd(cmd):
    if not isinstance(cmd, list):
        cmd = cmd.split()
    cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE)
    out, _ = cmd.communicate()
    return out.decode('utf8')


def short_anon(text):
    return re.sub("([a-z0-9]{7})[a-z0-9]{57}", r"\1*", text)


def volume_sort(volume):
    return (
        volume['Container'] or '' if opt.container else '',
        volume['Type'] == 'bind',
        re.match("[a-z0-9]{64}", volume['Source']) is not None,
        volume['Source'],
    )


containers = text_from_cmd("docker ps -a --format {{.Names}}").split()
running = text_from_cmd("docker ps --format {{.Names}}").split()
dangling = text_from_cmd("docker volume ls -q -f dangling=true").split()
mounts = {}
mounts = {
    container: json.loads(
        text_from_cmd(
            [
                "docker",
                "inspect",
                "--type",
                "container",
                "-f",
                "{{json .Mounts }}",
                container,
            ]
        )
    )
    for container in containers
}
volumes = []
seen = set()
shared = set()
for container, mounted in mounts.items():
    for mount in mounted:
        mount['Container'] = container
        mount['Running'] = container in running
        mount['Dangling'] = False
        mount['Lost'] = False
        try:
            os.stat(mount['Source'])
        except FileNotFoundError:
            mount['Lost'] = True
        except PermissionError:
            pass
        
        if not mount['Lost']:
            mount['Source'] = os.path.realpath(
                os.path.abspath(mount['Source'])
            )
        if mount['Source'] in seen:
            shared.add(mount['Source'])
        seen.add(mount['Source'])
        volumes.append(mount)
for volume in volumes:
    volume['Shared'] = volume['Source'] in shared


for volume in dangling:
    inspect = json.loads(text_from_cmd(f"docker volume inspect {volume}"))[0]
    inspect['Source'] = inspect['Mountpoint']
    assert inspect['Source'] not in seen
    inspect.update(
        dict(
            Container=None,
            Running=False,
            Shared=False,
            Dangling=True,
            Type='volume',
        )
    )
    inspect['Lost'] = False
    try:
        os.stat(inspect['Source'])
    except FileNotFoundError:
        inspect['Lost'] = True
    except PermissionError:
        pass
    volumes.append(inspect)

print("B:bind R:running D:dangling S:shared !:deleted")
for volume in sorted(volumes, key=volume_sort):
    data = dict(
        bind='B' if volume['Type'] == 'bind' else '_',
        running='R' if volume['Running'] else '_',
        dangling='D' if volume['Dangling'] else '_',
        lost='!' if volume['Lost'] else '_',
        shared='S' if volume['Shared'] else '_',
        source=short_anon(volume['Source']),
        name=short_anon(f"as {volume['Name']} ") if volume.get('Name') else '',
        container=f"in {volume['Container']}"
        if volume.get('Container')
        else '',
    )
    if opt.color:
        if data['name']:
            data['name'] = f"as {data['name'].split()[-1]} "
        if data['container']:
            data['container'] = f"in {data['container'].split()[-1]}"
    print(
        "{bind}{running}{dangling}{shared}{lost} "
        "{source} {name}{container}".format_map(data)
    )