Cleanup and reset docker on Jenkins workers / slaves

cleanup_docker.sh

#!/bin/bash
# This script should be located on each Jenkins slave, and the jenkins user should have permission to run it with sudo

# Attempts to cleanly stop and remove all containers, volumes and images.
docker ps -q | xargs --no-run-if-empty docker stop
docker ps -q -a | xargs --no-run-if-empty docker rm --force --volumes
docker volume ls -q | xargs --no-run-if-empty docker volume rm
docker images -a -q | xargs --no-run-if-empty docker rmi -f

# Stops the docker service, unmounts all docker-related mounts, removes the entire docker directory, and starts docker again.

service docker stop
while [[ -n $(mount | grep -E "^none") ]]; do
  echo "Unmounting none"
  umount none
done
umount /var/lib/docker/aufs
echo "Deleting content of /var/lib/docker"
rm -rf /var/lib/docker/*
service docker start

cleanup_docker_on_all_workers.sh

#!/bin/bash

# This script should be run on the Jenkins master. We set up a job in Jenkins to run this once a week on master.

function jenkins-cli {
  java -jar /var/cache/jenkins/war/WEB-INF/jenkins-cli.jar -s http://localhost:8080 "$@"
}

for slave in slavename1 slavename2 slavename3; do
  if [[ `curl -s "http://localhost:8080/computer/${slave}/api/xml?xpath=*/offline/text()"` = "true" ]]; then
    echo "$slave is already offline. Skipping cleanup"
  else
    echo "Cleaning up docker on $slave"
    echo "Taking $slave offline"
    jenkins-cli offline-node $slave -m "Scheduled docker cleanup is running" && \
      echo "Waiting on $slave to go offline" && \
      jenkins-cli wait-node-offline $slave && \
      while [[ `curl -s "http://localhost:8080/computer/${slave}/api/xml?xpath=*/idle/text()"` != "true" ]]; do echo "Waiting on $slave to be idle" && sleep 5; done && \
      echo "Running cleanup_docker on $slave" && \
      ssh -o "StrictHostKeyChecking no" $slave -i /var/lib/jenkins/.ssh/id_rsa "sudo /usr/local/bin/cleanup_docker"
    echo "Bringing $slave back online"
    jenkins-cli online-node $slave
  fi