Pull 4254 docker images and delete them by python | 并发拉取镜像，记录拉取成功、失败、超时的情况

check_image.py

#!/usr/bin/env python
#-*- coding: utf-8 -*-
import os
import sys
import datetime
import subprocess

import time
import gevent.pool
import gevent.monkey
from gevent import Timeout
gevent.monkey.patch_all()
from multiprocessing import Process

def shell_cmd(cmd, system=False):
    if system:
        return '', os.system(cmd)
    return subprocess.Popen(cmd, shell=not isinstance(cmd, list),
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE).communicate()

def slice_list(ls, n):
    if not isinstance(ls, list) or not isinstance(n, int):
        return []
    ls_len = len(ls)
    if n <= 0 or 0 == ls_len:
        return []
    if n > ls_len:
        return []
    elif n == ls_len:
        return [[i] for i in ls]
    else:
        j = ls_len / n
        k = ls_len % n
        ls_return = []
        for i in xrange(0, (n-1) * j, j):
            ls_return.append(ls[i:i+j])
        ls_return.append(ls[(n-1)*j:])
        return ls_return

def pull_worker(image):
    try:
        with Timeout(600):
            cmd = 'docker pull {0}'.format(image)
            shell_cmd(cmd)
            out, err = shell_cmd('docker images {0}'.format(image))
            if not err:
                out, err = shell_cmd('docker rmi {0}'.format(image))
                if err:
                    print "[ERROR docker rmi] {0} {1}".format(image, err)
                with open('good_image.txt','a') as goodImageFile:
                    goodImageFile.write(image + '\n')
            else:
                with open('error_image.txt','a') as timeout_file:
                    timeout_file.write(image + " {0}".format(err) + '\n')

    except Timeout:
        with open('timeout_image.txt','a') as timeout_file:
            timeout_file.write(image + '\n')

if __name__ == "__main__":
    start = time.time()
    all_task_list = []

    if len(sys.argv) == 4:
        pass
    else:
        print "Need three params:\n# 1 File\n# 2 Process numbers\n# 3 Parallels numbers for each process"
        sys.exit(-1)

    _, file, coreNum, poolNum = sys.argv

    def each_process(task_object_list):
        pool = gevent.pool.Pool(int(poolNum))
        pool.map(pull_worker, task_object_list)
        stop = time.time()
        elapsed = stop - start
        print "End precess with {0} s".format(elapsed)

    with open(file) as f:
        for line in f:
            line = line.strip()
            all_task_list.append(line)

    print "All task: {0}".format(len(all_task_list))
    for sliced_task_list in slice_list(all_task_list, int(coreNum)):
        print "Start process with tasks: {0}".format(len(sliced_task_list))
        p = Process(target=each_process, args=(sliced_task_list,))
        p.start()