Workaround if a command doesn't support concurrency well...

retry_exec.py

# Run function with delayed retry
#
# Motivation: BUG_WORKAROUND(ninja):
# It seems ninja has some issues with concurrency
# because it always tries to open deps log for write even for
# read-only operations like "ninja -t query"
#
# It's like how Pure ALOHA network deals with collision

### VERSION 1: using a wrapper function

# example:
#     def foo(a, b):
#         ...
#         return returned
#
#     exec_retry(5, lambda returned : returned == 0, foo, 1, 2) # will run foo() at most 5 times

import time, random

def exec_retry(max_times, return_is_ok, func, *params, **kwparams):
    assert(type(max_times) == int and max_times > 0)
    assert(callable(return_is_ok)) # should be a predicate
    assert(callable(func)) # the real function you'd like to run
    for i in range(max_times):
        ret = func(*params, **kwparams)
        if return_is_ok(ret) == True:
            return ret
        time.sleep(0.05 + random.uniform(-0.02, 0.02))
    return ret

### VERSION 2: using decorator, a prettier wrapper function

# example:
#     @with_retry(5, lambda returned : returned == 0)
#     def foo(a, b):
#         ...
#         return returned
#
#     foo(1, 2) # will run foo() at most 5 times

import time, random

class with_retry: # decorator class
    def __init__(self, max_times, return_is_ok):
        assert(type(max_times) == int and max_times > 0)
        assert(callable(return_is_ok))
        self.max_times = max_times       # int, > 0
        self.return_is_ok = return_is_ok # callable, a predicate (i.e. returns boolean)
    def __call__(self, func):
        assert(callable(func))
        def wrapped(*params, **kwparams):
            for i in range(self.max_times):
                ret = func(*params, **kwparams)
                if self.return_is_ok(ret):
                    return ret
                time.sleep(0.05 + random.uniform(-0.02, 0.02))
            return ret           
        return wrapped

range() exists in both Python2 and Python3 (though meanings differ: one returns list and the other iterator), but xrange() only exists in Python2. range() is used to accommodate both. Nasty Python.