utilization_and_cycle_time.py

import numpy
import random
from matplotlib import pyplot

percentiles = [50, 75, 90, 95, 99]
latencies = [[] for p in percentiles]
loads = []

n = 100000
for k in numpy.linspace(0.01, 1.0, 100):
    print('k =', k)
    query_enqueued = numpy.cumsum(numpy.random.exponential(size=(n,), scale=1./k))
    query_time = numpy.random.lognormal(size=(n,))
    query_finished = numpy.zeros((n, ))
    for i in range(n):
        query_finished[i] = query_enqueued[i] + query_time[i]
        if i > 0:
            # Can't finish before the previous query finished plus the time for this query
            query_finished[i] = max(query_finished[i], query_finished[i-1] + query_time[i])

    for p, l in zip(percentiles, latencies):
        l.append(numpy.percentile(query_finished - query_enqueued, p))
    loads.append(numpy.sum(query_time) / numpy.max(query_enqueued))

for p, l in zip(percentiles, latencies):
    pyplot.semilogy(loads, l, label='%d%% percentile latency' % p)
pyplot.grid()
pyplot.xlim([0, 1])
pyplot.ylim([0, 20])
pyplot.legend(loc='lower right')
pyplot.tight_layout()
pyplot.savefig('utilization_and_cycle_time.png')

Output: