Dispy benchmark

dispy-main.py

# 'compute' is distributed to each node running 'dispynode';
# runs on each processor in each of the nodes

from compute import compute,search,rgb2gray

def main(n):
    import dispy, random
    import logging
    import tifffile as tiff
    import time
    import numpy as np
    initiala = time.time()
    ips = ['192.168.1.1', '192.168.1.193']
    cluster = dispy.JobCluster(compute,nodes=ips, loglevel=logging.ERROR,secret='put the password',ext_ip_addr='XXX.XX.XXX.XX',
                               ip_addr='192.168.1.193'
    )
    jobs = []
    files = search()
    files = files[0:n]
    print('%d files.'%(len(files)))
    for i in range(len(files)):
        try:
            img = tiff.imread(files[i])
            if img.shape != (3,1024,1024):
                continue
            img = rgb2gray(img)
            job = cluster.submit(img)
            job.id = i
            jobs.append(job)
        except:
#            print('Error: ' + files[i])
            pass
    cluster.wait() # wait for all scheduled jobs to finish
    print('%d jobs submitted.'%(len(jobs)))
    spent = []
    for job in jobs:
#        print(job.exception)
        if job.result is None:
#            print('Job ID %d: Result was None'%job.id)
            continue
        host, res, err = job.result # waits for job to finish and returns results
#        print(host,res,err)
        if err:
            pass
	        # print(err)
        else:
            ds = res[1]
            spent.append(job.end_time-job.start_time)
#            print('%04d by Job ID %04d (%s): %d points. %.1f +/- %.1f pixels'%(i,job.id,host,len(res[0].points),np.mean(ds),np.std(ds)))

        # other fields of 'job' that may be useful:
        # print(job.stdout, job.stderr, job.exception, job.ip_addr, job.start_time, job.end_time)
    cluster.stats()
    cluster.close()
    finala = time.time()
    print('Process time: %.2f sec (%d jobs, %.3f sec/job)'%(finala-initiala, len(spent), np.mean(spent)))
    return (finala-initiala, len(spent), np.mean(spent))

if __name__ == '__main__':
    res = []
#    for n in [10,30]:
    for n in [10,30,100,300,1000,3000,10000]:
        res.append(main(n))
    print res