Dispy template

dispy_simplest.py

# 'compute' is distributed to each node running 'dispynode';
# runs on each processor in each of the nodes

from compute_simplest import compute,search

def main(n):
    import dispy
    import logging
    import time
    import numpy as np
    initiala = time.time()
    # Multiple computers.
    ips = ['192.168.1.193', '192.168.1.1']
    cluster = dispy.JobCluster(compute,nodes=ips, loglevel=logging.ERROR,secret='password here',ext_ip_addr='XXX.XXX.XXX.XXX',
                               ip_addr='192.168.1.193'
    )
    jobs = []
    files = search()
    files = files[0:n]
    print('%d files.'%(len(files)))
    for i in range(len(files)):
        try:
            job = cluster.submit(i)
            job.id = i
            jobs.append(job)
        except:
            print('Error: ' + files[i])
    cluster.wait() # wait for all scheduled jobs to finish
    print('%d jobs submitted.'%(len(jobs)))
    spent = []
    for job in jobs:
        host, res = job.result # waits for job to finish and returns results
        spent.append(job.end_time-job.start_time)
    cluster.stats()
    cluster.close()
    finala = time.time()
    print('Process time: %.2f sec (%d jobs, %.3f sec/job)'%(finala-initiala, len(spent), np.mean(spent)))
    return (finala-initiala, len(spent), np.mean(spent))

if __name__ == '__main__':
    res = []
#    for n in [10,30]:
    for n in [10,30,100,300,1000]:
        res.append(main(n))
    print res