simple background worker process

worker.py

'''
To test, first make a bunch of job files in the same directory this file is in:

    $ for i in `seq 300`; do echo $i > $i.job; done;

Then fire up a bunch of workers:

    $ python worker.py &
    $ python worker.py &
    $ python worker.py &
    $ python worker.py &
    $ python worker.py &

You will see results start to come in.

You can stop the workers by doing this:

    $ pkill -f worker.py
'''

import glob
import shutil
import random
import time

def process(filename):
    with open(filename) as f:
        x = int(f.read().strip())
    time.sleep(1) #simulate expensive calculation
    return x+1

def find_and_process_job():
    jobs = glob.glob("*.job")
    if not jobs:
        return
    jobfile = random.choice(jobs)
    workingfile = jobfile.replace(".job", ".work")
    completedfile = jobfile.replace(".job", ".done")
    try:
        shutil.move(jobfile, workingfile)
    except IOError, e:
        #another worker beat us to grabbing the job, find another
        return
    result = process(workingfile) #do something with the result
    print "finished with %s, result is: %s" % (workingfile, result)
    shutil.move(workingfile, completedfile)

if __name__ == "__main__":
    while True:
        time.sleep(0.05)
        find_and_process_job()