Perfect Parallel Planets

parallel_planets.py

import subprocess
import numpy as np

template = 'python ../code/mySpitzerTransit/4_finalPipelineMCMC.py '
template += '{0} {1} {2} {3} {4} {5} {6} > outfile_{7}.txt'

planets = ['Hatp32b',
           'XO1b',
           'Hatp1b',
           'Wasp17b',
           'Wasp39b',
           'Hatp12b',
           'Hatp18b',
           'TrES2b',
           'Wasp4b',
           'XO2b',
           'GJ3470b',
           'Wasp21b',
           'Wasp31b',
           'Wasp1b']

max_process = 5

# Gather files
files = {'input': [], 'paras': [], 'published': []}
for planet in planets:
    files['input'].append("{0}/{0}_fixainc_gp.txt".format(planet))
    files['paras'].append("{0}/pipelineParams_{0}.txt".format(planet))
    files['published'].append("{0}/{0}_publishedData.txt".format(planet))

# Gather input
burnin = ["2000"]*len(planets)
run = ["4000"]*len(planets)
walkers = ["200"]*len(planets)
foldext = ["/circ_median_fixainc_gp"]*len(planets)

# Set input
args = zip(files['input'],
           files['paras'],
           burnin,
           run,
           walkers,
           foldext,
           files['published'],
           planets)

# Run commands in parallel
for i in range(0, len(args), max_process):

    processes = []

    for arg in args[i:i + max_process]:
        command = template.format(*arg)
        process = subprocess.Popen(command, shell=True)
        processes.append(process)

    # Collect statuses
    output = [p.wait() for p in processes]