condor_run.py

#! /usr/bin/python

import os

PDFs      = '2'          # 3 for LO and NLO, 2 for NNLO
how_long  = '600'         # how long to run, in seconds
base_seed = 400000000    # base seed: 4E8 for MLB

idx       = 0            # the index to start from 
n_to_run  = 2

email = 'mleblanc@cern.ch'
queue = 'long.q'

outputDir = '/home/mlb/nnlo/out/'

for njob in range(idx,n_to_run):
    for njet in ['2','3']:
        for channel in ['1','2','3','4']:
            for contrib in ['DU0', 'DU1', 'FR0', 'FR1', 'SU0', 'SU1', 'RVF', 'RRF']:

                runfile = 'batch/runR32_'+njet+'_'+channel+'_'+contrib+'_'+PDFs+'_'+how_long+'_'+str(base_seed+njob)+'.sh'
                cmd = 'runR32.sh '+njet+' '+channel+' '+contrib+' '+PDFs+' '+how_long+' '+str(base_seed+njob)
                #cmd += '; cp *.xml '+outputDir

                #############################################
                # This is the singularity payload file

                payloadname = runfile.replace("run","payload")
                payload = open(payloadname,"w")                

                payload.write('#!/bin/sh\n')
                payload.write(cmd)
                payload.write('\n')
                payload.close()
                os.system('chmod a+x '+payloadname)

                ##############################################
                # Create the script run by the condor job

                fr=open(runfile,'w')
                fr.write('#!/bin/sh\n')
                fr.write('# '+cmd+' batch run script\n')
                
                fr.write('#!/bin/sh\n')
                fr.write('#$ -cwd\n')
                fr.write('#$ -j y\n')
                fr.write('#$ -l cvmfs\n')
                fr.write('#$ -l distro=sld6\n')
                fr.write('#$ -l arch=amd64\n')
                fr.write('#$ -l h_vmem=1G\n')
                #fr.write('#$ -o '+logfile+'\n')
                fr.write('#$ -q '+queue+'\n')
                fr.write('#$ -m '+'as'+'\n')
                fr.write('#$ -M '+email+'\n')
                fr.write('#$ -N R32 NNLO\n')
                
                fr.write('export TMPDIR=%s\n' % outputDir)
                fr.write("echo Temporary dir is $TMPDIR\n")
                #fr.write('export EOS_MGM_URL=root://eosuser.cern.ch\n')
                fr.write('export ATLAS_LOCAL_ROOT_BASE=/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase\n')
                fr.write('source /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/user/atlasLocalSetup.sh\n')
                
                #fr.write('cd /home/mlb/nnlo/\n')
                #fr.write('source /home/mlb/nnlo/enter_container.sh\n')
                #fr.write('setupATLAS -c llorente/threejet-nnlo:0.5\n')
                fr.write('export X509_USER_PROXY=$HOME/.globus/job_proxy.pem\n')
                fr.write('export DISPLAY=\n\n')
                fr.write('echo Running CMND:\n')
                fr.write('echo '+cmd+'\n\n')
                
                # run the command through the container interface, otherwise process hangs when entering the container
                fr.write('setupATLAS -c llorente/threejet-nnlo:0.5 -r ./'+payloadname[5:]+'\n')
                fr.write('echo DONE R32\n')
                fr.write('ls -lh\n')

                # cmd = 'runR32.sh '+njet+' '+channel+' '+contrib+' '+PDFs+' '+how_long+' '+str(base_seed+njob)
                outfile = "trijet_"+njet+"_"+contrib+"_"+channel+"-*.xml"
                
                fr.write('cp '+outfile+' %s/\n' % ( outputDir))
                fr.write('rm '+outfile+'\n')

                fr.close()
                os.system('chmod a+x '+runfile)

                #################################################
                # Create the condor JDL
                
                # Make a condor .job file for each foutname
                fsubcondorname = runfile.replace(".sh",".job")
                fsubcondor = open(fsubcondorname,"w")
                fsubcondor.write('Universe        = vanilla\n')
                fsubcondor.write('Executable      = '+runfile+'\n')
                #fsubcondor.write('+ProjectName      ="AtlasConnect"\n')
                fsubcondor.write('+ProjectName=\"ATLAS_org_UChicago\" \n')
                fsubcondor.write('Log             = /home/mlb/nnlo/logs/'+runfile.replace('.sh','').split('/')[-1] +'_$(Cluster).$(Process).log\n')
                fsubcondor.write('Output          = /home/mlb/nnlo/logs/'+runfile.replace('.sh','').split('/')[-1] +'_$(Cluster).$(Process).out\n')
                fsubcondor.write('Error           = /home/mlb/nnlo/logs/'+runfile.replace('.sh','').split('/')[-1] +'_$(Cluster).$(Process).err\n')
                fsubcondor.write('JobBatchName    =  R32 NNLO '+str(njob)+'\n')
                #if("Pythia" in infile): fsubcondor.write('RequestMemory = 16G\n')
                #fsubcondor.write('RequestMemory = 16G\n')
                fsubcondor.write('RequestMemory = 1G\n')
                fsubcondor.write('stream_output = TRUE\n')
                fsubcondor.write('stream_error = TRUE\n')
                fsubcondor.write('transfer_input_files = '+payloadname+'\n')
                #fsubcondor.write('x509UserProxy = /tmp/x509up_u43257\n')
                #fsubcondor.write('x509UserProxy = /afs/cern.ch/user/m/mleblanc/private/x509up_u56624\n')
                fsubcondor.write('x509UserProxy = /home/mlb/.globus/job_proxy.pem\n')
                fsubcondor.write('\nqueue 1\n')
                fsubcondor.close()

                subcmd = '/usr/bin/condor_submit '+fsubcondorname
                print(subcmd)
                os.system(subcmd)
                
print("All done!")