Skip to content

Instantly share code, notes, and snippets.

@brianv0

brianv0/L1Proc-Singularity.xml

Created October 9, 2019 12:04
Show Gist options
  • Save brianv0/a089756fbf15c337896dc00aad7b8de9 to your computer and use it in GitHub Desktop.
Save brianv0/a089756fbf15c337896dc00aad7b8de9 to your computer and use it in GitHub Desktop.
Download ZIP
L1Proc Singularity+centos7
Raw
L1Proc-Singularity.xml
<?xml version="1.0" encoding="UTF-8"?>
<pipeline
xmlns="http://glast-ground.slac.stanford.edu/pipeline"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://glast-ground.slac.stanford.edu/pipeline https://glast-ground.slac.stanford.edu/Pipeline-II/schemas/2.1/pipeline.xsd">
<task name="L1Proc-Singularity" version="5.7" type="Data">
<variables>
<var name="baseVersion">0</var>
<var name="creator">L1Proc-5.7</var>
<var name="deliveriesToIgnore">""</var> <!-- colon-separated list of deliveries to skip when merging -->
<var name="isocMode">ISOC_PROD</var>
<var name="isocRun">/afs/slac/g/glast/isoc/flightOps/rhel6_gcc44/${isocMode}/bin/isoc run</var>
<var name="L1ProcROOT">/afs/slac.stanford.edu/g/glast/ground/PipelineConfig/Level1/5.7</var>
<var name="L1_TASK_NAME">L1Proc</var>
<var name="L1_TASK_VERSION">5.7</var>
<var name="L1_INSTALL_DIR">/afs/slac.stanford.edu/g/glast/ground/PipelineConfig/Level1</var>
<var name="L1_BUILD_DIR">/afs/slac/g/glast/ground/releases/volume11/L1Proc</var>
<var name="L1TrendVersion">Prompt</var> <!-- override if reprocessing -->
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib</var>
<var name="logRoot">/nfs/farm/g/glast/u41/L1/logs/${pipeline.mode}</var>
<var name="nameManglingPrefix">L1</var>
<var name="highPriority">75</var>
<var name="midPriority">60</var>
<var name="standardPriority">50</var>
<var name="reconPriority">74</var>
<var name="frdPriority">76</var>
<var name="standardSelect">centos7</var>
<var name="standardRusage">scratch=1</var>
<var name="bigRusage">scratch=250</var>
</variables>
<prerequisites>
<prerequisite name="DOWNLINK_RAWDIR" type="string"/>
<prerequisite name="DOWNLINK_ID" type="string"/>
</prerequisites>
<process name="findRunDirs" autoRetryMaxAttempts="1">
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/findRunDirs.py" batchOptions=" -q glastdataq -sp ${frdPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<createsSubtasks>
<subtask>doRun</subtask>
<subtask>doLci</subtask>
</createsSubtasks>
</process>
<process name="kludgeAsp" autoRetryMaxAttempts="1">
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/kludgeAsp.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="doRun.registerExtendedFT1" status="SUCCESS"/>
<after process="doRun.registerFT2" status="SUCCESS"/>
</depends>
</process>
<process name="cleanupDl" autoRetryMaxAttempts="1">
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/deleteDirs.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="doLci.checkRunLci" status="SUCCESS"/>
<after process="doRun.checkRun" status="SUCCESS"/>
</depends>
</process>
<task name="doRun" version="5.7" type="Data">
<variables>
<var name="dataCatDir">/Data/Flight/Level1/${DATASOURCE}</var>
</variables>
<prerequisites>
<prerequisite name="DATASOURCE" type="string"/>
<prerequisite name="RUNID" type="string"/>
<prerequisite name="RUNSTATUS" type="string"/>
</prerequisites>
<process name="findChunks" autoRetryMaxAttempts="1">
<variables>
<var name="l1LockAction">LockAndThrottle</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 --add-env=flightops python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -E &quot;${isocRun} ${L1ProcROOT}/lockFile.py&quot; -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<createsSubtasks>
<subtask>doChunk</subtask>
</createsSubtasks>
</process>
<process name="setRunning" autoRetryMaxAttempts="1">
<variables>
<var name="l1RunStatus">Running</var>
</variables>
<script><![CDATA[
from java.util import HashMap
def setIfUnset(container, key, value):
oldValue = container.get(key)
if oldValue is None or oldValue == 'Unset':
print 'Setting %s to %s.' % (key, value)
container[key] = value
else: print 'Current value of %s is %s, not changing it.' % (key, oldValue)
return
print 'Setting L1RunStatus for run', runNumber, ' to ', l1RunStatus
runQuality.setL1RunStatus(runNumber, l1RunStatus)
reviewStatus = "Unset"
print 'Setting ReviewStatus for run', runNumber, ' to ', reviewStatus
runQuality.setReviewStatus(runNumber, reviewStatus)
newQuality = 'Good'
quality = runQuality.getRunQuality(runNumber)
print 'Current quality for run %s is %s.' % (runNumber, quality)
if quality is None or quality == 'Unset':
print 'Setting quality for run %s to %s.' % (runNumber, newQuality)
runQuality.setRunQuality(runNumber, newQuality)
quality = newQuality
else: print 'Not changing run quality.'
pipeline.setVariable('runQuality', quality)
fields = ['ACDQuality', 'CALQuality', 'GPSQuality', 'TRACKERQuality',
'TRIGGERQuality']
metaData = runQuality.getMetaData(runNumber)
print 'old metaData:', metaData
for key in fields: setIfUnset(metaData, key, newQuality)
print 'new metadata:', metaData
runQuality.setOrUpdateMetaData(runNumber, metaData)
]]>
</script>
<depends>
<after process="findChunks" status="SUCCESS"/>
</depends>
</process>
<process name="copyM7Hp" autoRetryMaxAttempts="1">
<variables>
<var name="L1_magic7Hp_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MAGIC7HP")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/copyM7.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="findChunks" status="SUCCESS"/>
</depends>
</process>
<process name="registerM7Hp" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">copyM7Hp</var>
<var name="fileType">magic7Hp</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="copyM7Hp" status="SUCCESS"/>
</depends>
</process>
<process name="makeM7L1" autoRetryMaxAttempts="1">
<variables>
<var name="L1_magic7L1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MAGIC7L1")}</var>
<var name="tStart">${pipeline.getCurrentStream().getProcessInstance("findChunks").getVariable("tStart")}</var>
<var name="tStop">${pipeline.getCurrentStream().getProcessInstance("findChunks").getVariable("tStop")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/makeM7.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="findChunks" status="SUCCESS"/>
</depends>
</process>
<process name="registerM7L1" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">makeM7L1</var>
<var name="fileType">magic7L1</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="makeM7L1" status="SUCCESS"/>
</depends>
</process>
<process name="drawOrbit" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">magic7L1</var>
<var name="L1_magic7L1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MAGIC7L1")}</var>
<var name="outFileTypes">orbitPlot</var>
<var name="L1_orbitPlot_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ORBITPLOT")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="registerM7L1" status="SUCCESS"/>
</depends>
</process>
<process name="registerOrbitPlot" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">drawOrbit</var>
<var name="fileType">orbitPlot</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="drawOrbit" status="SUCCESS"/>
</depends>
</process>
<process name="scanDigi" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">digitization</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.digitization" status="DONE"/>
</depends>
</process>
<process name="mergeDigi" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanDigi").getVariable("goodPis")}</var>
<var name="outFileTypes">digi</var>
<var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${bigRusage}]&quot; -app centos6 " />
<depends>
<after process="scanDigi" status="SUCCESS"/>
</depends>
</process>
<process name="registerDigi" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeDigi</var>
<var name="fileType">digi</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeDigi" status="SUCCESS"/>
</depends>
</process>
<process name="scanDigiHist" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">digiHist</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.digiHist" status="DONE"/>
</depends>
</process>
<process name="mergeDigiHist" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanDigiHist").getVariable("goodPis")}</var>
<var name="outFileTypes">digiHist</var>
<var name="L1_digiHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHIST")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanDigiHist" status="SUCCESS"/>
</depends>
</process>
<process name="registerDigiHist" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeDigiHist</var>
<var name="fileType">digiHist</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeDigiHist" status="SUCCESS"/>
</depends>
</process>
<process name="digiHistAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digiHist</var>
<var name="outFileTypes">digiHistAlarm</var>
<var name="L1_digiHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHIST")}</var>
<var name="L1_digiHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerDigiHist" status="SUCCESS"/>
</depends>
</process>
<process name="registerDigiHistAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">digiHistAlarm</var>
<var name="fileType">digiHistAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="digiHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestDigiHistRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">digiHistAlarm</var>
<var name="L1_digiHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerDigiHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="digiHistLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digiHistAlarm</var>
<var name="L1_digiHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerDigiHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="verify" autoRetryMaxAttempts="1">
<variables>
<var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var>
<var name="L1_verifyLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYLOG")}</var>
<var name="L1_verifyHisto_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYHISTO")}</var>
<var name="completeness">InProgress</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/verify.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerDigi" status="SUCCESS"/>
</depends>
</process>
<process name="registerVerifyHisto" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">verify</var>
<var name="fileType">verifyHisto</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="verify" status="SUCCESS"/>
</depends>
</process>
<process name="registerVerifyLog" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">verify</var>
<var name="fileType">verifyLog</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="verify" status="SUCCESS"/>
</depends>
</process>
<process name="ingestVerifyErrorRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">verifyLog</var>
<var name="L1_verifyLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYLOG")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyLog" status="SUCCESS"/>
</depends>
</process>
<process name="verifyErrorAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">verifyLog</var>
<var name="outFileTypes">verifyErrorAlarm</var>
<var name="L1_verifyLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYLOG")}</var>
<var name="L1_verifyErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyLog" status="SUCCESS"/>
</depends>
</process>
<process name="registerVerifyErrorAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">verifyErrorAlarm</var>
<var name="fileType">verifyErrorAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="verifyErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="verifyErrorLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">verifyErrorAlarm</var>
<var name="L1_verifyErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestVerifyErrorAlarmRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">verifyErrorAlarm</var>
<var name="L1_verifyErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="acdPlots" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digiHist</var>
<var name="outFileTypes">acdPlots</var>
<var name="L1_digiHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHIST")}</var>
<var name="L1_acdPlots_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ACDPLOTS")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerDigiHist" status="SUCCESS"/>
</depends>
</process>
<process name="registerAcdPlots" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">acdPlots</var>
<var name="fileType">acdPlots</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="acdPlots" status="SUCCESS"/>
</depends>
</process>
<process name="scanDigiTrend" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">digiTrend</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.digiTrend" status="DONE"/>
</depends>
</process>
<process name="mergeDigiTrend" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanDigiTrend").getVariable("goodPis")}</var>
<var name="outFileTypes">digiTrend</var>
<var name="L1_digiTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITREND")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanDigiTrend" status="SUCCESS"/>
</depends>
</process>
<process name="registerDigiTrend" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeDigiTrend</var>
<var name="fileType">digiTrend</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeDigiTrend" status="SUCCESS"/>
</depends>
</process>
<process name="ingestDigiTrend" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">digiTrend</var>
<var name="L1_digiTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITREND")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ingestTrending.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerDigiTrend" status="SUCCESS"/>
</depends>
</process>
<process name="digiTrendAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digiTrend</var>
<var name="outFileTypes">digiTrendAlarm</var>
<var name="L1_digiTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITREND")}</var>
<var name="L1_digiTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerDigiTrend" status="SUCCESS"/>
</depends>
</process>
<process name="registerDigiTrendAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">digiTrendAlarm</var>
<var name="fileType">digiTrendAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="digiTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="digiTrendLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digiTrendAlarm</var>
<var name="L1_digiTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerDigiTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestDigiTrendRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">digiTrendAlarm</var>
<var name="L1_digiTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerDigiTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="calHist" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digi</var>
<var name="outFileTypes">calHist</var>
<var name="L1_calHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHIST")}</var>
<var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${bigRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerDigi" status="SUCCESS"/>
</depends>
</process>
<process name="registerCalHist" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">calHist</var>
<var name="fileType">calHist</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="calHist" status="SUCCESS"/>
</depends>
</process>
<process name="calHistAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">calHist</var>
<var name="outFileTypes">calHistAlarm</var>
<var name="L1_calHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHIST")}</var>
<var name="L1_calHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerCalHist" status="SUCCESS"/>
</depends>
</process>
<process name="registerCalHistAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">calHistAlarm</var>
<var name="fileType">calHistAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="calHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="calHistLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">calHistAlarm</var>
<var name="L1_calHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerCalHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestCalHistRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">calHistAlarm</var>
<var name="L1_calHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerCalHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="scanCalTrend" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">calTrend</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.calTrend" status="DONE"/>
</depends>
</process>
<process name="mergeCalTrend" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanCalTrend").getVariable("goodPis")}</var>
<var name="outFileTypes">calTrend</var>
<var name="L1_calTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALTREND")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanCalTrend" status="SUCCESS"/>
</depends>
</process>
<process name="registerCalTrend" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeCalTrend</var>
<var name="fileType">calTrend</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeCalTrend" status="SUCCESS"/>
</depends>
</process>
<process name="ingestCalTrend" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">calTrend</var>
<var name="L1_calTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALTREND")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ingestTrending.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerCalTrend" status="SUCCESS"/>
</depends>
</process>
<process name="acdPedsAnalyzer" autoRetryMaxAttempts="1">
<variables>
<var name="fileType">calHist</var>
<var name="reportType">acdPedsAnalyzer</var>
<var name="L1_calHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHIST")}</var>
<var name="L1_acdPedsAnalyzer_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ACDPEDSANALYZER")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/gainsPedsAnalyzer.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerCalHist" status="SUCCESS"/>
</depends>
</process>
<process name="registerAcdPedsAnalyzer" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">acdPedsAnalyzer</var>
<var name="fileType">acdPedsAnalyzer</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="acdPedsAnalyzer" status="SUCCESS"/>
</depends>
</process>
<process name="acdPedsAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">acdPedsAnalyzer</var>
<var name="outFileTypes">acdPedsAlarm</var>
<var name="L1_acdPedsAnalyzer_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ACDPEDSANALYZER")}</var>
<var name="L1_acdPedsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ACDPEDSALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerAcdPedsAnalyzer" status="SUCCESS"/>
</depends>
</process>
<process name="registerAcdPedsAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">acdPedsAlarm</var>
<var name="fileType">acdPedsAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="acdPedsAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="acdPedsLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">acdPedsAlarm</var>
<var name="L1_acdPedsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ACDPEDSALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerAcdPedsAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestAcdPedsRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">acdPedsAlarm</var>
<var name="L1_acdPedsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ACDPEDSALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerAcdPedsAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="calGainsAnalyzer" autoRetryMaxAttempts="1">
<variables>
<var name="fileType">calHist</var>
<var name="reportType">calGainsAnalyzer</var>
<var name="L1_calHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHIST")}</var>
<var name="L1_calGainsAnalyzer_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALGAINSANALYZER")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/gainsPedsAnalyzer.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerCalHist" status="SUCCESS"/>
</depends>
</process>
<process name="registerCalGainsAnalyzer" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">calGainsAnalyzer</var>
<var name="fileType">calGainsAnalyzer</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="calGainsAnalyzer" status="SUCCESS"/>
</depends>
</process>
<process name="calGainsAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">calGainsAnalyzer</var>
<var name="outFileTypes">calGainsAlarm</var>
<var name="L1_calGainsAnalyzer_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALGAINSANALYZER")}</var>
<var name="L1_calGainsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALGAINSALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerCalGainsAnalyzer" status="SUCCESS"/>
</depends>
</process>
<process name="registerCalGainsAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">calGainsAlarm</var>
<var name="fileType">calGainsAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="calGainsAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="calGainsLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">calGainsAlarm</var>
<var name="L1_calGainsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALGAINSALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerCalGainsAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestCalGainsRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">calGainsAlarm</var>
<var name="L1_calGainsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALGAINSALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerCalGainsAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="calPedsAnalyzer" autoRetryMaxAttempts="1">
<variables>
<var name="fileType">calHist</var>
<var name="reportType">calPedsAnalyzer</var>
<var name="L1_calHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHIST")}</var>
<var name="L1_calPedsAnalyzer_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALPEDSANALYZER")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/gainsPedsAnalyzer.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerCalHist" status="SUCCESS"/>
</depends>
</process>
<process name="registerCalPedsAnalyzer" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">calPedsAnalyzer</var>
<var name="fileType">calPedsAnalyzer</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="calPedsAnalyzer" status="SUCCESS"/>
</depends>
</process>
<process name="calPedsAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">calPedsAnalyzer</var>
<var name="outFileTypes">calPedsAlarm</var>
<var name="L1_calPedsAnalyzer_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALPEDSANALYZER")}</var>
<var name="L1_calPedsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALPEDSALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerCalPedsAnalyzer" status="SUCCESS"/>
</depends>
</process>
<process name="registerCalPedsAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">calPedsAlarm</var>
<var name="fileType">calPedsAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="calPedsAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="calPedsLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">calPedsAlarm</var>
<var name="L1_calPedsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALPEDSALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerCalPedsAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestCalPedsRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">calPedsAlarm</var>
<var name="L1_calPedsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALPEDSALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerCalPedsAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="scanFastMonTuple" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">fastMonTuple</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.fastMonTuple" status="DONE"/>
</depends>
</process>
<process name="mergeFastMonTuple" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonTuple").getVariable("goodPis")}</var>
<var name="outFileTypes">fastMonTuple</var>
<var name="L1_fastMonTuple_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTUPLE")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanFastMonTuple" status="SUCCESS"/>
</depends>
</process>
<process name="registerFastMonTuple" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeFastMonTuple</var>
<var name="fileType">fastMonTuple</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeFastMonTuple" status="SUCCESS"/>
</depends>
</process>
<process name="mergeFastMonError" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonTuple").getVariable("goodPis")}</var>
<var name="outFileTypes">fastMonError</var>
<var name="L1_fastMonError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERROR")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanFastMonTuple" status="SUCCESS"/>
</depends>
</process>
<process name="registerFastMonError" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeFastMonError</var>
<var name="fileType">fastMonError</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeFastMonError" status="SUCCESS"/>
</depends>
</process>
<process name="ingestFastMonErrorRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">fastMonError</var>
<var name="L1_fastMonError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERROR")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerFastMonError" status="SUCCESS"/>
</depends>
</process>
<process name="fastMonErrorAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">fastMonError</var>
<var name="outFileTypes">fastMonErrorAlarm</var>
<var name="L1_fastMonError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERROR")}</var>
<var name="L1_fastMonErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerFastMonError" status="SUCCESS"/>
</depends>
</process>
<process name="registerFastMonErrorAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">fastMonErrorAlarm</var>
<var name="fileType">fastMonErrorAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="fastMonErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="fastMonErrorLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">fastMonErrorAlarm</var>
<var name="L1_fastMonErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerFastMonErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestFastMonErrorAlarmRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">fastMonErrorAlarm</var>
<var name="L1_fastMonErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerFastMonErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="scanFastMonHist" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">fastMonHist</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.fastMonHist" status="DONE"/>
</depends>
</process>
<process name="mergeFastMonHist" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonHist").getVariable("goodPis")}</var>
<var name="outFileTypes">fastMonHist</var>
<var name="L1_fastMonHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONHIST")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanFastMonHist" status="SUCCESS"/>
</depends>
</process>
<process name="registerFastMonHist" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeFastMonHist</var>
<var name="fileType">fastMonHist</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeFastMonHist" status="SUCCESS"/>
</depends>
</process>
<process name="fastMonHistAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">fastMonHist</var>
<var name="outFileTypes">fastMonHistAlarm</var>
<var name="L1_fastMonHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONHIST")}</var>
<var name="L1_fastMonHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerFastMonHist" status="SUCCESS"/>
</depends>
</process>
<process name="registerFastMonHistAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">fastMonHistAlarm</var>
<var name="fileType">fastMonHistAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="fastMonHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="fastMonHistLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">fastMonHistAlarm</var>
<var name="L1_fastMonHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerFastMonHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestFastMonHistRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">fastMonHistAlarm</var>
<var name="L1_fastMonHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerFastMonHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="scanFastMonTrend" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">fastMonTrend</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.fastMonTrend" status="DONE"/>
</depends>
</process>
<process name="mergeFastMonTrend" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonTrend").getVariable("goodPis")}</var>
<var name="outFileTypes">fastMonTrend</var>
<var name="L1_fastMonTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTREND")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanFastMonTrend" status="SUCCESS"/>
</depends>
</process>
<process name="registerFastMonTrend" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeFastMonTrend</var>
<var name="fileType">fastMonTrend</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeFastMonTrend" status="SUCCESS"/>
</depends>
</process>
<process name="ingestFastMonTrend" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">fastMonTrend</var>
<var name="L1_fastMonTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTREND")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ingestTrending.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerFastMonTrend" status="SUCCESS"/>
</depends>
</process>
<process name="fastMonTrendAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">fastMonTrend</var>
<var name="outFileTypes">fastMonTrendAlarm</var>
<var name="L1_fastMonTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTREND")}</var>
<var name="L1_fastMonTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerFastMonTrend" status="SUCCESS"/>
</depends>
</process>
<process name="registerFastMonTrendAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">fastMonTrendAlarm</var>
<var name="fileType">fastMonTrendAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="fastMonTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestFastMonTrendRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">fastMonTrendAlarm</var>
<var name="L1_fastMonTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerFastMonTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="fastMonTrendLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">fastMonTrendAlarm</var>
<var name="L1_fastMonTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerFastMonTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="scanGcrChunks" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">mergeGcrCrumbs</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.mergeGcrCrumbs" status="DONE"/>
</depends>
</process>
<process name="mergeGcrChunks" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanGcrChunks").getVariable("goodPis")}</var>
<var name="outFileTypes">gcr</var>
<var name="L1_gcr_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "GCR")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanGcrChunks" status="SUCCESS"/>
</depends>
</process>
<process name="registerGcr" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeGcrChunks</var>
<var name="fileType">gcr</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeGcrChunks" status="SUCCESS"/>
</depends>
</process>
<process name="scanMeritChunks" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">mergeMeritCrumbs</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.mergeMeritCrumbs" status="DONE"/>
</depends>
</process>
<process name="mergeMeritChunks" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanMeritChunks").getVariable("goodPis")}</var>
<var name="outFileTypes">merit</var>
<var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[scratch=2]&quot; -app centos6 " />
<depends>
<after process="scanMeritChunks" status="SUCCESS"/>
</depends>
</process>
<process name="registerMerit" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeMeritChunks</var>
<var name="fileType">merit</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeMeritChunks" status="SUCCESS"/>
</depends>
</process>
<!--
<process name="filterMerit" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">merit</var>
<var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var>
<var name="outFileTypes">filteredMerit</var>
<var name="L1_filteredMerit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FILTEREDMERIT")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[scratch=2]&quot; -app centos6 " />
<depends>
<after process="registerMerit" status="SUCCESS"/>
</depends>
</process>
<process name="registerFilteredMerit" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">filterMerit</var>
<var name="fileType">filteredMerit</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="filterMerit" status="SUCCESS"/>
</depends>
</process>
-->
<process name="electronMerit" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">merit</var>
<var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var>
<var name="outFileTypes">electronMerit</var>
<var name="L1_electronMerit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ELECTRONMERIT")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[scratch=2]&quot; -app centos6 " />
<depends>
<after process="registerMerit" status="SUCCESS"/>
</depends>
</process>
<process name="registerElectronMerit" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">electronMerit</var>
<var name="fileType">electronMerit</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="electronMerit" status="SUCCESS"/>
</depends>
</process>
<process name="verifyMerit" autoRetryMaxAttempts="1">
<variables>
<var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var>
<var name="L1_verifyMeritError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYMERITERROR")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/verifyMerit.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerMerit" status="SUCCESS"/>
</depends>
</process>
<process name="registerVerifyMeritError" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">verifyMerit</var>
<var name="fileType">verifyMeritError</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="verifyMerit" status="SUCCESS"/>
</depends>
</process>
<process name="ingestVerifyMeritErrorRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">verifyMeritError</var>
<var name="L1_verifyMeritError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYMERITERROR")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyMeritError" status="SUCCESS"/>
</depends>
</process>
<process name="verifyMeritErrorAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">verifyMeritError</var>
<var name="outFileTypes">verifyMeritErrorAlarm</var>
<var name="alarmFileType">verifyMeritErrorAlarm</var>
<var name="L1_verifyMeritError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYMERITERROR")}</var>
<var name="L1_verifyMeritErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYMERITERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyMeritError" status="SUCCESS"/>
</depends>
</process>
<process name="registerVerifyMeritErrorAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">verifyMeritErrorAlarm</var>
<var name="fileType">verifyMeritErrorAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="verifyMeritErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="verifyMeritErrorLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">verifyMeritErrorAlarm</var>
<var name="L1_verifyMeritErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYMERITERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyMeritErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestVerifyMeritErrorAlarmRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">verifyMeritErrorAlarm</var>
<var name="L1_verifyMeritErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYMERITERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyMeritErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="makeGaps" autoRetryMaxAttempts="1">
<variables>
<var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var>
<var name="L1_digiGap_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIGAP")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/makeGaps.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[scratch=5]&quot; -app centos6 " />
<depends>
<after process="registerDigi" status="SUCCESS"/>
</depends>
</process>
<process name="registerGaps" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">makeGaps</var>
<var name="fileType">digiGap</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="makeGaps" status="SUCCESS"/>
</depends>
</process>
<process name="ft2Runs" autoRetryMaxAttempts="1">
<variables>
<var name="outFileType">ft2SecondsNoQual</var>
<var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var>
<var name="L1_ft2SecondsNoQual_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDSNOQUAL")}</var>
<var name="L1_digiGap_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIGAP")}</var>
<var name="L1_magic7L1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MAGIC7L1")}</var>
<var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var>
<var name="runQuality">${pipeline.getCurrentStream.getProcessInstance("setRunning").getVariable("runQuality")}</var>
<var name="mootAlias">${pipeline.getCurrentStream().getProcessInstance("findChunks").getVariable("mootAlias")}</var>
<!-- <var name="L1_magic7L1_filename">${datacatalog.getDataset(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MAGIC7L1").getLocation("SLAC_XROOT").getPath()}</var> -->
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ft2Entries.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[scratch=15]&quot; -app centos6 " />
<depends>
<after process="registerGaps" status="SUCCESS"/>
<after process="registerM7L1" status="SUCCESS"/>
<after process="registerMerit" status="SUCCESS"/>
<after process="setRunning" status="SUCCESS"/>
</depends>
</process>
<process name="registerFt2SecondsNoQual" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">ft2Runs</var>
<var name="fileType">ft2SecondsNoQual</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="ft2Runs" status="SUCCESS"/>
</depends>
</process>
<!--
<process name="ingestFT2" autoRetryMaxAttempts="1">
<variables>
<var name="L1_ft2Txt_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2TXT")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ft2Db.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="registerFt2Txt" status="SUCCESS"/>
</depends>
</process>
-->
<process name="makeFT2" autoRetryMaxAttempts="1">
<variables>
<var name="inFileType">ft2SecondsNoQual</var>
<var name="outFileType">ft2NoQual</var>
<var name="L1_ft2SecondsNoQual_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDSNOQUAL")}</var>
<var name="L1_ft2NoQual_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2NOQUAL")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/makeFT2File.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<!-- <after process="ingestFT2" status="SUCCESS"/> -->
<after process="registerFt2SecondsNoQual" status="SUCCESS"/>
</depends>
</process>
<process name="registerFT2NoQual" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">makeFT2</var>
<var name="fileType">ft2NoQual</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="makeFT2" status="SUCCESS"/>
</depends>
</process>
<process name="setupQuality">
<script><![CDATA[
creator = pipeline.getTaskVersionPath()
try:
theRun = runNumber
except NameError:
theRun = int(RUNID[2:])
pass
theSet = runQuality.getMostRecentSubmittedTimeIntervalSet(theRun, creator)
if theSet:
intervals = theSet.getActiveIntervals()
print intervals
lines = []
for interval in intervals:
start = interval.getRequestedStartPeriod()
stop = interval.getRequestedEndPeriod()
qual = interval.getDataQual()
print start, stop, qual
lines.append("%s:%s:%s" % (start, stop, qual))
continue
rangeStr = ",".join(lines)
else:
rangeStr = "x"
pass
print "rangeStr:", rangeStr
pipeline.setVariable("badRanges", rangeStr)
#print "Failing on purpose for debugging."
#raise SystemExit
]]>
</script>
<depends>
<after process="findChunks" status="SUCCESS"/>
</depends>
</process>
<process name="flagFT2" autoRetryMaxAttempts="1">
<variables>
<var name="btiProcess">setupQuality</var>
<var name="badRanges">${pipeline.getCurrentStream.getProcessInstance(btiProcess).getVariable("badRanges")}</var>
<var name="L1_ft2_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2")}</var>
<var name="L1_ft2NoQual_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2NOQUAL")}</var>
<var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var>
<var name="L1_ft2SecondsNoQual_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDSNOQUAL")}</var>
<var name="inFileTypes">ft2NoQual/ft2SecondsNoQual</var>
<var name="outFileTypes">ft2/ft2Seconds</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerFT2NoQual" status="SUCCESS"/>
<after process="setupQuality" status="SUCCESS"/>
</depends>
</process>
<process name="registerFT2" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">flagFT2</var>
<var name="fileType">ft2</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="flagFT2" status="SUCCESS"/>
</depends>
</process>
<process name="exportFT2" autoRetryMaxAttempts="1">
<variables>
<var name="fileType">ft2</var>
<var name="L1_ft2_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/exportStuff.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="registerFT2" status="SUCCESS"/>
</depends>
</process>
<process name="registerFt2Seconds" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">flagFT2</var>
<var name="fileType">ft2Seconds</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="flagFT2" status="SUCCESS"/>
</depends>
</process>
<process name="exportFT2Seconds" autoRetryMaxAttempts="1">
<variables>
<var name="fileType">ft2Seconds</var>
<var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/exportStuff.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="registerFt2Seconds" status="SUCCESS"/>
</depends>
</process>
<process name="wrapupQuality">
<variables>
<var name="btiProcess">flagFT2</var>
</variables>
<script><![CDATA[
from java.sql import Timestamp
try:
theRun = runNumber
except NameError:
theRun = int(RUNID[2:])
pass
creator = pipeline.getTaskVersionPath()
theSet = runQuality.getMostRecentSubmittedTimeIntervalSet(theRun, creator)
ivs = []
if theSet:
intervals = theSet.getActiveIntervals()
for interval in intervals:
range = ((interval.getRequestedStartPeriod(), interval.getRequestedEndPeriod()), interval.getDataQual())
ivs.append((range, interval))
continue
ivs.sort()
pass
print 'ivs:', ivs
currentStream = pipeline.getCurrentStream()
btiProcess = currentStream.getProcessInstance(btiProcess)
badRanges = btiProcess.getVariable("badRanges")
ranges = []
if badRanges == 'x':
pass
else:
lines = badRanges.split(',')
for line in lines:
fields = line.split(":")
start = int(round(float(fields[0])))
stop = int(round(float(fields[1])))
qual = int(fields[2])
ranges.append(((start, stop), qual))
continue
ranges.sort()
pass
print 'ranges:', ranges
if len(ivs) != len(ranges):
print "Flagged and requested interval lists have different lengths!"
raise SystemExit
stuff = zip(ranges, ivs)
# # This fails if the interval overlaps the start or end of a run
# # or a gap, probably
# for (range, iv) in stuff:
# ((ft2Start, ft2Stop), ft2Qual) = range
# ((reqStart, reqStop), reqQual) = iv[0]
# if not ((ft2Start <= reqStart) and (ft2Stop >= reqStop) and (ft2Qual == reqQual)):
# print "Interval mismatch!"
# raise SystemExit
# continue
for (range, iv) in stuff:
((start, stop), qual) = range
interval = iv[1]
interval.setFt2StartPeriod(start)
interval.setFt2EndPeriod(stop)
continue
exportProcess = currentStream.getProcessInstance("exportFT2")
exportTime = exportProcess.getVariable("exportTime")
millis = long(float(exportTime) * 1000)
print millis
exportTime = Timestamp(millis)
if theSet:
theSet.setExportTime(exportTime)
theSet.setStreamId(pipeline.getStream())
theSet.setTaskName(creator)
runQuality.updateBadTimeIntervalSet(theSet)
pass
]]>
</script>
<depends>
<after process="exportFT2Seconds" status="SUCCESS"/>
<after process="exportFT2" status="SUCCESS"/>
</depends>
</process>
<process name="verifyFt2" autoRetryMaxAttempts="1">
<variables>
<var name="L1_ft2_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2")}</var>
<var name="L1_verifyFt2Error_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT2ERROR")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/verifyFt2.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerFT2" status="SUCCESS"/>
</depends>
</process>
<process name="registerVerifyFt2Error" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">verifyFt2</var>
<var name="fileType">verifyFt2Error</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="verifyFt2" status="SUCCESS"/>
</depends>
</process>
<process name="ingestVerifyFt2ErrorRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">verifyFt2Error</var>
<var name="L1_verifyFt2Error_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT2ERROR")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyFt2Error" status="SUCCESS"/>
</depends>
</process>
<process name="verifyFt2ErrorAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">verifyFt2Error</var>
<var name="outFileTypes">verifyFt2ErrorAlarm</var>
<var name="alarmFileType">verifyFt2ErrorAlarm</var>
<var name="L1_verifyFt2Error_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT2ERROR")}</var>
<var name="L1_verifyFt2ErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT2ERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyFt2Error" status="SUCCESS"/>
</depends>
</process>
<process name="registerVerifyFt2ErrorAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">verifyFt2ErrorAlarm</var>
<var name="fileType">verifyFt2ErrorAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="verifyFt2ErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="verifyFt2ErrorLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">verifyFt2ErrorAlarm</var>
<var name="L1_verifyFt2ErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT2ERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyFt2ErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestVerifyFt2ErrorAlarmRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">verifyFt2ErrorAlarm</var>
<var name="L1_verifyFt2ErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT2ERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyFt2ErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="scanFT1" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">diffRspFT1</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.diffRspFT1" status="DONE"/>
</depends>
</process>
<process name="mergeFT1" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFT1").getVariable("goodPis")}</var>
<var name="outFileTypes">ft1BadGti</var>
<var name="L1_ft1BadGti_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT1BADGTI")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanFT1" status="SUCCESS"/>
</depends>
</process>
<process name="registerFT1BadGti" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeFT1</var>
<var name="fileType">ft1BadGti</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeFT1" status="SUCCESS"/>
</depends>
</process>
<process name="gtiFT1" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">ft1BadGti/ft2Seconds</var>
<var name="outFileTypes">extendedFT1</var>
<var name="L1_ft1BadGti_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT1BADGTI")}</var>
<var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var>
<var name="L1_extendedFT1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "EXTENDEDFT1")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="registerFT1BadGti" status="SUCCESS"/>
<after process="registerFt2Seconds" status="SUCCESS"/>
</depends>
</process>
<process name="registerExtendedFT1" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">gtiFT1</var>
<var name="fileType">extendedFT1</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="gtiFT1" status="SUCCESS"/>
</depends>
</process>
<process name="filterFT1" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">extendedFT1/ft2Seconds</var>
<var name="outFileTypes">ft1</var>
<var name="L1_extendedFT1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "EXTENDEDFT1")}</var>
<var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var>
<var name="L1_ft1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT1")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="registerExtendedFT1" status="SUCCESS"/>
</depends>
</process>
<process name="registerFT1" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">filterFT1</var>
<var name="fileType">ft1</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="filterFT1" status="SUCCESS"/>
</depends>
</process>
<process name="exportFT1" autoRetryMaxAttempts="1">
<variables>
<var name="fileType">ft1</var>
<var name="L1_ft1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT1")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/exportStuff.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="registerFT1" status="SUCCESS"/>
</depends>
</process>
<process name="verifyFt1" autoRetryMaxAttempts="1">
<variables>
<var name="L1_extendedFT1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "EXTENDEDFT1")}</var>
<var name="L1_verifyFt1Error_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT1ERROR")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/verifyFt1.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerExtendedFT1" status="SUCCESS"/>
</depends>
</process>
<process name="registerVerifyFt1Error" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">verifyFt1</var>
<var name="fileType">verifyFt1Error</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="verifyFt1" status="SUCCESS"/>
</depends>
</process>
<process name="ingestVerifyFt1ErrorRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">verifyFt1Error</var>
<var name="L1_verifyFt1Error_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT1ERROR")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyFt1Error" status="SUCCESS"/>
</depends>
</process>
<process name="verifyFt1ErrorAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">verifyFt1Error</var>
<var name="outFileTypes">verifyFt1ErrorAlarm</var>
<var name="alarmFileType">verifyFt1ErrorAlarm</var>
<var name="L1_verifyFt1Error_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT1ERROR")}</var>
<var name="L1_verifyFt1ErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT1ERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyFt1Error" status="SUCCESS"/>
</depends>
</process>
<process name="registerVerifyFt1ErrorAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">verifyFt1ErrorAlarm</var>
<var name="fileType">verifyFt1ErrorAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="verifyFt1ErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="verifyFt1ErrorLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">verifyFt1ErrorAlarm</var>
<var name="L1_verifyFt1ErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT1ERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyFt1ErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestVerifyFt1ErrorAlarmRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">verifyFt1ErrorAlarm</var>
<var name="L1_verifyFt1ErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT1ERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyFt1ErrorAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="electronFT1" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">merit</var>
<var name="outFileTypes">electronFT1BadGti</var>
<var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var>
<var name="L1_electronFT1BadGti_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ELECTRONFT1BADGTI")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="registerElectronMerit" status="SUCCESS"/>
</depends>
</process>
<process name="registerElectronFT1BadGti" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">electronFT1</var>
<var name="fileType">electronFT1BadGti</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="electronFT1" status="SUCCESS"/>
</depends>
</process>
<process name="gtiElectronFT1" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">electronFT1BadGti/ft2Seconds</var>
<var name="outFileTypes">electronFT1</var>
<var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var>
<var name="L1_electronFT1BadGti_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ELECTRONFT1BADGTI")}</var>
<var name="L1_electronFT1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ELECTRONFT1")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="registerElectronFT1BadGti" status="SUCCESS"/>
<after process="registerFt2Seconds" status="SUCCESS"/>
</depends>
</process>
<process name="registerElectronFT1" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">gtiElectronFT1</var>
<var name="fileType">electronFT1</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="gtiElectronFT1" status="SUCCESS"/>
</depends>
</process>
<process name="makeLS1" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">merit</var>
<var name="outFileTypes">ls1BadGti</var>
<var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var>
<var name="L1_ls1BadGti_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "LS1BADGTI")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="registerMerit" status="SUCCESS"/>
</depends>
</process>
<process name="registerLS1BadGti" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">makeLS1</var>
<var name="fileType">ls1BadGti</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="makeLS1" status="SUCCESS"/>
</depends>
</process>
<process name="gtiLS1" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">ls1BadGti/ft2Seconds</var>
<var name="outFileTypes">extendedLS1</var>
<var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var>
<var name="L1_ls1BadGti_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "LS1BADGTI")}</var>
<var name="L1_extendedLS1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "EXTENDEDLS1")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="registerLS1BadGti" status="SUCCESS"/>
<after process="registerFt2Seconds" status="SUCCESS"/>
</depends>
</process>
<process name="registerExtendedLS1" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">gtiLS1</var>
<var name="fileType">extendedLS1</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="gtiLS1" status="SUCCESS"/>
</depends>
</process>
<process name="filterLS1" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">extendedLS1/ft2Seconds</var>
<var name="outFileTypes">ls1</var>
<var name="L1_extendedLS1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "EXTENDEDLS1")}</var>
<var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var>
<var name="L1_ls1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "LS1")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="registerExtendedLS1" status="SUCCESS"/>
</depends>
</process>
<process name="registerLS1" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">filterLS1</var>
<var name="fileType">ls1</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="filterLS1" status="SUCCESS"/>
</depends>
</process>
<process name="exportLS1" autoRetryMaxAttempts="1">
<variables>
<var name="fileType">ls1</var>
<var name="L1_ls1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "LS1")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/exportStuff.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="registerLS1" status="SUCCESS"/>
</depends>
</process>
<process name="scanRelationChunks" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">mergeRelationCrumbs</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.mergeRelationCrumbs" status="DONE"/>
</depends>
</process>
<process name="mergeRelationChunks" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanRelationChunks").getVariable("goodPis")}</var>
<var name="outFileTypes">relation</var>
<var name="L1_relation_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RELATION")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[scratch=250]&quot; -app centos6 "/>
<depends>
<after process="scanRelationChunks" status="SUCCESS"/>
</depends>
</process>
<process name="registerRelation" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeRelationChunks</var>
<var name="fileType">relation</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeRelationChunks" status="SUCCESS"/>
</depends>
</process>
<process name="scanCalChunks" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">mergeCalCrumbs</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.mergeCalCrumbs" status="DONE"/>
</depends>
</process>
<process name="mergeCalChunks" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanCalChunks").getVariable("goodPis")}</var>
<var name="outFileTypes">cal</var>
<var name="L1_cal_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CAL")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[scratch=250]&quot; -app centos6 "/>
<depends>
<after process="scanCalChunks" status="SUCCESS"/>
</depends>
</process>
<process name="registerCal" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeCalChunks</var>
<var name="fileType">cal</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeCalChunks" status="SUCCESS"/>
</depends>
</process>
<process name="scanMeritTrend" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">meritTrend</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.meritTrend" status="DONE"/>
</depends>
</process>
<process name="mergeMeritTrend" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanMeritTrend").getVariable("goodPis")}</var>
<var name="outFileTypes">meritTrend</var>
<var name="L1_meritTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTREND")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanMeritTrend" status="SUCCESS"/>
</depends>
</process>
<process name="registerMeritTrend" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeMeritTrend</var>
<var name="fileType">meritTrend</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeMeritTrend" status="SUCCESS"/>
</depends>
</process>
<process name="ingestMeritTrend" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">meritTrend</var>
<var name="L1_meritTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTREND")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ingestTrending.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerMeritTrend" status="SUCCESS"/>
</depends>
</process>
<process name="meritTrendAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">meritTrend</var>
<var name="outFileTypes">meritTrendAlarm</var>
<var name="L1_meritTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTREND")}</var>
<var name="L1_meritTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerMeritTrend" status="SUCCESS"/>
</depends>
</process>
<process name="registerMeritTrendAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">meritTrendAlarm</var>
<var name="fileType">meritTrendAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="meritTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="meritTrendLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">meritTrendAlarm</var>
<var name="L1_meritTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerMeritTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestMeritTrendRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">meritTrendAlarm</var>
<var name="L1_meritTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerMeritTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="meritHist" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digi/merit</var>
<var name="outFileTypes">meritHist</var>
<var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var>
<var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var>
<var name="L1_meritHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITHIST")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="doChunk.allRecon" status="SUCCESS"/>
<after process="registerDigi" status="SUCCESS"/>
<after process="registerMerit" status="SUCCESS"/>
</depends>
</process>
<process name="registerMeritHist" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">meritHist</var>
<var name="fileType">meritHist</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="meritHist" status="SUCCESS"/>
</depends>
</process>
<process name="meritHistAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">meritHist</var>
<var name="outFileTypes">meritHistAlarm</var>
<var name="L1_meritHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITHIST")}</var>
<var name="L1_meritHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerMeritHist" status="SUCCESS"/>
</depends>
</process>
<process name="registerMeritHistAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">meritHistAlarm</var>
<var name="fileType">meritHistAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="meritHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestMeritHistRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">meritHistAlarm</var>
<var name="L1_meritHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerMeritHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="meritHistLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">meritHistAlarm</var>
<var name="L1_meritHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerMeritHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="findSolarFlares" autoRetryMaxAttempts="1">
<variables>
<var name="L1_digiTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITREND")}</var>
<var name="L1_meritTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTREND")}</var>
<var name="L1_solarFlareHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "SOLARFLAREHIST")}</var>
<var name="L1_solarFlareLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "SOLARFLARELOG")}</var>
<var name="L1_solarFlarePlot_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "SOLARFLAREPLOT")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/findSolarFlares.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerDigiTrend" status="SUCCESS"/>
<after process="registerMeritTrend" status="SUCCESS"/>
<after process="doChunk.digiTrend" status="SUCCESS"/>
<after process="doChunk.meritTrend" status="SUCCESS"/>
</depends>
</process>
<process name="registerSolarFlareHist" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">findSolarFlares</var>
<var name="fileType">solarFlareHist</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="findSolarFlares" status="SUCCESS"/>
</depends>
</process>
<process name="registerSolarFlareLog" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">findSolarFlares</var>
<var name="fileType">solarFlareLog</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="findSolarFlares" status="SUCCESS"/>
</depends>
</process>
<process name="registerSolarFlarePlot" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">findSolarFlares</var>
<var name="fileType">solarFlarePlot</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="findSolarFlares" status="SUCCESS"/>
</depends>
</process>
<process name="suggestQuality" autoRetryMaxAttempts="1">
<variables>
<var name="btiProcess">findSolarFlares</var>
</variables>
<script><![CDATA[
creator = pipeline.getTaskVersionPath()
currentStream = pipeline.getCurrentStream()
btiProcess = currentStream.getProcessInstance(btiProcess)
badRanges = btiProcess.getVariable("badRanges")
ranges = []
if badRanges == 'x':
pass
else:
lines = badRanges.split(',')
for line in lines:
fields = line.split(":")
start = int(round(float(fields[0])))
stop = int(round(float(fields[1])))
qual = int(fields[2])
ranges.append(((start, stop), qual))
continue
ranges.sort()
pass
print 'ranges:', ranges
if ranges:
theSet = runQuality.createBadTimeIntervalSetForRun(runNumber, creator)
setId = theSet.getId()
for (start, stop), qual in ranges:
iv = runQuality.createBadTimeIntervalForSet(setId, creator)
iv.setSuggestedStartPeriod(start)
iv.setSuggestedEndPeriod(stop)
iv.setDataQual(qual)
runQuality.updateBadTimeInterval(iv)
continue
runQuality.updateBadTimeIntervalSet(theSet)
pass
]]>
</script>
<depends>
<after process="findSolarFlares" status="SUCCESS"/>
</depends>
</process>
<process name="scanReconTrend" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">reconTrend</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.reconTrend" status="DONE"/>
</depends>
</process>
<process name="mergeReconTrend" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconTrend").getVariable("goodPis")}</var>
<var name="outFileTypes">reconTrend</var>
<var name="L1_reconTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONTREND")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<!-- <after process="doChunk.mergeCalCrumbs" status="DONE"/> # This is not really necessary. In the situation it was designed to protect against, we'd rather not run at all. But we can't say that. -->
<!-- <after process="doChunk.mergeReconCrumbs" status="DONE"/> # This is not really necessary. In the situation it was designed to protect against, we'd rather not run at all. But we can't say that. -->
<after process="scanReconTrend" status="SUCCESS"/>
</depends>
</process>
<process name="registerReconTrend" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeReconTrend</var>
<var name="fileType">reconTrend</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeReconTrend" status="SUCCESS"/>
</depends>
</process>
<process name="ingestReconTrend" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">reconTrend</var>
<var name="L1_reconTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONTREND")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ingestTrending.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerReconTrend" status="SUCCESS"/>
</depends>
</process>
<process name="reconTrendAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">reconTrend</var>
<var name="outFileTypes">reconTrendAlarm</var>
<var name="L1_reconTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONTREND")}</var>
<var name="L1_reconTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONTRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerReconTrend" status="SUCCESS"/>
</depends>
</process>
<process name="registerReconTrendAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">reconTrendAlarm</var>
<var name="fileType">reconTrendAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="reconTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestReconTrendRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">reconTrendAlarm</var>
<var name="L1_reconTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONTRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerReconTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="reconTrendLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">reconTrendAlarm</var>
<var name="L1_reconTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONTRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerReconTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="scanReconHist" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">reconHist</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.reconHist" status="DONE"/>
</depends>
</process>
<process name="mergeReconHist" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconHist").getVariable("goodPis")}</var>
<var name="outFileTypes">reconHist</var>
<var name="L1_reconHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHIST")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<!-- <after process="doChunk.mergeCalCrumbs" status="DONE"/> # This is not really necessary. In the situation it was designed to protect against, we'd rather not run at all. But we can't say that. -->
<!-- <after process="doChunk.mergeReconCrumbs" status="DONE"/> # This is not really necessary. In the situation it was designed to protect against, we'd rather not run at all. But we can't say that. -->
<after process="scanReconHist" status="SUCCESS"/>
</depends>
</process>
<process name="registerReconHist" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeReconHist</var>
<var name="fileType">reconHist</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeReconHist" status="SUCCESS"/>
</depends>
</process>
<process name="reconHistAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">reconHist</var>
<var name="outFileTypes">reconHistAlarm</var>
<var name="L1_reconHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHIST")}</var>
<var name="L1_reconHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerReconHist" status="SUCCESS"/>
</depends>
</process>
<process name="registerReconHistAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">reconHistAlarm</var>
<var name="fileType">reconHistAlarm</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="reconHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="reconHistLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">reconHistAlarm</var>
<var name="L1_reconHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerReconHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestReconHistRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">reconHistAlarm</var>
<var name="L1_reconHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHISTALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerReconHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="reconHistPostProcessor" autoRetryMaxAttempts="1">
<variables>
<var name="fileType">reconHistAlarm</var>
<var name="L1_reconHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHISTALARM")}</var>
<var name="L1_reconHistAlarmDist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHISTALARMDIST")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/alarmPostProcessor.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerReconHistAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="registerReconHistAlarmDist" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">reconHistPostProcessor</var>
<var name="fileType">reconHistAlarmDist</var>
<var name="timeProcess">findChunks</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="reconHistPostProcessor" status="SUCCESS"/>
</depends>
</process>
<process name="scanTkrAnalysis" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">tkrAnalysis</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.tkrAnalysis" status="DONE"/>
</depends>
</process>
<process name="scanReconChunks" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunk</var>
<var name="subProcess">mergeReconCrumbs</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunk.mergeReconCrumbs" status="DONE"/>
</depends>
</process>
<process name="checkRun" autoRetryMaxAttempts="1">
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/checkRun.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="acdPedsLogger" status="SUCCESS"/>
<after process="calGainsLogger" status="SUCCESS"/>
<after process="calHistLogger" status="SUCCESS"/>
<after process="calPedsLogger" status="SUCCESS"/>
<after process="doChunk.checkChunk" status="SUCCESS"/>
<after process="digiHistLogger" status="SUCCESS"/>
<after process="digiTrendLogger" status="SUCCESS"/>
<after process="exportFT1" status="SUCCESS"/>
<after process="exportLS1" status="SUCCESS"/>
<after process="fastMonErrorLogger" status="SUCCESS"/>
<after process="fastMonHistLogger" status="SUCCESS"/>
<after process="fastMonTrendLogger" status="SUCCESS"/>
<after process="ingestAcdPedsRun" status="SUCCESS"/>
<after process="ingestCalGainsRun" status="SUCCESS"/>
<after process="ingestCalHistRun" status="SUCCESS"/>
<after process="ingestCalPedsRun" status="SUCCESS"/>
<after process="ingestCalTrend" status="SUCCESS"/>
<after process="ingestDigiTrend" status="SUCCESS"/>
<after process="ingestFastMonTrend" status="SUCCESS"/>
<after process="ingestMeritTrend" status="SUCCESS"/>
<after process="ingestReconTrend" status="SUCCESS"/>
<after process="ingestFastMonTrendRun" status="SUCCESS"/>
<after process="ingestFastMonHistRun" status="SUCCESS"/>
<after process="ingestFastMonErrorRun" status="SUCCESS"/>
<after process="ingestFastMonErrorAlarmRun" status="SUCCESS"/>
<after process="ingestDigiTrendRun" status="SUCCESS"/>
<after process="ingestDigiHistRun" status="SUCCESS"/>
<after process="ingestMeritTrendRun" status="SUCCESS"/>
<after process="ingestMeritHistRun" status="SUCCESS"/>
<after process="ingestReconTrendRun" status="SUCCESS"/>
<after process="ingestReconHistRun" status="SUCCESS"/>
<after process="ingestVerifyErrorRun" status="SUCCESS"/>
<after process="ingestVerifyErrorAlarmRun" status="SUCCESS"/>
<after process="ingestVerifyFt1ErrorRun" status="SUCCESS"/>
<after process="ingestVerifyFt1ErrorAlarmRun" status="SUCCESS"/>
<after process="ingestVerifyFt2ErrorRun" status="SUCCESS"/>
<after process="ingestVerifyFt2ErrorAlarmRun" status="SUCCESS"/>
<after process="ingestVerifyMeritErrorRun" status="SUCCESS"/>
<after process="ingestVerifyMeritErrorAlarmRun" status="SUCCESS"/>
<after process="meritHistLogger" status="SUCCESS"/>
<after process="meritTrendLogger" status="SUCCESS"/>
<after process="reconHistLogger" status="SUCCESS"/>
<after process="reconTrendLogger" status="SUCCESS"/>
<after process="registerAcdPlots" status="SUCCESS"/>
<after process="registerCal" status="SUCCESS"/>
<after process="registerElectronFT1" status="SUCCESS"/>
<after process="registerFastMonTuple" status="SUCCESS"/>
<!-- <after process="registerFilteredMerit" status="SUCCESS"/> -->
<after process="registerGcr" status="SUCCESS"/>
<after process="registerM7Hp" status="SUCCESS"/>
<after process="registerMeritHist" status="SUCCESS"/>
<after process="registerOrbitPlot" status="SUCCESS"/>
<after process="registerReconHistAlarmDist" status="SUCCESS"/>
<after process="registerRelation" status="SUCCESS"/>
<after process="registerSolarFlareHist" status="SUCCESS"/>
<after process="registerSolarFlareLog" status="SUCCESS"/>
<after process="registerSolarFlarePlot" status="SUCCESS"/>
<after process="registerVerifyHisto" status="SUCCESS"/>
<after process="scanReconChunks" status="SUCCESS"/>
<after process="suggestQuality" status="SUCCESS"/>
<after process="scanTkrAnalysis" status="SUCCESS"/>
<after process="verifyErrorLogger" status="SUCCESS"/>
<after process="verifyFt1ErrorLogger" status="SUCCESS"/>
<after process="verifyFt2ErrorLogger" status="SUCCESS"/>
<after process="wrapupQuality" status="SUCCESS"/>
<after process="verifyMeritErrorLogger" status="SUCCESS"/>
</depends>
<createsSubtasks>
<subtask>cleanupCompleteRun</subtask>
</createsSubtasks>
</process>
<process name="setStatus" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">checkRun</var>
</variables>
<script><![CDATA[
def good():
l1RunStatus = parentPi.getVariable('l1RunStatus')
print 'Setting L1RunStatus for run', runNumber, ' to ', l1RunStatus
runQuality.setL1RunStatus(runNumber, l1RunStatus)
reviewStatus = "Waiting review"
print 'Setting ReviewStatus for run', runNumber, ' to ', reviewStatus
runQuality.setReviewStatus(runNumber, reviewStatus)
return
def bad():
l1RunStatus = 'Failed'
print 'Setting L1RunStatus for run', runNumber, ' to ', l1RunStatus
runQuality.setL1RunStatus(runNumber, l1RunStatus)
return
currentStream = pipeline.getCurrentStream()
parentPi = currentStream.getProcessInstance(parentProcess)
parentStatus = parentPi.getStatus()
if parentStatus in ['SUCCESS']:
good()
else:
bad()
pass
]]>
</script>
<depends>
<after process="checkRun" status="DONE"/>
<after process="setRunning" status="DONE"/>
</depends>
</process>
<task name="cleanupCompleteRun" version="5.7" type="Data">
<process name="getTimes" autoRetryMaxAttempts="0"> <!-- Should use genericScript.py with a cleanup routine here -->
<variables>
<var name="l1LockAction">LockDirOnly</var>
<var name="tStart">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunks").getVariable("tStart")}</var>
<var name="tStop">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunks").getVariable("tStop")}</var>
<var name="mootKey">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunks").getVariable("mootKey")}</var>
<var name="mootAlias">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunks").getVariable("mootAlias")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/getTimes.py" batchOptions=" -E &quot;${isocRun} ${L1ProcROOT}/lockFile.py&quot; -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
</process>
<process name="mergeReconChunks" autoRetryMaxAttempts="1">
<variables>
<!-- <var name="goodPis">${goodReconPis}</var> -->
<var name="goodPis">${pipeline.getCurrentStream().getParentStream().getProcessInstance("scanReconChunks").getVariable("goodPis")}</var>
<var name="outFileTypes">recon</var>
<var name="L1_recon_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECON")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[scratch=250]&quot; -app centos6 "/>
<depends>
<after process="getTimes" status="SUCCESS"/>
</depends>
</process>
<process name="registerRecon" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeReconChunks</var>
<var name="fileType">recon</var>
<var name="timeProcess">getTimes</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeReconChunks" status="SUCCESS"/>
</depends>
</process>
<process name="verifyComp" autoRetryMaxAttempts="1">
<variables>
<var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var>
<var name="L1_verifyLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYLOG")}</var>
<var name="L1_verifyHisto_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYHISTO")}</var>
<var name="completeness">Complete</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/verify.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="getTimes" status="SUCCESS"/>
</depends>
</process>
<process name="registerVerifyHistoComp" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">verifyComp</var>
<var name="fileType">verifyHisto</var>
<var name="timeProcess">getTimes</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="verifyComp" status="SUCCESS"/>
</depends>
</process>
<process name="registerVerifyLogComp" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">verifyComp</var>
<var name="fileType">verifyLog</var>
<var name="timeProcess">getTimes</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="verifyComp" status="SUCCESS"/>
</depends>
</process>
<process name="ingestVerifyErrorRunComp" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">verifyLog</var>
<var name="L1_verifyLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYLOG")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyLogComp" status="SUCCESS"/>
</depends>
</process>
<process name="verifyErrorAlarmComp" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">verifyLog</var>
<var name="outFileTypes">verifyErrorAlarm</var>
<var name="L1_verifyLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYLOG")}</var>
<var name="L1_verifyErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyLogComp" status="SUCCESS"/>
</depends>
</process>
<process name="registerVerifyErrorAlarmComp" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">verifyErrorAlarmComp</var>
<var name="fileType">verifyErrorAlarm</var>
<var name="timeProcess">getTimes</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="verifyErrorAlarmComp" status="SUCCESS"/>
</depends>
</process>
<process name="verifyErrorLoggerComp" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">verifyErrorAlarm</var>
<var name="L1_verifyErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyErrorAlarmComp" status="SUCCESS"/>
</depends>
</process>
<process name="ingestVerifyErrorAlarmRunComp" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">verifyErrorAlarm</var>
<var name="L1_verifyErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYERRORALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerVerifyErrorAlarmComp" status="SUCCESS"/>
</depends>
</process>
<process name="mergeTkrAnalysis" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream().getParentStream().getProcessInstance("scanTkrAnalysis").getVariable("goodPis")}</var>
<var name="outFileTypes">tkrAnalysis</var>
<var name="L1_tkrAnalysis_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRANALYSIS")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<!-- <after process="doChunk.mergeReconCrumbs" status="DONE"/> # This is not really necessary. In the situation it was designed to protect against, we'd rather not run at all. But we can't say that. -->
<after process="getTimes" status="SUCCESS"/>
</depends>
</process>
<process name="registerTkrAnalysis" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeTkrAnalysis</var>
<var name="fileType">tkrAnalysis</var>
<var name="timeProcess">getTimes</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeTkrAnalysis" status="SUCCESS"/>
</depends>
</process>
<process name="tkrMonitor" autoRetryMaxAttempts="1">
<variables>
<var name="L1_tkrAnalysis_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRANALYSIS")}</var>
<var name="L1_tkrMonitor_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRMONITOR")}</var>
<var name="L1_tkrReport_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRREPORT")}</var>
<!-- <var name="L1_tkrAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRALARM")}</var> -->
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/tkrMonitor.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerTkrAnalysis" status="SUCCESS"/>
</depends>
</process>
<process name="registerTkrMonitor" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">tkrMonitor</var>
<var name="fileType">tkrMonitor</var>
<var name="timeProcess">getTimes</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="tkrMonitor" status="SUCCESS"/>
</depends>
</process>
<process name="registerTkrReport" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">tkrMonitor</var>
<var name="fileType">tkrReport</var>
<var name="timeProcess">getTimes</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="tkrMonitor" status="SUCCESS"/>
</depends>
</process>
<!-- We don't have TKR alarms yet
<process name="registerTkrAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">tkrMonitor</var>
<var name="fileType">tkrAlarm</var>
<var name="timeProcess">getTimes</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="tkrMonitor" status="SUCCESS"/>
</depends>
</process>
-->
<process name="tkrTrend" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">tkrMonitor</var>
<var name="outFileTypes">tkrTrend</var>
<var name="L1_tkrMonitor_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRMONITOR")}</var>
<var name="L1_tkrTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRTREND")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerTkrMonitor" status="SUCCESS"/>
</depends>
</process>
<process name="registerTkrTrend" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">tkrTrend</var>
<var name="fileType">tkrTrend</var>
<var name="timeProcess">getTimes</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="tkrTrend" status="SUCCESS"/>
</depends>
</process>
<process name="ingestTkrTrend" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">tkrTrend</var>
<var name="L1_tkrTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRTREND")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ingestTrending.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerTkrTrend" status="SUCCESS"/>
</depends>
</process>
<process name="tkrTrendAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">tkrTrend</var>
<var name="outFileTypes">tkrTrendAlarm</var>
<var name="L1_tkrTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRTREND")}</var>
<var name="L1_tkrTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRTRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerTkrTrend" status="SUCCESS"/>
</depends>
</process>
<process name="registerTkrTrendAlarm" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">tkrTrendAlarm</var>
<var name="fileType">tkrTrendAlarm</var>
<var name="timeProcess">getTimes</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="tkrTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="ingestTkrTrendRun" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">tkrTrendAlarm</var>
<var name="L1_tkrTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRTRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerTkrTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="tkrTrendLogger" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">tkrTrendAlarm</var>
<var name="L1_tkrTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRTRENDALARM")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerTkrTrendAlarm" status="SUCCESS"/>
</depends>
</process>
<process name="cleanupChunks" autoRetryMaxAttempts="1">
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/deleteDirs.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="registerRecon" status="SUCCESS"/>
<after process="registerTkrReport" status="SUCCESS"/>
<after process="ingestTkrTrend" status="SUCCESS"/>
<after process="ingestTkrTrendRun" status="SUCCESS"/>
<after process="ingestVerifyErrorRunComp" status="SUCCESS"/>
<after process="ingestVerifyErrorAlarmRunComp" status="SUCCESS"/>
<!-- We don't have TKR alarms (yet?)
<after process="registerTkrAlarm" status="SUCCESS"/>
-->
<after process="registerVerifyHistoComp" status="SUCCESS"/>
<after process="tkrTrendLogger" status="SUCCESS"/>
<after process="verifyErrorLoggerComp" status="SUCCESS"/>
</depends>
</process>
<process name="unLockRun" autoRetryMaxAttempts="1">
<variables>
<var name="l1LockAction">UnLock</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/lockFile.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="cleanupChunks" status="SUCCESS"/>
</depends>
</process>
</task> <!-- cleanupCompleteRun -->
<task name="doChunk" version="5.7" type="Data">
<variables>
<var name="EVENTSTOSKIP"></var>
</variables>
<prerequisites>
<prerequisite name="CHUNK_ID" type="string"/>
</prerequisites>
<process name="digitization" autoRetryMaxAttempts="1">
<variables>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
<var name="inFileTypes">event</var>
<var name="L1_event_ver">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunks").getVariable("L1_PI_version")}</var>
<var name="outFileTypes">digi</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
</process>
<process name="fakeFT2" autoRetryMaxAttempts="1">
<variables>
<var name="outFileTypes">ft2Fake</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
</process>
<process name="fastMonTuple" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">event</var>
<var name="L1_event_ver">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunks").getVariable("L1_PI_version")}</var>
<var name="reportType">fastMonTuple</var>
<var name="outFileTypes">fastMonTuple/fastMonError</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
</process>
<process name="fastMonHist" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">fastMonHist</var>
<var name="inFileTypes">fastMonTuple</var>
<var name="outFileTypes">fastMonHist</var>
<var name="L1_fastMonTuple_ver">${pipeline.getCurrentStream.getProcessInstance("fastMonTuple").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="fastMonTuple" status="SUCCESS"/>
</depends>
</process>
<process name="fastMonTrend" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">fastMonTuple</var>
<var name="outFileTypes">fastMonTrend</var>
<var name="L1_fastMonTuple_ver">${pipeline.getCurrentStream.getProcessInstance("fastMonTuple").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="fastMonTuple" status="SUCCESS"/>
</depends>
</process>
<process name="digiHist" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digi</var>
<var name="outFileTypes">digiHist</var>
<var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="digitization" status="SUCCESS"/>
</depends>
</process>
<process name="digiTrend" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digi</var>
<var name="outFileTypes">digiTrend</var>
<var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="digitization" status="SUCCESS"/>
</depends>
</process>
<process name="calTrend" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digi</var>
<var name="outFileTypes">calTrend</var>
<var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="digitization" status="SUCCESS"/>
</depends>
</process>
<process name="setupCrumbs" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digi</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
<var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var>
<var name="L1_ft2Fake_ver">${pipeline.getCurrentStream.getProcessInstance("fakeFT2").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="digitization" status="SUCCESS"/>
<after process="fakeFT2" status="SUCCESS"/>
</depends>
<createsSubtasks>
<subtask>doCrumb</subtask>
</createsSubtasks>
</process>
<process name="scanReconCrumbs" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doCrumb</var>
<var name="subProcess">recon</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doCrumb.recon" status="DONE"/>
</depends>
</process>
<process name="allRecon" autoRetryMaxAttempts="1">
<script><![CDATA[
# This script is automagically pasted into the XML at install time
print "Placeholder to aggregate dependency flow."
]]>
</script>
<depends>
<after process="doCrumb.recon" status="SUCCESS"/>
</depends>
</process>
<process name="mergeMeritCrumbs" autoRetryMaxAttempts="1">
<variables>
<var name="outFileTypes">merit</var>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconCrumbs").getVariable("goodPis")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="scanReconCrumbs" status="SUCCESS"/>
</depends>
</process>
<process name="mergeReconCrumbs" autoRetryMaxAttempts="1">
<variables>
<var name="outFileTypes">recon</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconCrumbs").getVariable("goodPis")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanReconCrumbs" status="SUCCESS"/>
</depends>
</process>
<process name="mergeRelationCrumbs" autoRetryMaxAttempts="1">
<variables>
<var name="outFileTypes">relation</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconCrumbs").getVariable("goodPis")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanReconCrumbs" status="SUCCESS"/>
</depends>
</process>
<process name="mergeCalCrumbs" autoRetryMaxAttempts="1">
<variables>
<var name="outFileTypes">cal</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconCrumbs").getVariable("goodPis")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanReconCrumbs" status="SUCCESS"/>
</depends>
</process>
<process name="mergeGcrCrumbs" autoRetryMaxAttempts="1">
<variables>
<var name="outFileTypes">gcr</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconCrumbs").getVariable("goodPis")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanReconCrumbs" status="SUCCESS"/>
</depends>
</process>
<process name="makeFT1" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">merit</var>
<var name="outFileTypes">ft1NoDiffRsp</var>
<var name="L1_merit_ver">${pipeline.getCurrentStream.getProcessInstance("mergeMeritCrumbs").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="mergeMeritCrumbs" status="SUCCESS"/>
</depends>
</process>
<process name="diffRspFT1" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">ft1NoDiffRsp/ft2Fake</var>
<var name="outFileTypes">ft1BadGti</var>
<var name="L1_ft1NoDiffRsp_ver">${pipeline.getCurrentStream.getProcessInstance("makeFT1").getVariable("L1_PI_version")}</var>
<var name="L1_ft2Fake_ver">${pipeline.getCurrentStream.getProcessInstance("fakeFT2").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="makeFT1" status="SUCCESS"/>
</depends>
</process>
<process name="meritTrend" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digi/merit</var>
<var name="outFileTypes">meritTrend</var>
<var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var>
<var name="L1_merit_ver">${pipeline.getCurrentStream.getProcessInstance("mergeMeritCrumbs").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="allRecon" status="SUCCESS"/>
<after process="mergeMeritCrumbs" status="SUCCESS"/>
</depends>
</process>
<process name="reconHist" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">cal/digi/recon</var>
<var name="outFileTypes">reconHist</var>
<var name="L1_cal_ver">${pipeline.getCurrentStream.getProcessInstance("mergeCalCrumbs").getVariable("L1_PI_version")}</var>
<var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var>
<var name="L1_recon_ver">${pipeline.getCurrentStream.getProcessInstance("mergeReconCrumbs").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="allRecon" status="SUCCESS"/>
<after process="mergeCalCrumbs" status="SUCCESS"/>
<after process="mergeReconCrumbs" status="SUCCESS"/>
</depends>
</process>
<process name="reconTrend" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">cal/digi/recon</var>
<var name="outFileTypes">reconTrend</var>
<var name="L1_cal_ver">${pipeline.getCurrentStream.getProcessInstance("mergeCalCrumbs").getVariable("L1_PI_version")}</var>
<var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var>
<var name="L1_recon_ver">${pipeline.getCurrentStream.getProcessInstance("mergeReconCrumbs").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="allRecon" status="SUCCESS"/>
<after process="mergeCalCrumbs" status="SUCCESS"/>
<after process="mergeReconCrumbs" status="SUCCESS"/>
</depends>
</process>
<process name="tkrAnalysis" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digi/recon</var>
<var name="outFileTypes">tkrAnalysis</var>
<var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var>
<var name="L1_recon_ver">${pipeline.getCurrentStream.getProcessInstance("mergeReconCrumbs").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="allRecon" status="SUCCESS"/>
<after process="mergeReconCrumbs" status="SUCCESS"/>
</depends>
</process>
<process name="checkChunk" autoRetryMaxAttempts="1">
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/checkChunk.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="fastMonHist" status="SUCCESS"/>
<after process="fastMonTrend" status="SUCCESS"/>
<after process="digiHist" status="SUCCESS"/>
<after process="digiTrend" status="SUCCESS"/>
<after process="calTrend" status="SUCCESS"/>
<after process="diffRspFT1" status="SUCCESS"/>
<after process="mergeGcrCrumbs" status="SUCCESS"/>
<after process="mergeRelationCrumbs" status="SUCCESS"/>
<after process="meritTrend" status="SUCCESS"/>
<after process="reconHist" status="SUCCESS"/>
<after process="reconTrend" status="SUCCESS"/>
<after process="tkrAnalysis" status="SUCCESS"/>
</depends>
</process>
<task name="doCrumb" version="5.7" type="Data">
<prerequisites>
<prerequisite name="CRUMB_ID" type="string"/>
<prerequisite name="crumbEvents" type="string"/>
<prerequisite name="crumbStart" type="string"/>
</prerequisites>
<process name="recon" autoRetryMaxAttempts="1">
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/recon.py" batchOptions=" -q glastdataq -sp ${reconPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
</process>
</task> <!-- doCrumb -->
</task> <!-- doChunk -->
</task> <!-- doRun -->
<task name="doLci" version="5.7" type="Data">
<variables>
<var name="dataCatDir">/Data/Flight/Level1/${DATASOURCE}</var>
</variables>
<prerequisites>
<prerequisite name="DATASOURCE" type="string"/>
<prerequisite name="RUNID" type="string"/>
<prerequisite name="RUNSTATUS" type="string"/>
</prerequisites>
<process name="findChunksLci" autoRetryMaxAttempts="1">
<variables>
<var name="l1LockAction">LockDirOnly</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 --add-env=flightops python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -E &quot;${isocRun} ${L1ProcROOT}/lockFile.py&quot; -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<createsSubtasks>
<subtask>doChunkLci</subtask>
</createsSubtasks>
</process>
<process name="setRunningLci" autoRetryMaxAttempts="1">
<variables>
<var name="l1RunStatus">Running</var>
</variables>
<script><![CDATA[
from java.util import HashMap
def setIfUnset(container, key, value):
oldValue = container.get(key)
if oldValue is None or oldValue == 'Unset':
print 'Setting %s to %s.' % (key, value)
container[key] = value
else: print 'Current value of %s is %s, not changing it.' % (key, oldValue)
return
print 'Setting L1RunStatus for run', runNumber, ' to ', l1RunStatus
runQuality.setL1RunStatus(runNumber, l1RunStatus)
reviewStatus = "Unset"
print 'Setting ReviewStatus for run', runNumber, ' to ', reviewStatus
runQuality.setReviewStatus(runNumber, reviewStatus)
newQuality = 'Good'
quality = runQuality.getRunQuality(runNumber)
print 'Current quality for run %s is %s.' % (runNumber, quality)
if quality is None or quality == 'Unset':
print 'Setting quality for run %s to %s.' % (runNumber, newQuality)
runQuality.setRunQuality(runNumber, newQuality)
quality = newQuality
else: print 'Not changing run quality.'
pipeline.setVariable('runQuality', quality)
fields = ['ACDQuality', 'CALQuality', 'GPSQuality', 'TRACKERQuality',
'TRIGGERQuality']
metaData = runQuality.getMetaData(runNumber)
print 'old metaData:', metaData
for key in fields: setIfUnset(metaData, key, newQuality)
print 'new metadata:', metaData
runQuality.setOrUpdateMetaData(runNumber, metaData)
]]>
</script>
<depends>
<after process="findChunksLci" status="SUCCESS"/>
</depends>
</process>
<process name="copyM7HpLci" autoRetryMaxAttempts="1">
<variables>
<var name="L1_magic7Hp_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MAGIC7HP")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/copyM7.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 " />
<depends>
<after process="findChunksLci" status="SUCCESS"/>
</depends>
</process>
<process name="registerM7HpLci" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">copyM7HpLci</var>
<var name="fileType">magic7Hp</var>
<var name="timeProcess">findChunksLci</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="copyM7HpLci" status="SUCCESS"/>
</depends>
</process>
<process name="scanDigiLci" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunkLci</var>
<var name="subProcess">digitizationLci</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunkLci.digitizationLci" status="DONE"/>
</depends>
</process>
<process name="mergeDigiLci" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanDigiLci").getVariable("goodPis")}</var>
<var name="outFileTypes">digi</var>
<var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[scratch=250]&quot; -app centos6 "/>
<depends>
<after process="scanDigiLci" status="SUCCESS"/>
</depends>
</process>
<process name="registerDigiLci" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeDigiLci</var>
<var name="fileType">digi</var>
<var name="timeProcess">findChunksLci</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeDigiLci" status="SUCCESS"/>
</depends>
</process>
<process name="scanDigiHistLci" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunkLci</var>
<var name="subProcess">digiHistLci</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunkLci.digiHistLci" status="DONE"/>
</depends>
</process>
<process name="mergeDigiHistLci" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanDigiHistLci").getVariable("goodPis")}</var>
<var name="outFileTypes">digiHist</var>
<var name="L1_digiHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHIST")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[scratch=10]&quot; -app centos6 "/>
<depends>
<after process="scanDigiHistLci" status="SUCCESS"/>
</depends>
</process>
<process name="registerDigiHistLci" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeDigiHistLci</var>
<var name="fileType">digiHist</var>
<var name="timeProcess">findChunksLci</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeDigiHistLci" status="SUCCESS"/>
</depends>
</process>
<process name="scanFastMonTupleLci" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunkLci</var>
<var name="subProcess">fastMonTupleLci</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunkLci.fastMonTupleLci" status="DONE"/>
</depends>
</process>
<process name="mergeFastMonErrorLci" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonTupleLci").getVariable("goodPis")}</var>
<var name="outFileTypes">fastMonError</var>
<var name="L1_fastMonError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERROR")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="scanFastMonTupleLci" status="SUCCESS"/>
</depends>
</process>
<process name="registerFastMonErrorLci" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeFastMonErrorLci</var>
<var name="fileType">fastMonError</var>
<var name="timeProcess">findChunksLci</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeFastMonErrorLci" status="SUCCESS"/>
</depends>
</process>
<process name="mergeFastMonTupleLci" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonTupleLci").getVariable("goodPis")}</var>
<var name="outFileTypes">fastMonTuple</var>
<var name="L1_fastMonTuple_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTUPLE")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[scratch=10]&quot; -app centos6 "/>
<depends>
<after process="scanFastMonTupleLci" status="SUCCESS"/>
</depends>
</process>
<process name="registerFastMonTupleLci" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeFastMonTupleLci</var>
<var name="fileType">fastMonTuple</var>
<var name="timeProcess">findChunksLci</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeFastMonTupleLci" status="SUCCESS"/>
</depends>
</process>
<process name="scanFastMonHistLci" autoRetryMaxAttempts="1">
<variables>
<var name="subTask">doChunkLci</var>
<var name="subProcess">fastMonHistLci</var>
</variables>
<script><![CDATA[
finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED']
def getStreams():
thisStream = pipeline.getCurrentStream()
thisId = thisStream.getId()
thisPath = thisStream.getIdPath()
print 'stream path: %s' % thisPath
thisTask = thisStream.getTask()
print 'this task: %s' % thisTask
delivsToIgnore = str(deliveriesToIgnore)
badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv]
print 'Skipping deliveries', badDelivs
rdlStreams = []
allStreams = thisTask.getStreams(thisId)
for stream in allStreams:
thatPath = stream.getIdPath()
status = stream.getStatus()
print stream, thatPath, status
if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs:
rdlStreams.append(stream)
pass
continue
return rdlStreams
def getChunkPis():
"""Examine chunk-level PIs to see what files we expect to find for merging
and their versions.
"""
rdlStreams = getStreams()
pis = []
for rdl in rdlStreams:
rdlChunks = rdl.getSubStreams(subTask)
rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks]
pis.extend(rdlPis)
continue
return pis
def getCrumbPis():
thisStream = pipeline.getCurrentStream()
subStreams = thisStream.getSubStreams(subTask)
pis = []
for stream in subStreams:
print 'subStream: %s' % stream
sId = stream.getId()
print 'subId: %s' % sId
sPi = stream.getProcessInstance(subProcess)
print 'subPi: %s' % sPi
pis.append((sId, sPi))
return pis
getPis = {
'doChunk': getChunkPis,
'doChunkLci': getChunkPis,
'doCrumb': getCrumbPis
}[subTask]
pis = getPis()
pis.sort()
versionTags = []
print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess)
for streamId, subPi in pis:
ec = subPi.getExitCode()
statStr = subPi.getStatus()
print streamId, ec, statStr
if ec or statStr != 'SUCCESS':
continue
varNames = ['L1_PI_ID', 'L1_PI_version']
# tup = tuple([subPi.getVariable(name) for name in varNames])
tup = []
varMap = subPi.getVariables()
print 'varMap: %s' % varMap
#raise SystemExit, 'quit here'
for name in varNames:
print 'key: %s' % name
value = subPi.getVariable(name)
print 'value: %s' % value
tup.append(value)
tup = tuple(tup)
print tup
if None in tup:
continue
tag = '%s:%s' % tup
versionTags.append(tag)
continue
if not versionTags: raise SystemExit, "No successful jobs, failing."
goodPis = ','.join(versionTags)
pipeline.setVariable('goodPis', goodPis)
]]>
</script>
<depends>
<after process="doChunkLci.fastMonHistLci" status="DONE"/>
</depends>
</process>
<process name="mergeFastMonHistLci" autoRetryMaxAttempts="1">
<variables>
<var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonHistLci").getVariable("goodPis")}</var>
<var name="outFileTypes">fastMonHist</var>
<var name="L1_fastMonHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONHIST")}</var>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[scratch=10]&quot; -app centos6 "/>
<depends>
<after process="scanFastMonHistLci" status="SUCCESS"/>
</depends>
</process>
<process name="registerFastMonHistLci" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">mergeFastMonHistLci</var>
<var name="fileType">fastMonHist</var>
<var name="timeProcess">findChunksLci</var>
</variables>
<script><![CDATA[
"""@brief Register output data with the data server.
@author W. Focke <[email protected]>
"""
# This script is automagically pasted into the XML at install time
from java.util import HashMap
def getVar(fileType, name):
mangledName = '_'.join([nameManglingPrefix, fileType, name])
value = parentPI.getVariable(mangledName)
return value
currentStream = pipeline.getCurrentStream()
parentPI = currentStream.getProcessInstance(parentProcess)
runNumber = int(RUNID[1:])
dsName = RUNID
fileFormat = getVar(fileType, 'format')
dcType = getVar(fileType, 'dcType')
dcGroup = getVar(fileType, 'group')
site = getVar(fileType, 'site')
fileName = getVar(fileType, 'fileName')
version = getVar(fileType, 'ver')
fcPi = currentStream.getProcessInstance(timeProcess)
lessBrokenTStart = fcPi.getVariable('tStart')
lessBrokenTStop = fcPi.getVariable('tStop')
mootAlias = fcPi.getVariable('mootAlias')
mootKey = fcPi.getVariable('mootKey')
attributes = HashMap()
attributes.put('sCreator', creator)
attributes.put('sDataSource', DATASOURCE)
attributes.put('nDownlink', DOWNLINK_ID)
attributes.put('sIntent', mootAlias)
attributes.put('nMetStart', lessBrokenTStart)
attributes.put('nMetStop', lessBrokenTStop)
attributes.put('nMootKey', mootKey)
attributes.put('nRun', runNumber)
mdRepr = getVar(fileType, 'metadata')
if mdRepr:
metadata = eval(mdRepr)
for key, value in metadata.items(): attributes.put(key, value)
print attributes
dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName)
dsNew.setVersionID(version)
ds = datacatalog.registerDataset(dsNew, attributes);
]]>
</script>
<depends>
<after process="mergeFastMonHistLci" status="SUCCESS"/>
</depends>
</process>
<process name="checkRunLci" autoRetryMaxAttempts="1">
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/checkRun.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="doChunkLci.checkChunkLci" status="SUCCESS"/>
<after process="registerDigiLci" status="SUCCESS"/>
<after process="registerDigiHistLci" status="SUCCESS"/>
<after process="registerFastMonErrorLci" status="SUCCESS"/>
<after process="registerFastMonHistLci" status="SUCCESS"/>
<after process="registerFastMonTupleLci" status="SUCCESS"/>
<after process="registerM7HpLci" status="SUCCESS"/>
</depends>
<createsSubtasks>
<subtask>cleanupCompleteRunLci</subtask>
</createsSubtasks>
</process>
<process name="setStatusLci" autoRetryMaxAttempts="1">
<variables>
<var name="parentProcess">checkRunLci</var>
</variables>
<script><![CDATA[
def good():
l1RunStatus = parentPi.getVariable('l1RunStatus')
print 'Setting L1RunStatus for run', runNumber, ' to ', l1RunStatus
runQuality.setL1RunStatus(runNumber, l1RunStatus)
reviewStatus = "Waiting review"
print 'Setting ReviewStatus for run', runNumber, ' to ', reviewStatus
runQuality.setReviewStatus(runNumber, reviewStatus)
return
def bad():
l1RunStatus = 'Failed'
print 'Setting L1RunStatus for run', runNumber, ' to ', l1RunStatus
runQuality.setL1RunStatus(runNumber, l1RunStatus)
return
currentStream = pipeline.getCurrentStream()
parentPi = currentStream.getProcessInstance(parentProcess)
parentStatus = parentPi.getStatus()
if parentStatus in ['SUCCESS']:
good()
else:
bad()
pass
]]>
</script>
<depends>
<after process="checkRunLci" status="DONE"/>
<after process="setRunningLci" status="DONE"/>
</depends>
</process>
<task name="cleanupCompleteRunLci" version="5.7" type="Data">
<process name="cleanupChunksLci" autoRetryMaxAttempts="1">
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/deleteDirs.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
</process>
</task> <!-- cleanupCompleteRunLci -->
<task name="doChunkLci" version="5.7" type="Data">
<variables>
<var name="EVENTSTOSKIP"></var>
</variables>
<prerequisites>
<prerequisite name="CHUNK_ID" type="string"/>
</prerequisites>
<process name="digitizationLci" autoRetryMaxAttempts="1">
<variables>
<var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var>
<var name="inFileTypes">event</var>
<var name="L1_event_ver">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunksLci").getVariable("L1_PI_version")}</var>
<var name="outFileTypes">digi</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
</process>
<process name="fastMonTupleLci" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">event</var>
<var name="L1_event_ver">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunksLci").getVariable("L1_PI_version")}</var>
<var name="reportType">fastMonTuple</var>
<var name="outFileTypes">fastMonTuple/fastMonError</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
</process>
<process name="fastMonHistLci" autoRetryMaxAttempts="1">
<variables>
<var name="reportType">fastMonHist</var>
<var name="inFileTypes">fastMonTuple</var>
<var name="outFileTypes">fastMonHist</var>
<var name="L1_fastMonTuple_ver">${pipeline.getCurrentStream.getProcessInstance("fastMonTupleLci").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="fastMonTupleLci" status="SUCCESS"/>
</depends>
</process>
<process name="digiHistLci" autoRetryMaxAttempts="1">
<variables>
<var name="inFileTypes">digi</var>
<var name="outFileTypes">digiHist</var>
<var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitizationLci").getVariable("L1_PI_version")}</var>
</variables>
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="digitizationLci" status="SUCCESS"/>
</depends>
</process>
<process name="checkChunkLci" autoRetryMaxAttempts="1">
<job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/checkChunk.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R &quot;select[${standardSelect}] rusage[${standardRusage}]&quot; -app centos6 "/>
<depends>
<after process="fastMonHistLci" status="SUCCESS"/>
<after process="digiHistLci" status="SUCCESS"/>
</depends>
</process>
</task> <!-- doChunkLci -->
</task> <!-- doLci -->
</task> <!-- L1Proc -->
</pipeline>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment