testT1

README.md

Instructions

1. Set up CMS Connect account: https://connect.uscms.org/
2. Log in to CMS connect and download files:

   wget https://gist.githubusercontent.com/kpedro88/dd5c8a43b1aafbfc7e5953d327be40e0/raw/testT1.sh
   wget https://gist.githubusercontent.com/kpedro88/dd5c8a43b1aafbfc7e5953d327be40e0/raw/testT1.jdl
   chmod +x testT1.sh
   

3. Edit JDL file to change output directory in arguments to an EOS area where you have write permission
4. Submit job:

   condor_submit testT1.jdl
   

testT1.jdl

universe = vanilla
Executable = testT1.sh
+REQUIRED_OS = "rhel7"
+DesiredOS = REQUIRED_OS
request_cpus = 1
Should_Transfer_Files = YES
WhenToTransferOutput = ON_EXIT_OR_EVICT
Transfer_Input_Files = testT1.sh
Output = testT1_$(Process)_$(Cluster).stdout
Error = testT1_$(Process)_$(Cluster).stderr
Log = testT1_$(Process)_$(Cluster).condor
notification = Never
x509userproxy = $ENV(X509_USER_PROXY)
Arguments = -o root://cmseos.fnal.gov//store/user/pedrok/test
want_graceful_removal = true
on_exit_remove = (ExitBySignal == False) && (ExitCode == 0)
on_exit_hold = ( (ExitBySignal == True) || (ExitCode != 0) )
on_exit_hold_reason = strcat("Job held by ON_EXIT_HOLD due to ",\
        ifThenElse((ExitBySignal == True), "exit by signal", \
strcat("exit code ",ExitCode)), ".")
job_machine_attrs = "GLIDEIN_CMSSite"
+AvoidSystemPeriodicRemove = True
+DESIRED_Sites = "T1_US_FNAL"
Queue 1

testT1.sh

#!/bin/bash

# helper function to stage out via xrdcp
stageOut() {
	WAIT=5
	NUMREP=5
	INPUT=""
	OUTPUT=""
	XRDARGS=""
	QUIET=0
	GFAL=0
	CMDSTR="xrdcp"
	REMOVE=0
	CLEANUP=""

	stageOut_usage() {
		case `uname` in
			Linux) ECHO="echo -e" ;;
			*) ECHO="echo" ;;
		esac

		$ECHO "stageOut [options]"
		$ECHO ""
		$ECHO "Options:"
		$ECHO "-i input      \tinput file name (required)"
		$ECHO "-o output     \toutput file name (required)"
		$ECHO "-w wait       \twait time in seconds (default = $WAIT)"
		$ECHO "-n num        \tnumber of repetitions (default = $NUMREP)"
		$ECHO "-x args       \tany arguments to pass to xrdcp/gfal-copy (should be quoted)"
		$ECHO "-g            \tUse gfal-copy rather than xrdcp"
		$ECHO "-q            \tquiet (don't print any messages)"
		$ECHO "-r            \tremove local file if successfully copied"
		$ECHO "-c files      \tcleanup: delete specified file(s) if copy fails"
	}

	# set vars used by getopts to local
	local OPTIND OPTARG
	while getopts "i:o:w:n:x:gqrc:" opt; do
		case "$opt" in
			i) INPUT="$OPTARG"
			;;
			o) OUTPUT="$OPTARG"
			;;
			w) WAIT="$OPTARG"
			;;
			n) NUMREP="$OPTARG"
			;;
			x) XRDARGS="$OPTARG"
			;;
			g) GFAL=1
			   CMDSTR="gfal-copy"
			;;
			q) QUIET=1
			;;
			r) REMOVE=1
			;;
			c) CLEANUP="$OPTARG"
			;;
		esac
	done

	if [[ -z "$INPUT" ]] || [[ -z "$OUTPUT" ]]; then
		stageOut_usage
		return 1
	fi

	# try to copy n times, increasing wait each time
	TMPWAIT=0
	for ((i=0; i < $NUMREP; i++)); do
		if [ $GFAL -eq 1 ]; then
			env -i X509_USER_PROXY=${X509_USER_PROXY} gfal-copy $XRDARGS $INPUT $OUTPUT
		else
			xrdcp $XRDARGS $INPUT $OUTPUT
		fi
		XRDEXIT=$?
		if [ $XRDEXIT -eq 0 ]; then
			if [ $REMOVE -eq 1 ]; then rm $INPUT; fi
			return 0
		fi
		# in case of bad exit, wait and try again
		TMPWAIT=$(($TMPWAIT + $WAIT))
		if [ $QUIET -eq 0 ]; then echo "Exit code $XRDEXIT, failure in $CMDSTR. Retry after $TMPWAIT seconds..."; fi
		sleep $TMPWAIT
	done

	# if we get here, it really didn't work
	if [ $QUIET -eq 0 ]; then echo "$CMDSTR failed $NUMREP times. It might be an actual problem."; fi
	if [ -n "$CLEANUP" ]; then rm $CLEANUP; fi
	return 60000
}

# helper function to parse job classads
# tries to use condor_q if available, if not uses awk
# usage example: VAL=$(getFromClassAd RequestCpus)
getFromClassAd() {
	ARG=$1
	if [ -z "$ARG" ]; then exit 1; fi

	if type condor_q > /dev/null 2>&1; then
		condor_q -jobads ${_CONDOR_SCRATCH_DIR}/.job.ad -af ${ARG}
	else
		ARG=${ARG}" = "
		ARGVAL=$(grep -m 1 "${ARG}" ${_CONDOR_SCRATCH_DIR}/.job.ad)
		if [ -z "$ARGVAL" ]; then exit 1; fi
		awk -v val="${ARGVAL}" -v rem="${ARG}" 'BEGIN { sub(rem,"",val) ; print val }'
	fi
}

export OUTDIR=""
export OPTIND=1
while [[ $OPTIND -le $# ]]; do
	# getopts in silent mode, don't exit on errors
	OPTOLD=$OPTIND
	getopts ":o:" opt || status=$?
	case "$opt" in
		o) export OUTDIR=$OPTARG
		;;
		# keep going if getopts had an error, but make sure not to skip anything
		\? | :) OPTIND=$((OPTOLD+1))
		;;
	esac
done

# make some random output
OUTFILE=$(uuidgen).txt
echo $OUTFILE > $OUTFILE

# copy output to eos
echo "CMSSITE currently set to: ${CMSSITE}"
if [[ -z "$CMSSITE" ]] || [[ "$CMSSITE" == "" ]]; then
	echo -e "\tGetting CMSSITE from the job ClassAd"
	CMSSITE=$(getFromClassAd MachineAttrGLIDEIN_CMSSite0)
	echo -e "\tCMSSITE is now set to: ${CMSSITE}"
fi
export CMDSTR="xrdcp"
export GFLAG=""
if [[ ( "$CMSSITE" == *"T1_US_FNAL"* && "${OUTDIR}" == *"root://cmseos.fnal.gov/"* ) ]]; then
	export CMDSTR="gfal-copy"
	export GFLAG="-g"
	export GSIFTP_ENDPOINT="gsiftp://cmseos-gridftp.fnal.gov//eos/uscms/store/user/"
	export OUTDIR=${GSIFTP_ENDPOINT}${OUTDIR#root://cmseos.fnal.gov//store/user/}
elif [[ "${OUTDIR}" == *"gsiftp://"* ]]; then
	export CMDSTR="gfal-copy"
	export GFLAG="-g"
fi
echo "$CMDSTR output for condor"
for FILE in *.txt; do
	echo "${CMDSTR} -f ${FILE} ${OUTDIR}/${FILE_DST}"
	stageOut ${GFLAG} -x "-f" -i ${FILE} -o ${OUTDIR}/${FILE_DST} -r -c '*.txt'
	XRDEXIT=$?
	if [[ $XRDEXIT -ne 0 ]]; then
		echo "exit code $XRDEXIT, failure in $CMDSTR"
		exit $XRDEXIT
	fi
done