Commit aa10d6e1 authored by Dan Guest's avatar Dan Guest
Browse files

Add more grid submit options for trigger jobs

In particular add the option to do a 'dry run' which doesn't submit anything
(to quickly debug what will happen) and an option to specify the files to run
over.
parent e7e1194f
......@@ -22,28 +22,51 @@ fi
#
P=10
#
DEFAULT_CONFIG=trigger.json
#
INPUT_DATASETS=(
# Samples from tuning campaign, see
#
# https://its.cern.ch/jira/browse/ATLMCPROD-9424
#
mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.recon.AOD.e6337_e5984_s3126_d1677_r12711_tid26386088_00
mc16_13TeV.427080.Pythia8EvtGen_A14NNPDF23LO_flatpT_Zprime.recon.AOD.e5362_e5984_s3126_d1677_r12711_tid26386090_00
)
function _help() {
cat <<EOF
usage: ${0##*/} [-h] [options] CONFIG_FILE
usage: ${0##*/} [-h] [options] [CONFIG_FILE=${DEFAULT_CONFIG}]
Submit a job to the grid! The CONFIG_FILE is required, and should live
Submit a job to the grid! The CONFIG_FILE is json, and should live
under /configs in this package. Note that this script will ask you to
commit your changes, but it's a good idea to tag them as well.
Options:
-f: force, don't require changes to be committed
-f: Force, don't require changes to be committed.
-p <n>: number of parallel submissions to run (default $P)
-d: Dry run, don't submit anything or make a tarball, but build the
submit directory.
-i: File listing input files, will override the default list.
With the current arguments, will run over:
EOF
local F
for F in ${INPUT_DATASETS[@]} ; do
echo $F
done
}
FORCE=''
while getopts ":hfp:" opt $@;
# this might be 'echo' to do a dry run
PRE=''
while getopts ":hfp:di:" opt $@;
do
case $opt in
h) _help; exit 1;;
f) FORCE=1 ;;
p) P=${OPTARG} ;;
d) PRE="echo DRY-RUNNING: " ;;
i) readarray -t INPUT_DATASETS < ${OPTARG} ;;
esac
done
shift $((OPTIND-1))
......@@ -56,23 +79,7 @@ shift $((OPTIND-1))
# Users's grid name
GRID_NAME=${RUCIO_ACCOUNT-${USER}}
#
DEFAULT_CONFIG=trigger.json
INPUT_DATASETS=(
# R22 validation samples from ATR-23543
#
# valid1.364703.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ3WithSW.recon.AOD.e7142_s3681_d1660_r12684
# valid1.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.AOD.e4993_s3214_d1656_r12684
# valid1.427080.Pythia8EvtGen_A14NNPDF23LO_flatpT_Zprime.recon.AOD.e5362_s3459_d1656_r12684
# mc16_valid.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.recon.AOD.e6337_s3227_d1671_r12782
#
# Samples from tuning campaign, see
#
# https://its.cern.ch/jira/browse/ATLMCPROD-9424
#
mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.recon.AOD.e6337_e5984_s3126_d1677_r12711_tid26386088_00
mc16_13TeV.427080.Pythia8EvtGen_A14NNPDF23LO_flatpT_Zprime.recon.AOD.e5362_e5984_s3126_d1677_r12711_tid26386090_00
)
######################################################
# Part 2: variables you probably don't have to change
......@@ -144,7 +151,7 @@ echo "making tarball of local files: ${ZIP}" >&2
# The --outTarBall, --noSubmit, and --useAthenaPackages arguments are
# important. The --outDS and --exec don't matter at all here, they are
# just placeholders to keep panda from complianing.
prun --outTarBall=${ZIP} --noSubmit --useAthenaPackages\
${PRE} prun --outTarBall=${ZIP} --noSubmit --useAthenaPackages\
--exec "ls"\
--outDS user.${GRID_NAME}.x
......@@ -184,7 +191,7 @@ function submit-job() (
# Now submit.
#
echo "Submitting for ${GRID_NAME} on ${DS} -> ${OUT_DS}"
prun --exec "${EXE} %IN -c ${CONFIG_FILE}"\
${PRE} prun --exec "${EXE} %IN -c ${CONFIG_FILE}"\
--outDS ${OUT_DS} --inDS ${DS}\
--useAthenaPackages --inTarBall=${ZIP}\
--mergeScript="hdf5-merge-nolock -o %OUT -i %IN"\
......@@ -196,7 +203,7 @@ function submit-job() (
# below
#
export -f submit-job
export CONFIG_PATH GRID_NAME GIT_TAG ZIP AtlasBuildStamp EXE
export CONFIG_PATH GRID_NAME GIT_TAG ZIP AtlasBuildStamp EXE PRE
#
# Use xargs to submit all these jobs in batches of $P
printf "%s\n" ${INPUT_DATASETS[*]} | xargs -P $P -I {} bash -c "submit-job {}"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment