From 3998b9b9466d17b2ee87021f0c07d408237b98e9 Mon Sep 17 00:00:00 2001
From: Tamara Vazquez Schroeder <tamara.vazquez.schroeder@cern.ch>
Date: Thu, 18 Jan 2018 12:46:41 +0000
Subject: [PATCH] Merge branch 'art-v0721' into '21.0'
fixes for ART
See merge request atlas/athena!8064
(cherry picked from commit 84abc09bdce6fd514592a939641ba7f443d9b0a6 [formerly 00520d021f8a3bb967109d6d813c8da9ea9c64f7])
8fad0901 fixes for ART
Former-commit-id: 6d92ad6328d09d527050d178164eb3992011b012
---
Tools/ART/python/ART/art_configuration.py | 2 ++
Tools/ART/python/ART/art_grid.py | 7 +++++-
Tools/ART/python/ART/art_header.py | 3 +--
Tools/ART/scripts/art-task-grid.sh | 30 ++++++++++++++---------
Tools/ART/scripts/art.py | 6 ++---
5 files changed, 30 insertions(+), 18 deletions(-)
diff --git a/Tools/ART/python/ART/art_configuration.py b/Tools/ART/python/ART/art_configuration.py
index 8f09e40d1a4..1e8910b62dd 100644
--- a/Tools/ART/python/ART/art_configuration.py
+++ b/Tools/ART/python/ART/art_configuration.py
@@ -85,6 +85,7 @@ class ArtConfiguration(object):
"""
log = logging.getLogger(MODULE)
+ log.debug("Looking for %s %s %s %s %s", nightly_release, project, platform, package, key)
if self.config is None:
log.debug("No configuration")
return default_value
@@ -108,6 +109,7 @@ class ArtConfiguration(object):
if key in release:
value = release[key]
+ log.debug("Value %s", value)
return value
def get_option(self, nightly_release, project, platform, package, key, option_key):
diff --git a/Tools/ART/python/ART/art_grid.py b/Tools/ART/python/ART/art_grid.py
index 25fa49b1d6c..a00e499f54b 100644
--- a/Tools/ART/python/ART/art_grid.py
+++ b/Tools/ART/python/ART/art_grid.py
@@ -404,7 +404,7 @@ class ArtGrid(ArtBase):
cmd = ' '.join((cmd,
'--inds ' + inds if inds != '' else '',
'--n-files ' + str(n_files) if n_files > 0 else '',
- '--split ' + str(split) if split > 0 else ''
+ '--split ' + str(split) if split > 0 else '',
'--in' if in_file else ''))
cmd = ' '.join((cmd,
@@ -429,10 +429,13 @@ class ArtGrid(ArtBase):
# run task from Bash Script as is needed in ATLAS setup
# FIXME we need to parse the output
+ log.info("Grid_options: %s", grid_options)
env = os.environ.copy()
env['PATH'] = '.:' + env['PATH']
env['ART_GRID_OPTIONS'] = grid_options
+ log.info("ART_GRID_OPTIONS %s", env['ART_GRID_OPTIONS'])
+
jedi_id = -1
# run the command, no_action is forwarded and used inside the script
(exit_code, out, err) = run_command(cmd, env=env)
@@ -448,12 +451,14 @@ class ArtGrid(ArtBase):
def get_grid_options(self, package, config_file):
"""Return grid options for a package."""
+ log = logging.getLogger(MODULE)
if self.skip_setup:
return ''
configuration = ArtConfiguration(config_file)
grid_options = configuration.get_option(self.nightly_release, self.project, self.platform, package, 'exclude-sites', '--excludedSite=')
grid_options += ' ' + configuration.get_option(self.nightly_release, self.project, self.platform, package, 'sites', '--site=')
+ log.info('grid_options: %s', grid_options)
return grid_options
def task(self, script_directory, package, job_type, sequence_tag, no_action=False, config_file=None):
diff --git a/Tools/ART/python/ART/art_header.py b/Tools/ART/python/ART/art_header.py
index 6b43ba9b9c7..61346f2cf2c 100644
--- a/Tools/ART/python/ART/art_header.py
+++ b/Tools/ART/python/ART/art_header.py
@@ -89,8 +89,7 @@ class ArtHeader(object):
# handle values
if key not in self.header:
log.warning("Unknown art-header %s: %s in file %s", key, value, filename)
- self.header[key] = {}
- self.header[key]['value'] = None
+ self.add(key, StringType)
if self.header[key]['value'] is None:
self.header[key]['value'] = value
else:
diff --git a/Tools/ART/scripts/art-task-grid.sh b/Tools/ART/scripts/art-task-grid.sh
index 86c13313771..807cbe82109 100755
--- a/Tools/ART/scripts/art-task-grid.sh
+++ b/Tools/ART/scripts/art-task-grid.sh
@@ -52,14 +52,19 @@ case ${TYPE} in
shift
fi
NFILES=""
+ NFILES_PER_JOB=""
if [ $1 == "--n-files" ]; then
- NFILES="--nFiles $2 --nFilesPerJob $2"
+ NFILES="--nFiles $2"
+ NFILES_PER_JOB="--nFilesPerJob $2"
shift
shift
fi
SPLIT=""
+ LARGE_JOB="--long --memory 4096"
if [ $1 == "--split" ]; then
SPLIT="--split $2"
+ NFILES_PER_JOB=""
+ LARGE_JOB=""
shift
shift
fi
@@ -116,6 +121,15 @@ case ${TYPE} in
;;
esac
+# general options
+PATHENA_OPTIONS="--destSE=CERN-PROD_SCRATCHDISK"
+OUT="%OUT.tar"
+
+# we seem to have to copy the env variables locally
+GRID_OPTIONS=$ART_GRID_OPTIONS
+echo "GRID_OPTIONS=${GRID_OPTIONS}"
+
+
if [ ${SKIP_SETUP} -eq 0 ]; then
# maybe not necessary
PLATFORM=${AtlasProject}_PLATFORM
@@ -131,19 +145,11 @@ if [ ${SKIP_SETUP} -eq 0 ]; then
lsetup panda "asetup --platform=${!PLATFORM} ${AtlasBuildBranch},${AtlasBuildStamp},${AtlasProject}" || true
echo "Setting up panda and release done"
- voms-proxy-init --rfc -noregen -cert ./grid.proxy -voms atlas || true
+ voms-proxy-init --rfc -noregen -cert ./grid.proxy -voms atlas --valid 24:00 || true
echo "Setting up proxy done"
fi
-# general options
-PATHENA_OPTIONS="--destSE=CERN-PROD_SCRATCHDISK"
-OUT="%OUT.tar"
-
-# we seem to have to copy the env variables locally
-GRID_OPTIONS=$ART_GRID_OPTIONS
-echo "GRID_OPTIONS=${GRID_OPTIONS}"
-
case ${TYPE} in
'batch')
@@ -157,7 +163,7 @@ case ${TYPE} in
'single')
# <script_directory> <sequence_tag> <package> <outfile> <job_name>
INTERNAL_COMMAND="grid single"
- PATHENA_TYPE_OPTIONS="--long --memory 4096 ${INDS} ${NFILES}"
+ PATHENA_TYPE_OPTIONS="${LARGE_JOB} ${INDS} ${NFILES} ${NFILES_PER_JOB}"
ARGS="${JOB_NAME}"
echo "PATHENA_TYPE_OPTIONS=${PATHENA_TYPE_OPTIONS}"
echo "ARGS=${ARGS}"
@@ -168,7 +174,7 @@ esac
# NOTE: for art-internal.py the current dir can be used as it is copied there
cd ${SUBMIT_DIRECTORY}/${PACKAGE}/run
SUBCOMMAND="./art-internal.py ${INTERNAL_COMMAND} ${IN_FILE} ${SCRIPT_DIRECTORY} ${SEQUENCE_TAG} ${PACKAGE} ${OUT} ${ARGS}"
-CMD="pathena --disableAutoRetry ${GRID_OPTIONS} ${PATHENA_OPTIONS} ${PATHENA_TYPE_OPTIONS} --noBuild --expertOnly_skipScout --trf \"${SUBCOMMAND}\" ${SPLIT} --outDS ${OUTFILE} --extOutFile art-job.json"
+CMD="pathena ${GRID_OPTIONS} ${PATHENA_OPTIONS} ${PATHENA_TYPE_OPTIONS} --noBuild --expertOnly_skipScout --trf \"${SUBCOMMAND}\" ${SPLIT} --outDS ${OUTFILE} --extOutFile art-job.json"
#--disableAutoRetry
#--excludedSite=ANALY_TECHNION-HEP-CREAM
diff --git a/Tools/ART/scripts/art.py b/Tools/ART/scripts/art.py
index 254fecacd06..7cc3486297d 100755
--- a/Tools/ART/scripts/art.py
+++ b/Tools/ART/scripts/art.py
@@ -73,7 +73,7 @@ Tests are called with:
"""
__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
-__version__ = '0.7.11'
+__version__ = '0.7.21'
import logging
import os
@@ -93,12 +93,12 @@ MODULE = "art"
@dispatch.on('compare', 'ref')
-def compare_ref(file_name, ref_file, **kwargs):
+def compare_ref(path, ref_path, **kwargs):
"""Compare the output of a job."""
set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
entries = kwargs['entries']
- exit(ArtBase(art_directory).compare_ref(file_name, ref_file, entries))
+ exit(ArtBase(art_directory).compare_ref(path, ref_path, entries))
@dispatch.on('compare', 'grid')
--
GitLab