Merge branch 'cherry-pick-84abc09b [formerly 00520d021f]-21.0-mc16d' into '21.0-mc16d'

Sweeping !8064 from 21.0 to 21.0-mc16d.
fixes for ART

See merge request atlas/athena!8137

Former-commit-id: 95d2391fb3e7a9da1c664b3fe8fb2ed21c946b8b

Tools/ART/python/ART/art_configuration.py

@@ -85,6 +85,7 @@ class ArtConfiguration(object):
 
         """
         log = logging.getLogger(MODULE)
+        log.debug("Looking for %s %s %s %s %s", nightly_release, project, platform, package, key)
         if self.config is None:
             log.debug("No configuration")
             return default_value
@@ -108,6 +109,7 @@ class ArtConfiguration(object):
                 if key in release:
                     value = release[key]
 
+        log.debug("Value %s", value)
         return value
 
     def get_option(self, nightly_release, project, platform, package, key, option_key):

Tools/ART/python/ART/art_grid.py

@@ -404,7 +404,7 @@ class ArtGrid(ArtBase):
             cmd = ' '.join((cmd,
                             '--inds ' + inds if inds != '' else '',
                             '--n-files ' + str(n_files) if n_files > 0 else '',
-                            '--split ' + str(split) if split > 0 else ''
+                            '--split ' + str(split) if split > 0 else '',
                             '--in' if in_file else ''))
 
         cmd = ' '.join((cmd,
@@ -429,10 +429,13 @@ class ArtGrid(ArtBase):
 
         # run task from Bash Script as is needed in ATLAS setup
         # FIXME we need to parse the output
+        log.info("Grid_options: %s", grid_options)
         env = os.environ.copy()
         env['PATH'] = '.:' + env['PATH']
         env['ART_GRID_OPTIONS'] = grid_options
 
+        log.info("ART_GRID_OPTIONS %s", env['ART_GRID_OPTIONS'])
+
         jedi_id = -1
         # run the command, no_action is forwarded and used inside the script
         (exit_code, out, err) = run_command(cmd, env=env)
@@ -448,12 +451,14 @@ class ArtGrid(ArtBase):
 
     def get_grid_options(self, package, config_file):
         """Return grid options for a package."""
+        log = logging.getLogger(MODULE)
         if self.skip_setup:
             return ''
 
         configuration = ArtConfiguration(config_file)
         grid_options = configuration.get_option(self.nightly_release, self.project, self.platform, package, 'exclude-sites', '--excludedSite=')
         grid_options += ' ' + configuration.get_option(self.nightly_release, self.project, self.platform, package, 'sites', '--site=')
+        log.info('grid_options: %s', grid_options)
         return grid_options
 
     def task(self, script_directory, package, job_type, sequence_tag, no_action=False, config_file=None):

Tools/ART/python/ART/art_header.py

@@ -89,8 +89,7 @@ class ArtHeader(object):
                         # handle values
                         if key not in self.header:
                             log.warning("Unknown art-header %s: %s in file %s", key, value, filename)
-                            self.header[key] = {}
-                            self.header[key]['value'] = None
+                            self.add(key, StringType)
                         if self.header[key]['value'] is None:
                             self.header[key]['value'] = value
                         else:

Tools/ART/scripts/art-task-grid.sh

@@ -52,14 +52,19 @@ case ${TYPE} in
             shift
         fi
         NFILES=""
+        NFILES_PER_JOB=""
         if [ $1 == "--n-files" ]; then
-            NFILES="--nFiles $2 --nFilesPerJob $2"
+            NFILES="--nFiles $2"
+            NFILES_PER_JOB="--nFilesPerJob $2"
             shift
             shift
         fi
         SPLIT=""
+        LARGE_JOB="--long --memory 4096"
         if [ $1 == "--split" ]; then
             SPLIT="--split $2"
+            NFILES_PER_JOB=""
+            LARGE_JOB=""
             shift
             shift
         fi
@@ -116,6 +121,15 @@ case ${TYPE} in
         ;;
 esac
 
+# general options
+PATHENA_OPTIONS="--destSE=CERN-PROD_SCRATCHDISK"
+OUT="%OUT.tar"
+
+# we seem to have to copy the env variables locally
+GRID_OPTIONS=$ART_GRID_OPTIONS
+echo "GRID_OPTIONS=${GRID_OPTIONS}"
+
+
 if [ ${SKIP_SETUP} -eq 0 ]; then
     # maybe not necessary
     PLATFORM=${AtlasProject}_PLATFORM
@@ -131,19 +145,11 @@ if [ ${SKIP_SETUP} -eq 0 ]; then
     lsetup panda "asetup --platform=${!PLATFORM} ${AtlasBuildBranch},${AtlasBuildStamp},${AtlasProject}" || true
     echo "Setting up panda and release done"
 
-    voms-proxy-init --rfc -noregen -cert ./grid.proxy -voms atlas || true
+    voms-proxy-init --rfc -noregen -cert ./grid.proxy -voms atlas --valid 24:00 || true
     echo "Setting up proxy done"
 
 fi
 
-# general options
-PATHENA_OPTIONS="--destSE=CERN-PROD_SCRATCHDISK"
-OUT="%OUT.tar"
-
-# we seem to have to copy the env variables locally
-GRID_OPTIONS=$ART_GRID_OPTIONS
-echo "GRID_OPTIONS=${GRID_OPTIONS}"
-
 case ${TYPE} in
 
     'batch')
@@ -157,7 +163,7 @@ case ${TYPE} in
     'single')
         # <script_directory> <sequence_tag> <package> <outfile> <job_name>
         INTERNAL_COMMAND="grid single"
-        PATHENA_TYPE_OPTIONS="--long --memory 4096 ${INDS} ${NFILES}"
+        PATHENA_TYPE_OPTIONS="${LARGE_JOB} ${INDS} ${NFILES} ${NFILES_PER_JOB}"
         ARGS="${JOB_NAME}"
         echo "PATHENA_TYPE_OPTIONS=${PATHENA_TYPE_OPTIONS}"
         echo "ARGS=${ARGS}"
@@ -168,7 +174,7 @@ esac
 # NOTE: for art-internal.py the current dir can be used as it is copied there
 cd ${SUBMIT_DIRECTORY}/${PACKAGE}/run
 SUBCOMMAND="./art-internal.py ${INTERNAL_COMMAND} ${IN_FILE} ${SCRIPT_DIRECTORY} ${SEQUENCE_TAG} ${PACKAGE} ${OUT} ${ARGS}"
-CMD="pathena --disableAutoRetry ${GRID_OPTIONS} ${PATHENA_OPTIONS} ${PATHENA_TYPE_OPTIONS} --noBuild --expertOnly_skipScout --trf \"${SUBCOMMAND}\" ${SPLIT} --outDS ${OUTFILE} --extOutFile art-job.json"
+CMD="pathena ${GRID_OPTIONS} ${PATHENA_OPTIONS} ${PATHENA_TYPE_OPTIONS} --noBuild --expertOnly_skipScout --trf \"${SUBCOMMAND}\" ${SPLIT} --outDS ${OUTFILE} --extOutFile art-job.json"
 
 #--disableAutoRetry
 #--excludedSite=ANALY_TECHNION-HEP-CREAM

Tools/ART/scripts/art.py

@@ -73,7 +73,7 @@ Tests are called with:
 """
 
 __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
-__version__ = '0.7.11'
+__version__ = '0.7.21'
 
 import logging
 import os
@@ -93,12 +93,12 @@ MODULE = "art"
 
 
 @dispatch.on('compare', 'ref')
-def compare_ref(file_name, ref_file, **kwargs):
+def compare_ref(path, ref_path, **kwargs):
     """Compare the output of a job."""
     set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     entries = kwargs['entries']
-    exit(ArtBase(art_directory).compare_ref(file_name, ref_file, entries))
+    exit(ArtBase(art_directory).compare_ref(path, ref_path, entries))
 
 
 @dispatch.on('compare', 'grid')