Merge branch 'cherry-pick-4bb12250-5' into '21.0-mc16d'

Merge branch 'art-v0.6.5' into '21.0-mc16d'

See merge request atlas/athena!6855

Tools/ART/python/ART/art_base.py

@@ -6,13 +6,20 @@ __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
 
 import fnmatch
 import inspect
+import logging
 import os
-import sys
 import yaml
 
+try:
+    import scandir as scan
+except ImportError:
+    import os as scan
+
 from art_misc import is_exe, run_command
 from art_header import ArtHeader
 
+MODULE = "art.base"
+
 
 class ArtBase(object):
     """TBD."""
@@ -51,26 +58,37 @@ class ArtBase(object):
 
     def validate(self, script_directory):
         """TBD."""
+        log = logging.getLogger(MODULE)
         directories = self.get_test_directories(script_directory.rstrip("/"))
+
+        found_test = False
         for directory in directories.itervalues():
             files = self.get_files(directory)
             for fname in files:
                 test_name = os.path.join(directory, fname)
-                print test_name
+                found_test = True
+                log.debug(test_name)
                 if not is_exe(test_name):
-                    print "ERROR: ", test_name, "is not executable."
+                    log.error("%s is not executable.", test_name)
                 ArtHeader(test_name).validate()
+
+        if not found_test:
+            log.warning('No scripts found in %s directory', directories.values()[0])
+            return 0
+
+        log.info("Scripts in %s directory are validated", script_directory)
         return 0
 
     def included(self, script_directory, job_type, index_type, nightly_release, project, platform):
         """TBD."""
+        log = logging.getLogger(MODULE)
         directories = self.get_test_directories(script_directory.rstrip("/"))
         for directory in directories.itervalues():
             files = self.get_files(directory, job_type, index_type)
             for fname in files:
                 test_name = os.path.join(directory, fname)
                 if self.is_included(test_name, nightly_release, project, platform):
-                    print test_name, ArtHeader(test_name).get('art-include')
+                    log.info("%s %s", test_name, ArtHeader(test_name).get('art-include'))
         return 0
 
     def download(self, input_file):
@@ -84,6 +102,8 @@ class ArtBase(object):
         """TBD."""
         import PyUtils.PoolFile as PF
 
+        log = logging.getLogger(MODULE)
+
         # diff-pool
         df = PF.DiffFiles(refFileName=ref_file, chkFileName=file_name, ignoreList=['RecoTimingObj_p1_RAWtoESD_timings', 'RecoTimingObj_p1_ESDtoAOD_timings'])
         df.printSummary()
@@ -94,22 +114,24 @@ class ArtBase(object):
         # diff-root
         (code, out, err) = run_command("acmd.py diff-root " + file_name + " " + ref_file + " --error-mode resilient --ignore-leaves RecoTimingObj_p1_HITStoRDO_timings RecoTimingObj_p1_RAWtoESD_mems RecoTimingObj_p1_RAWtoESD_timings RAWtoESD_mems RAWtoESD_timings ESDtoAOD_mems ESDtoAOD_timings HITStoRDO_timings RAWtoALL_mems RAWtoALL_timings RecoTimingObj_p1_RAWtoALL_mems RecoTimingObj_p1_RAWtoALL_timings RecoTimingObj_p1_EVNTtoHITS_timings --entries " + str(entries))
         if code != 0:
-            print "Error:", code
-            print "StdErr:", err
+            log.error("Error: %d", code)
+            print(err)
 
-        print out
-        sys.stdout.flush()
-        return err
+        log.info(out)
+        return code
 
     #
     # Protected Methods
     #
     def get_config(self):
-        """Retrieve dictionary of ART configuration file."""
-        config_file = open("art-configuration.yml", "r")
-        config = yaml.load(config_file)
-        config_file.close()
-        return config
+        """Retrieve dictionary of ART configuration file, or None if file does not exist."""
+        try:
+            config_file = open("art-configuration.yml", "r")
+            config = yaml.load(config_file)
+            config_file.close()
+            return config
+        except IOError:
+            return None
 
     def get_files(self, directory, job_type=None, index_type="all", nightly_release=None, project=None, platform=None):
         """
@@ -163,7 +185,7 @@ class ArtBase(object):
         A dictionary key=<package>, value=<directory> is returned
         """
         result = {}
-        for root, dirs, files in os.walk(directory):
+        for root, dirs, files in scan.walk(directory):
             if root.endswith('/test'):
                 package = os.path.basename(os.path.dirname(root))
                 result[package] = root

Tools/ART/python/ART/art_build.py

@@ -7,6 +7,7 @@ __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
 import collections
 import fnmatch
 import json
+import logging
 import multiprocessing
 import os
 import re
@@ -17,12 +18,16 @@ from art_header import ArtHeader
 
 from parallelScheduler import ParallelScheduler
 
+MODULE = "art.build"
+
 
 def run_job(art_directory, sequence_tag, script_directory, package, job_type, index, test_name, nightly_release, project, platform, nightly_tag):
     """TBD."""
-    print "job started", art_directory, sequence_tag, script_directory, package, job_type, index, test_name, nightly_release, project, platform, nightly_tag
+    log = logging.getLogger(MODULE)
+
+    log.info("job started %s %s %s %s %s %d %s %s %s %s %s", art_directory, sequence_tag, script_directory, package, job_type, index, test_name, nightly_release, project, platform, nightly_tag)
     (exit_code, out, err) = run_command(' '.join((os.path.join(art_directory, './art-internal.py'), "job", "build", script_directory, package, job_type, sequence_tag, str(index), "out", nightly_release, project, platform, nightly_tag)))
-    print "job ended", art_directory, sequence_tag, script_directory, package, job_type, index, test_name, nightly_release, project, platform, nightly_tag
+    log.info("job ended %s %s %s %s %s %d %s %s %s %s %s", art_directory, sequence_tag, script_directory, package, job_type, index, test_name, nightly_release, project, platform, nightly_tag)
 
     return (test_name, exit_code, out, err)
 
@@ -33,7 +38,8 @@ class ArtBuild(ArtBase):
     def __init__(self, art_directory, nightly_release, project, platform, nightly_tag, script_directory, max_jobs=0, ci=False):
         """TBD."""
         super(ArtBuild, self).__init__(art_directory)
-        # print "ArtBuild", art_directory, script_directory, max_jobs
+        log = logging.getLogger(MODULE)
+        log.debug("ArtBuild %s %s %d", art_directory, script_directory, max_jobs)
         self.art_directory = art_directory
         self.script_directory = script_directory.rstrip("/")
         self.nightly_release = nightly_release
@@ -45,10 +51,11 @@ class ArtBuild(ArtBase):
 
     def task_list(self, job_type, sequence_tag):
         """TBD."""
-        # print "task_list", job_type, sequence_tag
+        log = logging.getLogger(MODULE)
+        log.debug("task_list %s %s", job_type, sequence_tag)
         test_directories = self.get_test_directories(self.script_directory)
         if not test_directories:
-            print 'WARNING: No tests found in directories ending in "test"'
+            log.warning('No tests found in directories ending in "test"')
 
         status = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.defaultdict()))
 
@@ -81,7 +88,8 @@ class ArtBuild(ArtBase):
 
     def task(self, package, job_type, sequence_tag):
         """TBD."""
-        # print "task", package, job_type, sequence_tag
+        log = logging.getLogger(MODULE)
+        log.debug("task %s %s %s", package, job_type, sequence_tag)
         test_names = self.get_list(self.script_directory, package, job_type, "all")
         scheduler = ParallelScheduler(self.max_jobs + 1)
 
@@ -101,7 +109,7 @@ class ArtBuild(ArtBase):
 
             if not os.access(fname, os.X_OK):
                 schedule_test = False
-                print "job skipped, file not executable: ", fname
+                log.warning("job skipped, file not executable: %s", fname)
 
             if schedule_test:
                 scheduler.add_task(task_name="t" + str(index), dependencies=[], description="d", target_function=run_job, function_kwargs={'art_directory': self.art_directory, 'sequence_tag': sequence_tag, 'script_directory': self.script_directory, 'package': package, 'job_type': job_type, 'index': index, 'test_name': test_name, 'nightly_release': self.nightly_release, 'project': self.project, 'platform': self.platform, 'nightly_tag': self.nightly_tag})
@@ -112,7 +120,8 @@ class ArtBuild(ArtBase):
 
     def job(self, package, job_type, sequence_tag, index, out):
         """TBD."""
-        # print "job", package, job_type, sequence_tag, index, out
+        log = logging.getLogger(MODULE)
+        log.debug("job %s %s %s %d %s", package, job_type, sequence_tag, index, out)
         test_directories = self.get_test_directories(self.script_directory)
         test_directory = os.path.abspath(test_directories[package])
         test_name = self.get_files(test_directory, job_type)[int(index)]

Tools/ART/python/ART/art_header.py

@@ -4,12 +4,15 @@
 
 __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
 
+import logging
 import re
 
 from types import IntType
 from types import ListType
 from types import StringType
 
+MODULE = "art.header"
+
 
 class ArtHeader(object):
     """TBD."""
@@ -36,9 +39,7 @@ class ArtHeader(object):
         self.add('art-include', ListType, ['*'])
         self.add('art-output', ListType, [])
         self.add('art-input', StringType, None)
-        self.add('art-input-file', ListType, [])
-        self.add('art-input-nfiles', IntType, 0)
-        self.add('art-input-nevents', IntType, 0)
+        self.add('art-input-nfiles', IntType, 1)
         self.add('art-input-split', IntType, 0)
 
         self.read(filename)
@@ -49,7 +50,7 @@ class ArtHeader(object):
         self.header[key]['type'] = value_type
         self.header[key]['default'] = default_value
         self.header[key]['constraint'] = constraint
-        self.header[key]['value'] = None    # never set
+        self.header[key]['value'] = None    # e.g. the value was never set
 
     def is_list(self, key):
         """TBD."""
@@ -57,26 +58,38 @@ class ArtHeader(object):
 
     def read(self, filename):
         """Read all headers from file."""
+        log = logging.getLogger(MODULE)
         for line in open(filename, "r"):
             line_match = self.header_format.match(line)
             if line_match:
-                key = line_match.group(1)
-                value = line_match.group(2)
-                if key in self.header and self.header[key]['type'] == StringType:
-                    value = value.strip()
-
-                if self.is_list(key):
-                    if self.header[key]['value'] is None:
-                        self.header[key]['value'] = []
-                    self.header[key]['value'].append(value)
-                else:
-                    if key not in self.header:
-                        self.header[key] = {}
-                    self.header[key]['value'] = value
+                try:
+                    key = line_match.group(1)
+                    value = line_match.group(2)
+                    if key in self.header:
+                        if self.header[key]['type'] == StringType:
+                            value = value.strip()
+                        elif self.header[key]['type'] == IntType:
+                            value = int(value)
+
+                    if self.is_list(key):
+                        # handle list types
+                        if self.header[key]['value'] is None:
+                            self.header[key]['value'] = []
+                        self.header[key]['value'].append(value)
+                    else:
+                        # handle values
+                        if key not in self.header:
+                            log.warning("Unknown art-header %s: %s in file %s", key, value, filename)
+                            self.header[key] = {}
+                        self.header[key]['value'] = value
+                except ValueError:
+                    log.error("Invalid value in art-header %s: %s in file %s", key, value, filename)
 
     def get(self, key):
         """TBD."""
+        log = logging.getLogger(MODULE)
         if key not in self.header:
+            log.warning("Art seems to look for a header key %s which is not in the list of defined headers.", key)
             return None
 
         if self.header[key]['value'] is None:
@@ -86,8 +99,9 @@ class ArtHeader(object):
 
     def print_it(self):
         """TBD."""
+        log = logging.getLogger(MODULE)
         for key in self.header:
-            print key, self.header[key]['type'], self.header[key]['default'], self.header[key]['value'], self.header[key]['constraint']
+            log.info("%s: %s %s %s %s", key, self.header[key]['type'], self.header[key]['default'], self.header[key]['value'], self.header[key]['constraint'])
 
     def validate(self):
         """
@@ -100,34 +114,35 @@ class ArtHeader(object):
         - a value is found of the wrong value_type
         - a value is found outside the constraint
         """
+        log = logging.getLogger(MODULE)
         for line in open(self.filename, "r"):
             if self.header_format_error1.match(line):
-                print "LINE: ", line.rstrip()
-                print "ERROR: Header Validation - invalid header format, use space between '# and art-xxx' in file", self.filename
-                print
+                log.error("LINE: %s", line.rstrip())
+                log.error("Header Validation - invalid header format, use space between '# and art-xxx' in file %s", self.filename)
+                log.error("")
             if self.header_format_error2.match(line):
-                print "LINE: ", line.rstrip()
-                print "ERROR: Header Validation - invalid header format, too many spaces between '# and art-xxx' in file", self.filename
-                print
+                log.error("LINE: %s", line.rstrip())
+                log.error("Header Validation - invalid header format, too many spaces between '# and art-xxx' in file %s", self.filename)
+                log.error("")
             if self.header_format_error3.match(line):
-                print "LINE: ", line.rstrip()
-                print "ERROR: Header Validation - invalid header format, use at least one space between ': and value' in file", self.filename
-                print
+                log.error("LINE: %s", line.rstrip())
+                log.error("Header Validation - invalid header format, use at least one space between ': and value' in file %s", self.filename)
+                log.error("")
 
         for key in self.header:
             if 'type' not in self.header[key]:
-                print "ERROR: Header Validation - Invalid key:", key, "in file", self.filename
-                print
+                log.error("Header Validation - Invalid key: %s in file %s", key, self.filename)
+                log.error("")
                 continue
             if type(self.header[key]['value']) != self.header[key]['type']:
                 if not isinstance(self.header[key]['value'], type(None)):
-                    print "ERROR: Header Validation - value_type:", type(self.header[key]['value']), "not valid for key:", key, "expected value_type:", self.header[key]['type'], "in file", self.filename
-                    print
+                    log.error("Header Validation - value_type: %s not valid for key: %s, expected value_type: %s in file %s", type(self.header[key]['value']), key, self.header[key]['type'], self.filename)
+                    log.error("")
             if self.header[key]['constraint'] is not None and self.header[key]['value'] not in self.header[key]['constraint']:
                 if self.header[key]['value'] is None:
-                    print "ERROR: Header Validation - missing key:", key, "in file", self.filename
+                    log.error("Header Validation - missing key: %s in file %s", key, self.filename)
                 else:
-                    print "ERROR: Header Validation - value:", self.header[key]['value'], "for key:", key, "not in constraints:", self.header[key]['constraint'], "in file", self.filename
-                print
+                    log.error("Header Validation - value: %s for key: %s not in constraints: %s in file %s", self.header[key]['value'], key, self.header[key]['constraint'], self.filename)
+                log.error("")
 
         return 0

Tools/ART/python/ART/art_misc.py

@@ -5,14 +5,40 @@
 __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
 
 import errno
+import logging
 import os
 import shlex
 import subprocess
+import sys
+
+MODULE = "art.misc"
+
+
+def set_log(kwargs):
+    """TBD."""
+    level = logging.DEBUG if kwargs['verbose'] else logging.WARN if kwargs['quiet'] else logging.INFO
+    log = logging.getLogger("art")
+    log.setLevel(level)
+
+    # create and attach new handler, disable propagation to root logger to avoid double messages
+    handler = logging.StreamHandler(sys.stdout)
+    format_string = "%(asctime)s %(name)15s.%(funcName)-15s %(levelname)8s %(message)s"
+    date_format_string = None
+    formatter = logging.Formatter(format_string, date_format_string)
+    handler.setFormatter(formatter)
+    log.addHandler(handler)
+    log.propagate = False
 
 
 def run_command(cmd, dir=None, shell=False, env=None):
-    """Run the given command locally and returns the output, err and exit_code."""
-    # print "Execute: " + cmd
+    """
+    Run the given command locally.
+
+    The command runs as separate subprocesses for every piped command.
+    Returns tuple of exit_code, output and err.
+    """
+    log = logging.getLogger(MODULE)
+    log.debug("Execute: %s", cmd)
     if "|" in cmd:
         cmd_parts = cmd.split('|')
     else:
@@ -33,39 +59,6 @@ def run_command(cmd, dir=None, shell=False, env=None):
     return exit_code, str(output), str(err)
 
 
-def exit_on_failure((exit_code, out, err)):
-    """Check exitcode and print statement and exit if needed."""
-    if exit_code == 0:
-        print err
-        return out
-
-    print "Error:", exit_code
-    print "StdOut:", out
-    print "StdErr:", err
-
-    print 'art-status: error'
-
-    exit(exit_code)
-
-
-def code_tbr((exit_code, out, err)):
-    """Check exitcode and print statement."""
-    if exit_code == 0:
-        print out
-        return exit_code
-
-    print "Error:", exit_code
-    print "StdOut:", out
-    print "StdErr:", err
-
-    return exit_code
-
-
-def redirect_tbr((exitcode, out, err)):
-    """Check exitcode."""
-    return exitcode
-
-
 def is_exe(fpath):
     """Return True if fpath is executable."""
     return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

Tools/ART/scripts/art-get-tar.sh

@@ -2,14 +2,14 @@
 # Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
 #
 # NOTE do NOT run with /bin/bash -x as the output is too big for gitlab-ci
-# arguments:  PACKAGE INDEX EXTENSION NIGHTLY_RELEASE PROJECT PLATFORM NIGHTLY_TAG
+# arguments:  PACKAGE GRID_INDEX EXTENSION NIGHTLY_RELEASE PROJECT PLATFORM NIGHTLY_TAG
 #
 # author : Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>
 #
 # example: Tier0ChainTests 4 _EXT0 21.0 Athena x86_64-slc6-gcc62-opt 2017-07-24T2151
 
 if [ $# -ne 7 ]; then
-    echo 'Usage: art-get-tar.sh PACKAGE INDEX EXTENSION NIGHTLY_RELEASE PROJECT PLATFORM NIGHTLY_TAG'
+    echo 'Usage: art-get-tar.sh PACKAGE GRID_INDEX EXTENSION NIGHTLY_RELEASE PROJECT PLATFORM NIGHTLY_TAG'
     exit 1
 fi
 
@@ -19,7 +19,7 @@ ART_USER='artprod'
 
 PACKAGE=$1
 shift
-INDEX=$1
+GRID_INDEX=$1
 shift
 EXTENSION=$1
 shift
@@ -54,7 +54,7 @@ echo "Tar files in the Container: ${FILELIST}"
 CONTAINER=`rucio list-dids ${CONTAINER_LIST} --filter type=container | grep ${NIGHTLY_TAG} | sort -r | cut -d ' ' -f 2 | head -n 1`
 echo "Container: ${CONTAINER}"
 
-printf -v INDEX_FORMAT '_%06d.tar' ${INDEX}
+printf -v INDEX_FORMAT '_%06d.tar' ${GRID_INDEX}
 TAR_NAME=`rucio list-files --csv ${CONTAINER} | grep ${INDEX_FORMAT} | cut -d ',' -f 1`
 echo "Tar Name: ${TAR_NAME}"
 

Tools/ART/scripts/art-internal.py

@@ -4,20 +4,19 @@
 ART-internal - ATLAS Release Tester (internal command).
 
 Usage:
-  art-internal.py job build   [-v]  <script_directory> <package> <job_type> <sequence_tag> <index> <out> <nightly_release> <project> <platform> <nightly_tag>
-  art-internal.py job grid    [-v --skip-setup]  <script_directory> <package> <job_type> <sequence_tag> <index_type> <index_or_name> <out> <nightly_release> <project> <platform> <nightly_tag>
-  art-internal.py task build  [-v]  <script_directory> <package> <job_type> <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
-  art-internal.py task grid   [-v --skip-setup]  <submit_directory> <script_directory> <package> <job_type> <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
+  art-internal.py job build   [-v -q]  <script_directory> <package> <job_type> <sequence_tag> <index> <out> <nightly_release> <project> <platform> <nightly_tag>
+  art-internal.py job grid    [-v -q --skip-setup]  <script_directory> <package> <job_type> <sequence_tag> <index_type> <index_or_name> <out> <nightly_release> <project> <platform> <nightly_tag>
 
 Options:
   --skip-setup      Do not run atlas setup or voms
   -h --help         Show this screen.
-  -v, --verbose     Show details.
+  -q --quiet        Show less information, only warnings and errors
+  -v --verbose      Show more information, debug level
   --version         Show version.
 
 Sub-commands:
-  job               Runs a single job, given a particular index
-  task              Runs a single task, consisting of given number of jobs
+  job               Run a single job, given a particular index
+  copy              Copy outputs to eos area
 
 Arguments:
   index_type        Type of index used (e.g. batch or single)
@@ -37,6 +36,7 @@ Arguments:
 
 __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
 
+import logging
 import os
 import sys
 
@@ -44,47 +44,45 @@ from ART.docopt_dispatch import dispatch
 
 from ART import ArtGrid, ArtBuild
 
+from ART.art_misc import set_log
+
+MODULE = "art.internal"
+
 
 @dispatch.on('job', 'build')
 def job_build(script_directory, package, job_type, sequence_tag, index, out, nightly_release, project, platform, nightly_tag, **kwargs):
-    """TBD.
+    """Run a single job, given a particular index.
 
     Tests are called with the following parameters:
     SCRIPT_DIRECTORY, PACKAGE, TYPE, TEST_NAME
     """
+    set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     exit(ArtBuild(art_directory, nightly_release, project, platform, nightly_tag, script_directory).job(package, job_type, sequence_tag, index, out))
 
 
 @dispatch.on('job', 'grid')
 def job_grid(script_directory, package, job_type, sequence_tag, index_type, index_or_name, out, nightly_release, project, platform, nightly_tag, **kwargs):
-    """TBD.
+    """Run a single job, given a particular index.
 
     Tests are called with the following parameters:
     SCRIPT_DIRECTORY, PACKAGE, TYPE, TEST_NAME, NIGHTLY_RELEASE, PROJECT, PLATFORM, NIGHTLY_TAG
     """
+    set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     skip_setup = kwargs['skip_setup']
     exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, skip_setup).job(package, job_type, sequence_tag, index_type, index_or_name, out))
 
 
-@dispatch.on('task', 'build')
-def task_build(script_directory, package, job_type, sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
-    """TBD."""
-    art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
-    exit(ArtBuild(art_directory, nightly_release, project, platform, nightly_tag, script_directory).task(package, job_type, sequence_tag))
-
-
-@dispatch.on('task', 'grid')
-def task_grid(submit_directory, script_directory, package, job_type, sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
-    """TBD."""
-    art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
-    skip_setup = kwargs['skip_setup']
-    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, skip_setup, submit_directory).task(package, job_type, sequence_tag))
-
-
 if __name__ == '__main__':
+    if sys.version_info < (2, 7, 0):
+        sys.stderr.write("You need python 2.7 or later to run this script\n")
+        exit(1)
+
     # NOTE: import should be here, to keep the order of the decorators (module first, art last and unused)
     from art import __version__
-    print "ART_PATH", os.path.dirname(os.path.realpath(sys.argv[0]))
+    logging.basicConfig()
+    log = logging.getLogger(MODULE)
+    log.setLevel(logging.INFO)
+    log.info("ART_PATH %s", os.path.dirname(os.path.realpath(sys.argv[0])))
     dispatch(__doc__, version=os.path.splitext(os.path.basename(__file__))[0] + ' ' + __version__)

Tools/ART/scripts/art-share.py

+#!/usr/bin/env python
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+"""
+ART  - ATLAS Release Tester - Share.
+
+Usage:
+  art-share.py [options] <data_directory>
+
+Options:
+  -h --help               Show this screen
+  --version               Show version
+  --q --quiet             Show less information, only warnings and errors
+  --write                 Write to directory
+  -v --verbose            Show more information, debug level
+
+Arguments:
+  data_directory          directory to scan for shared files
+
+"""
+
+__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
+
+import collections
+import hashlib
+import logging
+import os
+import sys
+
+try:
+    import scandir as scan
+except ImportError:
+    import os as scan
+
+from ART.docopt import docopt
+
+MODULE = "art.share"
+
+
+class ArtShare(object):
+    """Class for copying input files.
+
+    Files are copies to the .art area under their SHA1 sum. The orignal file is replaced
+    by a symbolic link. Any duplicates will result in the same SHA1 sum and thus just
+    be replaced by their symbiolic link. Removing a file results in removing a link.
+    If the .art directory has files without links pointing to them, these files are also
+    removed.
+    """
+
+    def __init__(self, data_directory, write):
+        """Constructor of ArtShare."""
+        log = logging.getLogger(MODULE)
+        self.data_directory = data_directory
+        self.write = write
+
+        self.art_directory = '.art'
+        self.extension = '.art'
+        self.count = collections.defaultdict(int)   # key is sha1
+
+        if not os.path.isdir(self.data_directory):
+            log.critical("data_directory does not exist: %s", self.data_directory)
+            sys.exit(1)
+
+        if self.write:
+            log.warning("NOTE - Changing File System")
+        else:
+            log.warning("NOT Changing File System, use --write to change the File System")
+
+        self.share()
+
+    def create_sha1sum(self, path):
+        """Calculate SHA1 from file on path."""
+        BUF_SIZE = 65536
+        sha1 = hashlib.sha1()
+        with open(path, 'rb') as f:
+            while True:
+                data = f.read(BUF_SIZE)
+                if not data:
+                    break
+                sha1.update(data)
+        return sha1.hexdigest()
+
+    def sha1sum(self, artpath):
+        """Retrieve SHA1 from artpath specification (in the filename)."""
+        f = os.path.basename(artpath)
+        return os.path.splitext(f)[0]
+
+    def share(self):
+        """Share the files by copying."""
+        log = logging.getLogger(MODULE)
+        art_root = os.path.join(self.data_directory, self.art_directory)
+        if not os.path.isdir(art_root):
+            log.info("NOTE - art_directory does not exist.")
+            log.info("       creating... %s", art_root)
+            if self.write:
+                os.makedirs(art_root)
+
+        if os.path.isdir(art_root):
+            for f in os.listdir(art_root):
+                sha1art = os.path.join(art_root, f)
+                if os.path.isfile(sha1art):
+                    sha1 = self.sha1sum(sha1art)
+                    self.count[sha1] = 0
+
+        i = 0
+        for root, dirs, files in scan.walk(self.data_directory):
+            for f in files:
+                if os.path.basename(root) == self.art_directory:
+                    continue
+
+                path = os.path.join(root, f)
+                i += 1
+                if os.path.islink(path):
+                    # link
+                    if not os.path.exists(path):
+                        log.warning("WARNING - Stale link/file, skipping")
+                        log.warning("          path: %s", path)
+                        continue
+                    sha1 = self.sha1sum(os.path.realpath(path))
+                    log.debug("Link %d path %s", i, path)
+                    log.debug("SHA1 %s", sha1)
+                    self.count[sha1] += 1
+                else:
+                    # file
+                    byte_size = os.path.getsize(path)
+                    if byte_size <= 0:
+                        log.warning("WARNING - zero sized file, skipping")
+                        log.warning("          path: %s", path)
+                        continue
+
+                    megabyte_size = byte_size / 1024 / 1024
+                    log.debug("File %d %s", i, path)
+                    log.debug("File size %d", megabyte_size)
+                    sha1 = self.create_sha1sum(path) if self.write or megabyte_size < 100 else "????????????????????????????????????????"
+                    log.debug("SHA1 %s", sha1)
+
+                    art_path = os.path.join(art_root, sha1 + self.extension)
+                    art_relpath = os.path.relpath(art_path, os.path.dirname(path))
+
+                    if sha1 not in self.count.keys():
+                        log.info("    Moving file from %s", path)
+                        log.info("                  to %s", art_path)
+                        if self.write:
+                            os.rename(path, art_path)
+                        self.count[sha1] = 0
+                    else:
+                        log.info("    Removing file from %s", path)
+                        if self.write:
+                            os.remove(path)
+
+                    log.info("    Creating link from %s", path)
+                    log.info("                    to %s", art_relpath)
+                    if self.write:
+                        os.symlink(art_relpath, path)
+                    self.count[sha1] += 1
+
+        for sha1, count in self.count.iteritems():
+            if count <= 0:
+                art_path = os.path.join(art_root, sha1 + self.extension)
+                log.info("    Removing file  %s", art_path)
+                if self.write:
+                    os.remove(art_path)
+
+
+if __name__ == '__main__':
+    if sys.version_info < (2, 7, 0):
+        sys.stderr.write("You need python 2.7 or later to run this script\n")
+        exit(1)
+
+    # NOTE: import should be here, to keep the order of the decorators (module first, art last and unused)
+    from art import __version__
+
+    logging.basicConfig()
+    log = logging.getLogger(MODULE)
+
+    arguments = docopt(__doc__, version=os.path.splitext(os.path.basename(__file__))[0] + ' ' + __version__)
+    level = logging.DEBUG if arguments['--verbose'] else logging.WARN if arguments['--quiet'] else logging.INFO
+    log.setLevel(level)
+    ArtShare(arguments['<data_directory>'], arguments['--write'])

Tools/ART/scripts/art-task-grid.sh

@@ -21,9 +21,10 @@ if [ $1 == "--skip-setup" ]; then
 fi
 TYPE_OPTION="batch %RNDM:0"
 PATHENA_OPTIONS="--destSE=CERN-PROD_SCRATCHDISK"
+PATHENA_TYPE_OPTIONS=""
 if [ $1 == "--test-name" ]; then
   TYPE_OPTION="single $2"
-  PATHENA_OPTIONS="--destSE=CERN-PROD_SCRATCHDISK --forceStaged"
+  PATHENA_TYPE_OPTIONS="--forceStaged"
   shift
   shift
 fi
@@ -39,18 +40,6 @@ if [ $1 == "--nFiles" ]; then
   shift
   shift
 fi
-NEVENTS=""
-if [ $1 == "--nEvents" ]; then
-  NEVENTS="--nEventsPerFile $2"
-  shift
-  shift
-fi
-FILELIST=""
-if [ $1 == "--fileList" ]; then
-  FILELIST="--fileList $2"
-  shift
-  shift
-fi
 SUBMIT_DIRECTORY=$1
 shift
 SCRIPT_DIRECTORY=$1
@@ -101,7 +90,7 @@ fi
 # NOTE: for art-internal.py the current dir can be used as it is copied there
 cd ${SUBMIT_DIRECTORY}/${PACKAGE}/run
 SUBCOMMAND="./art-internal.py job grid ${SCRIPT_DIRECTORY} ${PACKAGE} ${TYPE} ${SEQUENCE_TAG} ${TYPE_OPTION} %OUT.tar ${NIGHTLY_RELEASE_SHORT} ${PROJECT} ${PLATFORM} ${NIGHTLY_TAG}"
-CMD="pathena ${GRID_OPTIONS} ${PATHENA_OPTIONS} --noBuild --expertOnly_skipScout --trf \"${SUBCOMMAND}\" ${SPLIT} --outDS ${OUTFILE} --extOutFile art-job.json ${INDS} ${NFILES} ${NEVENTS} ${FILELIST}"
+CMD="pathena ${GRID_OPTIONS} ${PATHENA_OPTIONS} ${PATHENA_TYPE_OPTIONS} --noBuild --expertOnly_skipScout --trf \"${SUBCOMMAND}\" ${SPLIT} --outDS ${OUTFILE} --extOutFile art-job.json ${INDS} ${NFILES}"
 
 #--disableAutoRetry
 #--excludedSite=ANALY_TECHNION-HEP-CREAM

Tools/ART/scripts/art.py

@@ -4,38 +4,44 @@
 ART - ATLAS Release Tester.
 
 Usage:
-  art.py run             [-v --type=<T> --max-jobs=<N> --ci] <script_directory> <sequence_tag>
-  art.py grid            [-v --type=<T>] <script_directory> <sequence_tag>
-  art.py submit          [-v --type=<T>] <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
-  art.py validate        [-v] <script_directory>
-  art.py included        [-v --type=<T> --test-type=<TT>] <script_directory> [<nightly_release> <project> <platform>]
-  art.py compare grid    [-v --days=<D>] <nightly_release> <project> <platform> <nightly_tag> <package> <test_name> <file_name>...
-  art.py compare ref     [-v]  <file_name> <ref_file>
-  art.py download        [-v] <input_file>
-  art.py list grid       [-v --json --type=<T> --test-type=<TT>] <package> <nightly_release> <project> <platform> <nightly_tag>
-  art.py log grid        [-v] <package> <test_name> <nightly_release> <project> <platform> <nightly_tag>
-  art.py output grid     [-v] <package> <test_name> <file_name> <nightly_release> <project> <platform> <nightly_tag>
+  art.py run             [-v -q --type=<T> --max-jobs=<N> --ci] <script_directory> <sequence_tag>
+  art.py grid            [-v -q --type=<T> -n] <script_directory> <sequence_tag>
+  art.py submit          [-v -q --type=<T> -n] <sequence_tag> <nightly_release> <project> <platform> <nightly_tag> [<package>]
+  art.py copy            [-v -q --user=<user> --dst=<dir>] <sequence_tag> <nightly_release> <project> <platform> <nightly_tag> [<package>]
+  art.py validate        [-v -q] <script_directory>
+  art.py included        [-v -q --type=<T> --test-type=<TT>] <script_directory> [<nightly_release> <project> <platform>]
+  art.py compare grid    [-v -q --days=<D> --user=<user>] <nightly_release> <project> <platform> <nightly_tag> <package> <test_name> <file_name>...
+  art.py compare ref     [-v -q]  <file_name> <ref_file>
+  art.py download        [-v -q] <input_file>
+  art.py list grid       [-v -q --user=<user> --json --type=<T> --test-type=<TT>] <package> <nightly_release> <project> <platform> <nightly_tag>
+  art.py log grid        [-v -q --user=<user>] <package> <test_name> <nightly_release> <project> <platform> <nightly_tag>
+  art.py output grid     [-v -q --user=<user>] <package> <test_name> <nightly_release> <project> <platform> <nightly_tag>
 
 Options:
   --ci              Run Continuous Integration tests only (using env: AtlasBuildBranch)
   --days=<D>        Number of days ago to pick up reference for compare [default: 1]
+  --dst=<dir>       Destination directory for downloaded files
+  -h --help         Show this screen.
   --json            Output in json format
   --max-jobs=<N>    Maximum number of concurrent jobs to run [default: 0]
+  -n --no-action    No real submit will be done
+  -q --quiet        Show less information, only warnings and errors
+  --test-type=<TT>  Type of test (e.g. all, batch or single) [default: all]
   --type=<T>        Type of job (e.g. grid, build)
-  --test-type=<TT>  Type of test (e.g. all, batch or single)
-  -h --help         Show this screen.
-  -v, --verbose     Show details.
+  --user=<user>     User to use for RUCIO
+  -v --verbose      Show more information, debug level
   --version         Show version.
 
 Sub-commands:
   run               Run jobs from a package in a local build (needs release and grid setup)
   grid              Run jobs from a package on the grid (needs release and grid setup)
   submit            Submit nightly jobs to the grid (NOT for users)
+  copy              Copy outputs and logs from RUCIO
   validate          Check headers in tests
-  included          Shows list of files which will be included for art submit/art grid
+  included          Show list of files which will be included for art submit/art grid
   compare           Compare the output of a job
   download          Download a file from rucio
-  list              Lists the jobs of a package
+  list              List the jobs of a package
   log               Show the log of a job
   output            Get the output of a job
 
@@ -55,8 +61,9 @@ Arguments:
 """
 
 __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
-__version__ = '0.5.4'
+__version__ = '0.6.5'
 
+import logging
 import os
 import sys
 
@@ -64,62 +71,77 @@ from ART.docopt_dispatch import dispatch
 
 from ART import ArtBase, ArtGrid, ArtBuild
 
+from ART.art_misc import set_log
 
 #
 # First list the double commands
 #
 
+
 @dispatch.on('compare', 'ref')
 def compare_ref(file_name, ref_file, **kwargs):
-    """TBD."""
+    """Compare the output of a job."""
+    set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     exit(ArtBase(art_directory).compare_ref(file_name, ref_file))
 
 
 @dispatch.on('compare', 'grid')
 def compare_grid(package, test_name, nightly_release, project, platform, nightly_tag, **kwargs):
-    """TBD."""
+    """Compare the output of a job."""
+    set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     days = int(kwargs['days'])
     file_names = kwargs['file_name']
-    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).compare(package, test_name, days, file_names))
+    user = kwargs['user']
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).compare(package, test_name, days, file_names, user))
 
 
 @dispatch.on('list', 'grid')
 def list(package, nightly_release, project, platform, nightly_tag, **kwargs):
-    """TBD."""
+    """List the jobs of a package."""
+    set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     job_type = 'grid' if kwargs['type'] is None else kwargs['type']
-    index_type = 'all' if kwargs['test_type'] is None else kwargs['test_type']
+    index_type = kwargs['test_type']
     json_format = kwargs['json']
-    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).list(package, job_type, index_type, json_format))
+    user = kwargs['user']
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).list(package, job_type, index_type, json_format, user))
 
 
 @dispatch.on('log', 'grid')
 def log(package, test_name, nightly_release, project, platform, nightly_tag, **kwargs):
-    """TBD."""
+    """Show the log of a job."""
+    set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
-    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).log(package, test_name))
+    user = kwargs['user']
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).log(package, test_name, user))
 
 
 @dispatch.on('output', 'grid')
-def output(package, test_name, file_name, nightly_release, project, platform, nightly_tag, **kwargs):
-    """TBD."""
+def output(package, test_name, nightly_release, project, platform, nightly_tag, **kwargs):
+    """Get the output of a job."""
+    set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
-    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).output(package, test_name, file_name))
+    user = kwargs['user']
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).output(package, test_name, user))
 
 
 @dispatch.on('submit')
 def submit(sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
-    """TBD."""
+    """Submit nightly jobs to the grid, NOT for users."""
+    set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
-    job_type = 'grid' if kwargs['type'] is None else kwargs['type']
-    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).task_list(job_type, sequence_tag))
+    job_type = kwargs['type']
+    package = kwargs['package']
+    no_action = kwargs['no_action']
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).task_list(job_type, sequence_tag, package, no_action))
 
 
 @dispatch.on('grid')
 def grid(script_directory, sequence_tag, **kwargs):
-    """TBD."""
+    """Run jobs from a package on the grid, needs release and grid setup."""
+    set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     try:
         nightly_release = os.environ['AtlasBuildBranch']
@@ -127,15 +149,18 @@ def grid(script_directory, sequence_tag, **kwargs):
         platform = os.environ[project + '_PLATFORM']
         nightly_tag = os.environ['AtlasBuildStamp']
     except KeyError, e:
-        print "Environment variable not set", e
+        log.critical("Environment variable not set %s", e)
         sys.exit(1)
     art_type = 'grid' if kwargs['type'] is None else kwargs['type']
-    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, True).task_list(art_type, sequence_tag))
+    package = None
+    no_action = kwargs['no_action']
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, True).task_list(art_type, sequence_tag, package, no_action))
 
 
 @dispatch.on('run')
 def run(script_directory, sequence_tag, **kwargs):
-    """TBD."""
+    """Run jobs from a package in a local build, needs release and grid setup."""
+    set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     try:
         nightly_release = os.environ['AtlasBuildBranch']
@@ -143,37 +168,57 @@ def run(script_directory, sequence_tag, **kwargs):
         platform = os.environ[project + '_PLATFORM']
         nightly_tag = os.environ['AtlasBuildStamp']
     except KeyError, e:
-        print "Environment variable not set", e
+        log.critical("Environment variable not set %s", e)
         sys.exit(1)
     art_type = 'build' if kwargs['type'] is None else kwargs['type']
     exit(ArtBuild(art_directory, nightly_release, project, platform, nightly_tag, script_directory, max_jobs=int(kwargs['max_jobs']), ci=kwargs['ci']).task_list(art_type, sequence_tag))
 
 
+@dispatch.on('copy')
+def copy(sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
+    """Copy outputs to eos area."""
+    set_log(kwargs)
+    art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+    package = kwargs['package']
+    # NOTE: default depends on USER, not set it here but in ArtGrid.copy
+    dst = kwargs['dst']
+    user = kwargs['user']
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).copy(sequence_tag, package, dst, user))
+
+
 @dispatch.on('validate')
 def validate(script_directory, **kwargs):
-    """TBD."""
+    """Check headers in tests."""
+    set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     exit(ArtBase(art_directory).validate(script_directory))
 
 
 @dispatch.on('included')
 def included(script_directory, **kwargs):
-    """TBD."""
+    """Show list of files which will be included for art submit/art grid."""
+    set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     nightly_release = os.environ['AtlasBuildBranch'] if kwargs['nightly_release'] is None else kwargs['nightly_release']
     project = os.environ['AtlasProject'] if kwargs['project'] is None else kwargs['project']
     platform = os.environ[project + '_PLATFORM'] if kwargs['platform'] is None else kwargs['platform']
     art_type = 'grid' if kwargs['type'] is None else kwargs['type']
-    index_type = 'all' if kwargs['test_type'] is None else kwargs['test_type']
+    index_type = kwargs['test_type']
     exit(ArtBase(art_directory).included(script_directory, art_type, index_type, nightly_release, project, platform))
 
 
 @dispatch.on('download')
 def download(input_file, **kwargs):
-    """TBD."""
+    """Download a file from rucio."""
+    set_log(kwargs)
     art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     exit(ArtBase(art_directory).download(input_file))
 
 
 if __name__ == '__main__':
+    if sys.version_info < (2, 7, 0):
+        sys.stderr.write("You need python 2.7 or later to run this script\n")
+        exit(1)
+
+    logging.basicConfig()
     dispatch(__doc__, version=os.path.splitext(os.path.basename(__file__))[0] + ' ' + __version__)