Merge branch 'ART-19june' into '21.0'

reorganize python dir, rename art_local.py as art_build.py, add share/localSetup…

See merge request !2772

Former-commit-id: ff55c54a6c279027dcd8bcfebcf546791afb3b4d

Tools/ART/python/ART/__init__.py

+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+"""TBD."""
+
+from art_base import ArtBase
+from art_build import ArtBuild
+from art_grid import ArtGrid

Tools/ART/python/art_base.py → Tools/ART/python/ART/art_base.py

 #!/usr/bin/env python
 # Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
 """TBD."""
+
 __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
 
 import fnmatch
 import inspect
 import os
-import re
 import sys
 import yaml
 
 from art_misc import check, run_command
+from art_header import ArtHeader
 
 
 class ArtBase(object):
@@ -48,13 +49,14 @@ class ArtBase(object):
         """TBD."""
         self.not_implemented()
 
-    def included(self):
+    def validate(self, script_directory):
         """TBD."""
-        self.not_implemented()
-
-    def wait_for(self):
-        """TBD."""
-        self.not_implemented()
+        directories = self.get_test_directories(script_directory)
+        for directory in directories.itervalues():
+            files = self.get_files(directory)
+            for fname in files:
+                ArtHeader(os.path.join(directory, fname)).validate()
+        return 0
 
     #
     # Default implementations
@@ -64,6 +66,7 @@ class ArtBase(object):
         out = check(run_command("acmd.py diff-root " + file_name + " " + ref_file + " --error-mode resilient --ignore-leaves RecoTimingObj_p1_HITStoRDO_timings RecoTimingObj_p1_RAWtoESD_mems RecoTimingObj_p1_RAWtoESD_timings RAWtoESD_mems RAWtoESD_timings ESDtoAOD_mems ESDtoAOD_timings HITStoRDO_timings RAWtoALL_mems RAWtoALL_timings RecoTimingObj_p1_RAWtoALL_mems RecoTimingObj_p1_RAWtoALL_timings --entries " + str(entries)))
         print out
         sys.stdout.flush()
+        return 0
 
     #
     # Protected Methods
@@ -75,32 +78,25 @@ class ArtBase(object):
         config_file.close()
         return config
 
-    def get_art_headers(self, filename):
-        """Return dictionary with art headers."""
-        result = {}
-        for line in open(filename, "r"):
-            line_match = re.match(r'#\s*art-(\w+):\s+(.+)$', line)
-            if line_match:
-                result[line_match.group(1)] = line_match.group(2)
-        return result
+    def get_files(self, directory, type=None):
+        """
+        Return a list of all test files matching 'test_*.sh' of given 'type'.
 
-    def get_files(self, directory, type):
-        """Return a list of all test files matching 'test_*.sh' of given 'queue'."""
+        If type is None, all files are returned. Only the filenames are returned.
+        """
         result = []
         if directory is not None:
             files = os.listdir(directory)
             files.sort()
             for fname in files:
                 if fnmatch.fnmatch(fname, 'test_*.sh') or fnmatch.fnmatch(fname, 'test_*.py'):
-                    headers = self.get_art_headers(os.path.join(directory, fname))
-                    if 'type' in headers and headers['type'] == type:
+                    if type is None or ArtHeader(os.path.join(directory, fname)).get('art-type') == type:
                         result.append(fname)
         return result
 
     def get_type(self, directory, test_name):
         """Return the 'type' of a test."""
-        headers = self.get_art_headers(os.path.join(directory, test_name))
-        return None if 'type' not in headers else headers['type']
+        return ArtHeader(os.path.join(directory, test_name)).get('art-type')
 
     def get_test_directories(self, directory):
         """

Tools/ART/python/art_local.py → Tools/ART/python/ART/art_build.py

@@ -4,56 +4,84 @@
 
 __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
 
-import os
-import sys
+import collections
+import json
 import multiprocessing
+import re
+import os
 
-from art_misc import run_command, verify, redirect, mkdir_p
+from art_misc import run_command, mkdir_p
 from art_base import ArtBase
+from art_header import ArtHeader
 
 from parallelScheduler import ParallelScheduler
 
 
-def run_job(sequence_tag, script_directory, package, type, index, test_name):
+def run_job(art_directory, sequence_tag, script_directory, package, type, index, test_name):
     """TBD."""
-    print "run_job", sequence_tag, script_directory, package, type, index, test_name
-    art_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
-    verify(run_command(os.path.join(art_dir, './art-internal.py') + " job local " + script_directory + " " + package + " " + type + " " + sequence_tag + " " + str(index) + " " + "out"))
-    # print out
+    # print "run_job", art_directory, sequence_tag, script_directory, package, type, index, test_name
+    (exit_code, out, err) = run_command(os.path.join(art_directory, './art-internal.py') + " job build " + script_directory + " " + package + " " + type + " " + sequence_tag + " " + str(index) + " " + "out")
+    return (test_name, exit_code, out, err)
 
 
-class ArtLocal(ArtBase):
+class ArtBuild(ArtBase):
     """TBD."""
 
-    def __init__(self, script_directory, max_jobs=0):
+    def __init__(self, art_directory, script_directory, max_jobs=0, ci=False):
         """TBD."""
-        print "ArtLocal", script_directory, max_jobs
+        # print "ArtBuild", art_directory, script_directory, max_jobs
+        self.art_directory = art_directory
         self.script_directory = script_directory
         self.max_jobs = multiprocessing.cpu_count() if max_jobs <= 0 else max_jobs
+        self.ci = ci
 
     def task_list(self, type, sequence_tag):
         """TBD."""
-        print "task_list", type, sequence_tag
+        # print "task_list", type, sequence_tag
         test_directories = self.get_test_directories(self.script_directory)
+        status = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.defaultdict()))
+
         for package, root in test_directories.items():
-            self.task(package, type, sequence_tag)
+            job_results = self.task(package, type, sequence_tag)
+            for job_result in job_results:
+                status[package][job_result[0]]['exit_code'] = job_result[1]
+                status[package][job_result[0]]['out'] = job_result[2]
+                status[package][job_result[0]]['err'] = job_result[3]
+
+        with open(os.path.join(sequence_tag, "status.json"), 'w') as outfile:
+            json.dump(status, outfile, sort_keys=True, indent=4, ensure_ascii=False)
+
+        return 0
 
     def task(self, package, type, sequence_tag):
         """TBD."""
-        print "task", package, type, sequence_tag
+        # print "task", package, type, sequence_tag
         test_names = self.get_list(self.script_directory, package, type)
         scheduler = ParallelScheduler(self.max_jobs + 1)
 
         index = 0
         for test_name in test_names:
-            scheduler.add_task(task_name="t" + str(index), dependencies=[], description="d", target_function=run_job, function_kwargs={'sequence_tag': sequence_tag, 'script_directory': self.script_directory, 'package': package, 'type': type, 'index': index, 'test_name': test_name})
+            schedule_test = False
+            if self.ci:
+                branch_name = os.environ['AtlasBuildBranch']
+                fname = os.path.join(self.get_test_directories(self.script_directory)[package], test_name)
+                cis = ArtHeader(fname).get('art-ci')
+                for ci in cis:
+                    if re.match(ci, branch_name):
+                        schedule_test = True
+                    break
+            else:
+                schedule_test = True
+
+            if schedule_test:
+                scheduler.add_task(task_name="t" + str(index), dependencies=[], description="d", target_function=run_job, function_kwargs={'art_directory': self.art_directory, 'sequence_tag': sequence_tag, 'script_directory': self.script_directory, 'package': package, 'type': type, 'index': index, 'test_name': test_name})
             index += 1
 
-        scheduler.run()
+        return scheduler.run()
 
     def job(self, package, type, sequence_tag, index, out):
         """TBD."""
-        print "job", package, type, sequence_tag, index, out
+        # print "job", package, type, sequence_tag, index, out
         test_directories = self.get_test_directories(self.script_directory)
         test_directory = os.path.abspath(test_directories[package])
         test_name = self.get_files(test_directory, type)[int(index)]
@@ -61,5 +89,11 @@ class ArtLocal(ArtBase):
         work_directory = os.path.join(sequence_tag, package, os.path.splitext(test_name)[0])
         mkdir_p(work_directory)
 
-        output = redirect(run_command(os.path.join(test_directory, test_name) + ' ' + '.' + ' ' + package + ' ' + type + ' ' + test_name, dir=work_directory, redirect=True))
-        print output
+        (exit_code, output, err) = run_command(os.path.join(test_directory, test_name) + ' ' + '.' + ' ' + package + ' ' + type + ' ' + test_name, dir=work_directory)
+
+        with open(os.path.join(work_directory, "stdout.txt"), "w") as text_file:
+            text_file.write(output)
+        with open(os.path.join(work_directory, "stderr.txt"), "w") as text_file:
+            text_file.write(err)
+
+        return exit_code

Tools/ART/python/art_grid.py → Tools/ART/python/ART/art_grid.py

@@ -10,17 +10,17 @@ import re
 import shutil
 import sys
 import tarfile
-import time
 
 from art_base import ArtBase
-from art_misc import mkdir_p, make_executable, check, run_command, count_string_occurrence
+from art_misc import mkdir_p, make_executable, check, run_command
 
 
 class ArtGrid(ArtBase):
     """TBD."""
 
-    def __init__(self, nightly_release, project, platform, nightly_tag):
+    def __init__(self, art_directory, nightly_release, project, platform, nightly_tag):
         """TBD."""
+        self.art_directory = art_directory
         self.nightly_release = nightly_release
         self.project = project
         self.platform = platform
@@ -39,9 +39,6 @@ class ArtGrid(ArtBase):
 
     def task_list(self, type, sequence_tag):
         """TBD."""
-        # get the path of the art.py script
-        art_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
-
         # job will be submitted from tmp directory
         submit_directory = 'tmp'
 
@@ -82,38 +79,42 @@ class ArtGrid(ArtBase):
                     sys.stdout.flush()
                     submit_dir = os.path.join(submit_directory, package)
                     run = os.path.join(submit_dir, "run")
-                    mkdir_p(run)
-
-                    shutil.copy(os.path.join(art_dir, 'art.py'), run)
-                    shutil.copy(os.path.join(art_dir, 'art-internal.py'), run)
-                    shutil.copy(os.path.join(art_dir, 'art_base.py'), run)
-                    shutil.copy(os.path.join(art_dir, 'art_local.py'), run)
-                    shutil.copy(os.path.join(art_dir, 'art_grid.py'), run)
-                    shutil.copy(os.path.join(art_dir, 'art_batch.py'), run)
-                    shutil.copy(os.path.join(art_dir, 'art_misc.py'), run)
-                    shutil.copy(os.path.join(art_dir, 'serialScheduler.py'), run)
-                    shutil.copy(os.path.join(art_dir, 'parallelScheduler.py'), run)
-                    shutil.copy(os.path.join(art_dir, 'docopt.py'), run)
-                    shutil.copy(os.path.join(art_dir, 'docopt_dispatch.py'), run)
+                    ART = os.path.join(run, "ART")
+                    mkdir_p(ART)
+
+                    # get the path of the python classes and support scripts
+                    art_python_directory = os.path.join(self.art_directory, '..', 'python', 'ART')
+
+                    # FIXME we need to know where all those files went in the Athena install
+                    shutil.copy(os.path.join(self.art_directory, 'art.py'), run)
+                    shutil.copy(os.path.join(self.art_directory, 'art-internal.py'), run)
+                    shutil.copy(os.path.join(art_python_directory, '__init__.py'), ART)
+                    shutil.copy(os.path.join(art_python_directory, 'art_base.py'), ART)
+                    shutil.copy(os.path.join(art_python_directory, 'art_build.py'), ART)
+                    shutil.copy(os.path.join(art_python_directory, 'art_grid.py'), ART)
+                    shutil.copy(os.path.join(art_python_directory, 'art_header.py'), ART)
+                    shutil.copy(os.path.join(art_python_directory, 'art_misc.py'), ART)
+                    shutil.copy(os.path.join(art_python_directory, 'docopt.py'), ART)
+                    shutil.copy(os.path.join(art_python_directory, 'docopt_dispatch.py'), ART)
+                    shutil.copy(os.path.join(art_python_directory, 'parallelScheduler.py'), ART)
+                    shutil.copy(os.path.join(art_python_directory, 'serialScheduler.py'), ART)
 
                     make_executable(os.path.join(run, 'art.py'))
                     make_executable(os.path.join(run, 'art-internal.py'))
 
-                    command = os.path.join(art_dir, 'art-internal.py') + ' task grid ' + package + ' ' + type + ' ' + sequence_tag + ' ' + self.nightly_release + ' ' + self.project + ' ' + self.platform + ' ' + self.nightly_tag
+                    command = os.path.join(self.art_directory, 'art-internal.py') + ' task grid ' + package + ' ' + type + ' ' + sequence_tag + ' ' + self.nightly_release + ' ' + self.project + ' ' + self.platform + ' ' + self.nightly_tag
                     print command
                     sys.stdout.flush()
                     out = check(run_command(command))
                     print out
                     sys.stdout.flush()
+        return 0
 
     def task(self, package, type, sequence_tag):
         """TBD."""
         print 'Running art task'
         sys.stdout.flush()
 
-        # get the path of the art.py script
-        art_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
-
         number_of_tests = len(self.get_list(self.get_script_directory(), package, type))
 
         print self.nightly_release + " " + self.project + " " + self.platform + " " + self.nightly_tag + " " + sequence_tag + " " + package + " " + type + " " + str(number_of_tests)
@@ -121,9 +122,10 @@ class ArtGrid(ArtBase):
 
         # run task from Bash Script as is needed in ATLAS setup
         # FIXME we need to parse the output
-        out = check(run_command(os.path.join(art_dir, 'art-task-grid.sh') + " " + package + " " + type + " " + sequence_tag + " " + str(number_of_tests) + " " + self.nightly_release + " " + self.project + " " + self.platform + " " + self.nightly_tag))
+        out = check(run_command(os.path.join(self.art_directory, 'art-task-grid.sh') + " " + package + " " + type + " " + sequence_tag + " " + str(number_of_tests) + " " + self.nightly_release + " " + self.project + " " + self.platform + " " + self.nightly_tag))
         print out
         sys.stdout.flush()
+        return 0
 
     def job(self, package, type, sequence_tag, index, out):
         """TBD."""
@@ -164,6 +166,7 @@ class ArtGrid(ArtBase):
                         if os.path.exists(out_name[0]):
                             tar_file.add(out_name[0])
         tar_file.close()
+        return 0
 
     def list(self, package, type):
         """TBD."""
@@ -173,6 +176,7 @@ class ArtGrid(ArtBase):
             print str(i) + ' ' + job
             sys.stdout.flush()
             i += 1
+        return 0
 
     def log(self, package, test_name):
         """TBD."""
@@ -185,6 +189,7 @@ class ArtGrid(ArtBase):
                 print content
                 break
         tar.close()
+        return 0
 
     def output(self, package, test_name, file_name):
         """TBD."""
@@ -195,46 +200,6 @@ class ArtGrid(ArtBase):
                 tar.extractall(path='.', members=[member])
                 break
         tar.close()
-
-    def included(self):
-        """TBD."""
-        package_name = "__name_never_used__"
-
-        if self.excluded(self.get_config(), package_name):
-            print 'Excluding ' + 'all' + ' for ' + self.nightly_release + ' project ' + self.project + ' on ' + self.platform
-            print 'art-status: excluded'
-            sys.stdout.flush()
-            return 1
-        else:
-            print 'art-status: included'
-            return 0
-
-    def wait_for(self):
-        """TBD."""
-        directory = os.path.join(self.cvmfs_directory, self.nightly_release, self.nightly_tag)
-        path = os.path.join(directory, self.nightly_release + "__" + self.project + "__" + self.platform + "*" + self.nightly_tag + "__*.ayum.log")
-
-        count = 0
-        needed = 1
-        value = count_string_occurrence(path, "Install looks to have been successful")
-        print "art-status: waiting"
-        print path
-        print "count: " + str(value) + " mins: " + str(count)
-        sys.stdout.flush()
-        while (value < needed) and (count < 30):
-            time.sleep(60)
-            count += 1
-            value = count_string_occurrence(path, "Install looks to have been successful")
-            print "count: " + str(value) + " mins: " + str(count)
-            sys.stdout.flush()
-
-        if value < needed:
-            print "art-status: no release"
-            sys.stdout.flush()
-            return -2
-
-        print "art-status: setup"
-        sys.stdout.flush()
         return 0
 
     def compare(self, package, test_name, days, file_names):
@@ -255,6 +220,7 @@ class ArtGrid(ArtBase):
             ref_file = os.path.join(ref_dir, file_name)
 
             self.compare_ref(file_name, ref_file, 10)
+        return 0
 
     #
     # Protected Methods

Tools/ART/python/ART/art_header.py

+#!/usr/bin/env python
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+"""TBD."""
+
+__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
+
+import re
+
+from copy import copy
+from types import ListType
+from types import StringType
+
+
+class ArtHeader(object):
+    """TBD."""
+
+    def __init__(self, filename):
+        """TBD."""
+        self.header = re.compile(r'#\s(art-[\w-]+):\s+(.+)$')
+        self.header_format_error1 = re.compile(r'#(art-[\w-]+):\s*(.+)$')
+        self.header_format_error2 = re.compile(r'#\s\s+(art-[\w-]+):\s*(.+)$')
+        self.header_format_error3 = re.compile(r'#\s(art-[\w-]+):\S(.*)$')
+
+        self.filename = filename
+
+        self.type = {}
+        self.defaultValue = {}
+        self.constraint = {}
+        self.value = {}
+
+        self.add('art-description', StringType, '')
+        self.add('art-type', StringType, None, ['build', 'grid'])
+        self.add('art-ci', ListType, [])
+
+        self.read(filename)
+
+    def add(self, key, type, defaultValue=None, constraint=None):
+        """TBD."""
+        self.type[key] = type
+        self.defaultValue[key] = defaultValue
+        self.constraint[key] = constraint
+        self.value[key] = copy(defaultValue)
+
+    def is_list(self, key):
+        """TBD."""
+        return self.type[key] is ListType if key in self.type else False
+
+    def read(self, filename):
+        """Read all headers from file."""
+        for line in open(filename, "r"):
+            line_match = self.header.match(line)
+            if line_match:
+                key = line_match.group(1)
+                value = line_match.group(2)
+                if self.is_list(key):
+                    self.value[key].append(value)
+                else:
+                    self.value[key] = value
+
+    def get(self, key):
+        """TBD."""
+        return self.value[key] if key in self.value else None
+
+    def print_it(self):
+        """TBD."""
+        for key in self.type:
+            print key, self.type[key], self.defaultValue[key], self.value[key], self.constraint[key]
+
+    def validate(self):
+        """
+        Validate the '# art-*' headers in the file.
+
+        Validation fails if:
+        - a header is spaced correctly (e.g. '#art-header: value')
+        - a value in a header is not spaced correctly (e.g. '# art-header:value')
+        - a key is found which is not defined
+        - a value is found of the wrong type
+        - a value is found outside the constraint
+        """
+        for line in open(self.filename, "r"):
+            if self.header_format_error1.match(line):
+                print "LINE: ", line.rstrip()
+                print "ERROR: Header Validation - invalid header format, use space between '# and art-xxx' in file", self.filename
+                print
+            if self.header_format_error2.match(line):
+                print "LINE: ", line.rstrip()
+                print "ERROR: Header Validation - invalid header format, too many spaces between '# and art-xxx' in file", self.filename
+                print
+            if self.header_format_error3.match(line):
+                print "LINE: ", line.rstrip()
+                print "ERROR: Header Validation - invalid header format, use at least one space between ': and value' in file", self.filename
+                print
+
+        for key in self.value:
+            if key not in self.type:
+                print "ERROR: Header Validation - Invalid key: '" + key + "' in file", self.filename
+                print
+                continue
+            if type(self.value[key]) != self.type[key]:
+                if not isinstance(self.value[key], type(None)):
+                    print "ERROR: Header Validation - type:", type(self.value[key]), "not valid for key:", key, "expected type:", self.type[key], "in file", self.filename
+                    print
+            if self.constraint[key] is not None and self.value[key] not in self.constraint[key]:
+                if self.value[key] is None:
+                    print "ERROR: Header Validation - missing key: '" + key + "' in file", self.filename
+                else:
+                    print "ERROR: Header Validation - value: '" + self.value[key] + "' for key: '" + key + "' not in constraints:", self.constraint[key], "in file", self.filename
+                print
+
+        return 0

Tools/ART/python/art_misc.py → Tools/ART/python/ART/art_misc.py

@@ -5,14 +5,12 @@
 __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
 
 import errno
-import glob
 import os
 import shlex
 import subprocess
-import sys
 
 
-def run_command(cmd, dir=None, shell=False, redirect=False):
+def run_command(cmd, dir=None, shell=False):
     """Run the given command locally and returns the output, err and exit_code."""
     print "Execute: " + cmd
     if "|" in cmd:
@@ -32,12 +30,6 @@ def run_command(cmd, dir=None, shell=False, redirect=False):
     (output, err) = p[i - 1].communicate()
     exit_code = p[0].wait()
 
-    if redirect:
-        with open(os.path.join(dir, "stdout.txt"), "w") as text_file:
-            text_file.write(str(output))
-        with open(os.path.join(dir, "stderr.txt"), "w") as text_file:
-            text_file.write(str(err))
-
     return exit_code, str(output), str(err)
 
 
@@ -60,13 +52,13 @@ def verify((exitcode, out, err)):
     """Check exitcode and print statement."""
     if exitcode == 0:
         print out
-        return out
+        return exitcode
 
     print "Error:", exitcode
     print "StdOut:", out
     print "StdErr:", err
 
-    print 'art-status: error'
+    # print 'art-status: error'
 
     return exitcode
 
@@ -114,15 +106,3 @@ def which(program):
                 return exe_file
 
     return None
-
-
-def count_string_occurrence(path, string):
-    """Count number of occurences of 'string' inside file 'path'. Returns count or 0."""
-    for file in glob.iglob(path):
-        print file
-        sys.stdout.flush()
-        f = open(file)
-        contents = f.read()
-        f.close()
-        return contents.count(string)
-    return 0

Tools/ART/python/parallelScheduler.py → Tools/ART/python/ART/parallelScheduler.py

@@ -3,7 +3,7 @@ Created on 16/05/2012
 
  * Repository : https://github.com/victor-gil-sepulveda/pyScheduler
  * Licensed under the MIT license (see LICENSE-MIT)
- * Copyright (C) 2013 Víctor Alejandro Gil Sepúlveda
+ * Copyright (C) 2013 Victor Alejandro Gil Sepulveda
 
 @author: victor
 '''

Tools/ART/python/serialScheduler.py → Tools/ART/python/ART/serialScheduler.py

@@ -3,7 +3,7 @@ Created on 16/08/2012
 
  * Repository : https://github.com/victor-gil-sepulveda/pyScheduler
  * Licensed under the MIT license (see LICENSE-MIT)
- * Copyright (C) 2013 Víctor Alejandro Gil Sepúlveda
+ * Copyright (C) 2013 Victor Alejandro Gil Sepulveda
 
 @author: victor
 '''

Tools/ART/scripts/art-internal.py

@@ -4,12 +4,10 @@
 ART-internal - ATLAS Release Tester (internal command).
 
 Usage:
-  art.py included            [-v]  <nightly_release> <project> <platform>
-  art.py job local           [-v]  <script_directory> <package> <job_type> <sequence_tag> <index> <out>
-  art.py job (grid|batch)    [-v]  <package> <job_type> <sequence_tag> <index> <out> <nightly_release> <project> <platform> <nightly_tag>
-  art.py task local          [-v]  <script_directory> <package> <job_type> <sequence_tag>
-  art.py task (grid|batch)   [-v]  <package> <job_type> <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
-  art.py wait_for            [-v]  <nightly_release> <project> <platform> <nightly_tag>
+  art-internal.py job build   [-v]  <script_directory> <package> <job_type> <sequence_tag> <index> <out>
+  art-internal.py job grid    [-v]  <package> <job_type> <sequence_tag> <index> <out> <nightly_release> <project> <platform> <nightly_tag>
+  art-internal.py task build  [-v]  <script_directory> <package> <job_type> <sequence_tag>
+  art-internal.py task grid   [-v]  <package> <job_type> <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
 
 Options:
   -h --help         Show this screen.
@@ -17,10 +15,8 @@ Options:
   -v, --verbose     Show details.
 
 Sub-commands:
-  included          Check if a release and platform is included
   job               Runs a single job, given a particular index
   task              Runs a single task, consisting of given number of jobs
-  wait_for          Wait for the release to be available
 
 Arguments:
   index             Index of the test inside the package
@@ -39,28 +35,21 @@ __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
 
 import os
 import sys
-from docopt_dispatch import dispatch
 
-from art_local import ArtLocal
-from art_grid import ArtGrid
-from art_batch import ArtBatch
+from ART.docopt_dispatch import dispatch
 
-
-@dispatch.on('included')
-def included(nightly_release, project, platform, **kwargs):
-    """TBD."""
-    sys.exit(ArtGrid(nightly_release, project, platform, None).included())
+from ART import ArtGrid, ArtBuild
 
 
-@dispatch.on('job', 'local')
-def job_local(script_directory, package, job_type, sequence_tag, index, out, **kwargs):
+@dispatch.on('job', 'build')
+def job_build(script_directory, package, job_type, sequence_tag, index, out, **kwargs):
     """TBD.
 
     Tests are called with the following parameters:
     SCRIPT_DIRECTORY, PACKAGE, TYPE, TEST_NAME
     """
-    print "job_local", script_directory, package, job_type, sequence_tag, index, out, kwargs
-    ArtLocal(script_directory).job(package, job_type, sequence_tag, index, out)
+    art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+    exit(ArtBuild(art_directory, script_directory).job(package, job_type, sequence_tag, index, out))
 
 
 @dispatch.on('job', 'grid')
@@ -70,44 +59,26 @@ def job_grid(package, job_type, sequence_tag, index, out, nightly_release, proje
     Tests are called with the following parameters:
     SCRIPT_DIRECTORY, PACKAGE, TYPE, TEST_NAME, NIGHTLY_RELEASE, PROJECT, PLATFORM, NIGHTLY_TAG
     """
-    ArtGrid(nightly_release, project, platform, nightly_tag).job(package, job_type, sequence_tag, index, out)
-
-
-@dispatch.on('job', 'batch')
-def job_batch(package, job_type, sequence_tag, index, out, nightly_release, project, platform, nightly_tag, **kwargs):
-    """TBD.
+    art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).job(package, job_type, sequence_tag, index, out))
 
-    Tests are called with the following parameters:
-    SCRIPT_DIRECTORY, PACKAGE, TYPE, TEST_NAME, NIGHTLY_RELEASE, PROJECT, PLATFORM, NIGHTLY_TAG
-    """
-    ArtBatch(nightly_release, project, platform, nightly_tag).job(package, job_type, sequence_tag, index, out)
 
-
-@dispatch.on('task', 'local')
-def task_local(script_directory, job_type, sequence_tag, **kwargs):
+@dispatch.on('task', 'build')
+def task_build(script_directory, job_type, sequence_tag, **kwargs):
     """TBD."""
-    ArtLocal(script_directory).task(job_type, sequence_tag)
+    art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+    exit(ArtBuild(art_directory, script_directory).task(job_type, sequence_tag))
 
 
 @dispatch.on('task', 'grid')
 def task_grid(package, job_type, sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
     """TBD."""
-    ArtGrid(nightly_release, project, platform, nightly_tag).task(package, job_type, sequence_tag)
-
-
-@dispatch.on('task', 'batch')
-def task_batch(package, job_type, sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
-    """TBD."""
-    ArtBatch(nightly_release, project, platform, nightly_tag).task(package, sequence_tag)
-
-
-@dispatch.on('wait_for')
-def wait_for(nightly_release, project, platform, nightly_tag, **kwargs):
-    """TBD."""
-    sys.exit(ArtGrid(nightly_release, project, platform, nightly_tag).wait_for())
+    art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).task(package, job_type, sequence_tag))
 
 
 if __name__ == '__main__':
     # NOTE: import should be here, to keep the order of the decorators (module first, art last and unused)
     from art import __version__
+    print "ART_PATH", os.path.dirname(os.path.realpath(sys.argv[0]))
     dispatch(__doc__, version=os.path.splitext(os.path.basename(__file__))[0] + ' ' + __version__)

Tools/ART/scripts/art-task-build.sh

+#!/bin/bash
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+# arguments: RELEASE_BASE, PROJECT, PLATFORM, DATESTAMP
+# author : Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>, Emil Obreshkov <Emil.Obreshkov@cern.ch>
+
+whoami
+date
+
+RELEASE_BASE=$1
+PROJECT=$2
+PLATFORM=$3
+DATESTAMP=$4
+
+echo Release base $RELEASE_BASE
+echo Project $PROJECT
+echo Platform $PLATFORM
+echo Date $DATESTAMP
+
+BRANCH=`echo $RELEASE_BASE |tr "/" " " |awk '{print $5}'`
+
+# setup for the build
+[[ "${ATLAS_LOCAL_ROOT_BASE}" = "" ]] && export ATLAS_LOCAL_ROOT_BASE="/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase"
+source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh --quiet
+lsetup asetup
+asetup none,cmakesetup --platform ${PLATFORM}
+source ${RELEASE_BASE}/build/install/${PROJECT}/*/InstallArea/${PLATFORM}/setup.sh
+
+# run build tests
+SUBDIR=${BRANCH}/${PROJECT}/${PLATFORM}/${DATESTAMP}
+OUTDIR="art-build/${SUBDIR}"
+CMD="art.py run ${RELEASE_BASE} ${OUTDIR}"
+echo ${CMD}
+RESULT=`eval "${CMD}"`
+echo ${RESULT}
+
+# copy the test results to EOS area
+TARGETDIR=/eos/atlas/atlascerngroupdisk/art-build/${SUBDIR}
+if [[ ! -e ${TARGETDIR} ]]; then
+  eos mkdir -p ${TARGET}
+  xrdcp -vr $OUTDIR $TARGETDIR
+fi

Tools/ART/scripts/art-task-grid.sh

 #!/bin/bash
 # Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+#
 # NOTE do NOT run with /bin/bash -x as the output is too big for gitlab-ci
 # arguments: PACKAGE, SEQUENCE_TAG, NUMBER_OF_TESTS, NIGHTLY_RELEASE, PROJECT, PLATFORM, NIGHTLY_TAG
 #
+# author : Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>
+#
 # example: Tier0ChainTests grid 316236 32 21.0 Athena x86_64-slc6-gcc62-opt 2017-02-26T2119
 #set -e
 
@@ -23,6 +26,8 @@ NIGHTLY_TAG=$8
 export ATLAS_LOCAL_ROOT_BASE=/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase
 source $ATLAS_LOCAL_ROOT_BASE/user/atlasLocalSetup.sh
 
+export RUCIO_ACCOUNT=artprod
+
 lsetup panda
 
 voms-proxy-init --rfc -noregen -cert ./grid.proxy -voms atlas
@@ -32,6 +37,7 @@ NIGHTLY_RELEASE_SHORT=${NIGHTLY_RELEASE/-VAL-*/-VAL}
 
 asetup --platform=${PLATFORM} ${NIGHTLY_RELEASE_SHORT},${NIGHTLY_TAG},${PROJECT}
 
+# NOTE: for art-internal.py the current dir can be used as it is copied there
 cd ./tmp/${PACKAGE}/run
 OUTFILE="user.${USER}.atlas.${NIGHTLY_RELEASE_SHORT}.${PROJECT}.${PLATFORM}.${NIGHTLY_TAG}.${SEQUENCE_TAG}.${PACKAGE}"
 CMD="pathena --noBuild --skipScout --trf \"./art-internal.py job grid ${PACKAGE} ${TYPE} ${SEQUENCE_TAG} %RNDM:0 %OUT.tar ${NIGHTLY_RELEASE_SHORT} ${PROJECT} ${PLATFORM} ${NIGHTLY_TAG}\" --split ${NUMBER_OF_TESTS} --outDS ${OUTFILE}"

Tools/ART/scripts/art.py

@@ -4,8 +4,9 @@
 ART - ATLAS Release Tester.
 
 Usage:
-  art.py run             [-v --type=<T> --max-jobs=<N>]  <script_directory> <sequence_tag>
+  art.py run             [-v --type=<T> --max-jobs=<N> --ci]  <script_directory> <sequence_tag>
   art.py submit          [-v --type=<T>]  <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
+  art.py validate        [-v]  <script_directory>
   art.py compare grid    [-v --days=<D>]  <nightly_release> <project> <platform> <nightly_tag> <package> <test_name> <file_name>...
   art.py compare ref     [-v]  <file_name> <ref_file>
   art.py list grid       [-v]  <package> <job_type> <nightly_release> <project> <platform> <nightly_tag>
@@ -16,13 +17,15 @@ Options:
   -h --help         Show this screen.
   --version         Show version.
   -v, --verbose     Show details.
-  --type=<T>        Type of job (e.g. grid, ci, build)
+  --type=<T>        Type of job (e.g. grid, build)
   --days=<D>        Number of days ago to pick up reference for compare [default: 1]
   --max-jobs=<N>    Maximum number of concurrent jobs to run
+  --ci              Run Continuous Integration tests only (using env: AtlasBuildBranch)
 
 Sub-commands:
-  run               Run tests from a package locally
+  run               Run tests from a package in a local build
   submit            Submit tests to the grid
+  validate          Check headers in tests
   compare           Compare the output of a job
   list              Lists the jobs of a package
   log               Show the log of a job
@@ -40,7 +43,7 @@ Arguments:
   script_directory  Directory containing the packages with tests
   sequence_tag      Sequence tag (e.g. 0 or PIPELINE_ID)
   test_name         Name of the test inside the package (e.g. test_q322.sh)
-  job_type          Type of job (e.g. grid, ci, build)
+  job_type          Type of job (e.g. grid, build)
 """
 
 __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
@@ -51,57 +54,67 @@ import requests
 import os
 import sys
 
+from ART.docopt_dispatch import dispatch
 
-from docopt_dispatch import dispatch
-from art_base import ArtBase
-from art_local import ArtLocal
-from art_grid import ArtGrid
+from ART import ArtBase, ArtGrid, ArtBuild
 
 
 @dispatch.on('submit')
 def submit_grid(sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
     """TBD."""
+    art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     type = 'grid' if kwargs['type'] is None else kwargs['type']
-    ArtGrid(nightly_release, project, platform, nightly_tag).task_list(type, sequence_tag)
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).task_list(type, sequence_tag))
 
 
 @dispatch.on('run')
 def run(script_directory, sequence_tag, **kwargs):
     """TBD."""
+    art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     type = 'build' if kwargs['type'] is None else kwargs['type']
-    ArtLocal(script_directory, max_jobs=kwargs['max_jobs']).task_list(type, sequence_tag)
+    exit(ArtBuild(art_directory, script_directory, max_jobs=kwargs['max_jobs'], ci=kwargs['ci']).task_list(type, sequence_tag))
+
+
+@dispatch.on('validate')
+def validate(script_directory, **kwargs):
+    """TBD."""
+    exit(ArtBase().validate(script_directory))
 
 
 @dispatch.on('compare', 'ref')
-def compare_local(file_name, ref_file, **kwargs):
+def compare_ref(file_name, ref_file, **kwargs):
     """TBD."""
-    ArtBase().compare_ref(file_name, ref_file)
+    exit(ArtBase().compare_ref(file_name, ref_file))
 
 
 @dispatch.on('compare', 'grid')
 def compare_grid(package, test_name, nightly_release, project, platform, nightly_tag, **kwargs):
     """TBD."""
+    art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
     days = int(kwargs['days'])
     file_names = kwargs['file_name']
-    ArtGrid(nightly_release, project, platform, nightly_tag).compare(package, test_name, days, file_names)
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).compare(package, test_name, days, file_names))
 
 
 @dispatch.on('list', 'grid')
 def list(package, job_type, nightly_release, project, platform, nightly_tag, **kwargs):
     """TBD."""
-    ArtGrid(nightly_release, project, platform, nightly_tag).list(package, job_type)
+    art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).list(package, job_type))
 
 
 @dispatch.on('log', 'grid')
 def log(package, test_name, nightly_release, project, platform, nightly_tag, **kwargs):
     """TBD."""
-    ArtGrid(nightly_release, project, platform, nightly_tag).log(package, test_name)
+    art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).log(package, test_name))
 
 
 @dispatch.on('output', 'grid')
 def output(package, test_name, file_name, nightly_release, project, platform, nightly_tag, **kwargs):
     """TBD."""
-    ArtGrid(nightly_release, project, platform, nightly_tag).output(package, test_name, file_name)
+    art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+    exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).output(package, test_name, file_name))
 
 
 @dispatch.on('retrieve')
@@ -158,4 +171,5 @@ def retrieve(job_id, **kwargs):
 
 
 if __name__ == '__main__':
+    print "ART_PATH", os.path.dirname(os.path.realpath(sys.argv[0]))
     dispatch(__doc__, version=os.path.splitext(os.path.basename(__file__))[0] + ' ' + __version__)

Tools/ART/share/localSetupART.sh

+SCRIPTPATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+export PYTHONPATH=${SCRIPTPATH}/python:${PYTHONPATH}