diff --git a/Tools/ART/python/ART/art_base.py b/Tools/ART/python/ART/art_base.py
index 9e5db0d990253f0c12878fe352094c6e42a780e7..c9032de7cb6e744b42b8688111c4024d781d69c2 100755
--- a/Tools/ART/python/ART/art_base.py
+++ b/Tools/ART/python/ART/art_base.py
@@ -10,26 +10,26 @@ import os
import sys
import yaml
-from art_misc import run_command
+from art_misc import is_exe, run_command
from art_header import ArtHeader
class ArtBase(object):
"""TBD."""
- def __init__(self):
+ def __init__(self, art_directory):
"""TBD."""
- pass
+ self.art_directory = art_directory
- def task_list(self, type, sequence_tag):
+ def task_list(self, job_type, sequence_tag):
"""TBD."""
self.not_implemented()
- def task(self, package, type, sequence_tag):
+ def task(self, package, job_type, sequence_tag):
"""TBD."""
self.not_implemented()
- def job(self, package, type, sequence_tag, index, out):
+ def job(self, package, job_type, sequence_tag, index, out):
"""TBD."""
self.not_implemented()
@@ -37,7 +37,7 @@ class ArtBase(object):
"""TBD."""
self.not_implemented()
- def list(self, package, type, json_format=False):
+ def list(self, package, job_type, json_format=False):
"""TBD."""
self.not_implemented()
@@ -55,9 +55,28 @@ class ArtBase(object):
for directory in directories.itervalues():
files = self.get_files(directory)
for fname in files:
- ArtHeader(os.path.join(directory, fname)).validate()
+ test_name = os.path.join(directory, fname)
+ print test_name
+ if not is_exe(test_name):
+ print "ERROR: ", test_name, "is not executable."
+ ArtHeader(test_name).validate()
return 0
+ def included(self, script_directory, job_type, index_type, nightly_release, project, platform):
+ """TBD."""
+ directories = self.get_test_directories(script_directory.rstrip("/"))
+ for directory in directories.itervalues():
+ files = self.get_files(directory, job_type, index_type)
+ for fname in files:
+ test_name = os.path.join(directory, fname)
+ if self.is_included(test_name, nightly_release, project, platform):
+ print test_name, ArtHeader(test_name).get('art-include')
+ return 0
+
+ def download(self, input_file):
+ """TBD."""
+ return self.get_input(input_file)
+
#
# Default implementations
#
@@ -92,24 +111,49 @@ class ArtBase(object):
config_file.close()
return config
- def get_files(self, directory, type=None):
+ def get_files(self, directory, job_type=None, index_type="all", nightly_release=None, project=None, platform=None):
"""
- Return a list of all test files matching 'test_*.sh' of given 'type'.
+ Return a list of all test files matching 'test_*.sh' of given 'job_type', 'index_type' and nightly/project/platform.
+
+ 'index_type' can be 'all', 'batch' or 'single'.
- If type is None, all files are returned. Only the filenames are returned.
+ If "given" is None, all files are returned.
+
+ Only the filenames are returned.
"""
result = []
if directory is not None:
files = os.listdir(directory)
files.sort()
for fname in files:
- if fnmatch.fnmatch(fname, 'test_*.sh') or fnmatch.fnmatch(fname, 'test_*.py'):
- if type is None or ArtHeader(os.path.join(directory, fname)).get('art-type') == type:
- result.append(fname)
+ # is not a test ?
+ if not fnmatch.fnmatch(fname, 'test_*.sh') and not fnmatch.fnmatch(fname, 'test_*.py'):
+ continue
+
+ test_name = os.path.join(directory, fname)
+
+ # is not of correct type
+ if job_type is not None and ArtHeader(test_name).get('art-type') != job_type:
+ continue
+
+ # is not included in nightly_release, project, platform
+ if nightly_release is not None and not self.is_included(test_name, nightly_release, project, platform):
+ continue
+
+ # batch and does specify art-input
+ if index_type == "batch" and ArtHeader(test_name).get('art-input') is not None:
+ continue
+
+ # single and does not specify art-input
+ if index_type == "single" and ArtHeader(test_name).get('art-input') is None:
+ continue
+
+ result.append(fname)
+
return result
def get_type(self, directory, test_name):
- """Return the 'type' of a test."""
+ """Return the 'job_type' of a test."""
return ArtHeader(os.path.join(directory, test_name)).get('art-type')
def get_test_directories(self, directory):
@@ -125,11 +169,46 @@ class ArtBase(object):
result[package] = root
return result
- def get_list(self, directory, package, type):
+ def get_list(self, directory, package, job_type, index_type):
"""Return a list of tests for a particular package."""
test_directories = self.get_test_directories(directory)
test_dir = test_directories[package]
- return self.get_files(test_dir, type)
+ return self.get_files(test_dir, job_type, index_type)
+
+ def is_included(self, test_name, nightly_release, project, platform):
+ """Return true if a match is found for test_name in nightly_release, project, platform."""
+ patterns = ArtHeader(test_name).get('art-include')
+
+ for pattern in patterns:
+ nightly_release_pattern = "*"
+ project_pattern = "*"
+ platform_pattern = "*-*-*-opt"
+
+ count = pattern.count('/')
+ if count >= 2:
+ (nightly_release_pattern, project_pattern, platform_pattern) = pattern.split('/', 3)
+ elif count == 1:
+ (nightly_release_pattern, project_pattern) = pattern.split('/', 2)
+ else:
+ nightly_release_pattern = pattern
+
+ if fnmatch.fnmatch(nightly_release, nightly_release_pattern) and fnmatch.fnmatch(project, project_pattern) and fnmatch.fnmatch(platform, platform_pattern):
+ return True
+ return False
+
+ def get_input(self, input_name):
+ """Download input file from rucio. Retuns path of inputfile."""
+ work_dir = '.'
+
+ # run in correct environment
+ env = os.environ.copy()
+ env['PATH'] = '.:' + env['PATH']
+
+ (code, out, err) = run_command(os.path.join(self.art_directory, "art-get-input.sh") + " " + input_name, dir=work_dir, env=env)
+ if code == 0 and out != '':
+ return os.path.join(work_dir, input_name.replace(':', '/', 1))
+
+ return None
#
# Private Methods
diff --git a/Tools/ART/python/ART/art_build.py b/Tools/ART/python/ART/art_build.py
index 39fd869de8eb55332b171fdb7f2e6a200cc6750d..3c62cb3b11b5396ea66bea42f6b105dd9f4c13e0 100644
--- a/Tools/ART/python/ART/art_build.py
+++ b/Tools/ART/python/ART/art_build.py
@@ -5,10 +5,11 @@
__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
import collections
+import fnmatch
import json
import multiprocessing
-import re
import os
+import re
from art_misc import run_command, mkdir_p
from art_base import ArtBase
@@ -17,11 +18,12 @@ from art_header import ArtHeader
from parallelScheduler import ParallelScheduler
-def run_job(art_directory, sequence_tag, script_directory, package, type, index, test_name, nightly_release, project, platform, nightly_tag):
+def run_job(art_directory, sequence_tag, script_directory, package, job_type, index, test_name, nightly_release, project, platform, nightly_tag):
"""TBD."""
- print "job started", art_directory, sequence_tag, script_directory, package, type, index, test_name, nightly_release, project, platform, nightly_tag
- (exit_code, out, err) = run_command(os.path.join(art_directory, './art-internal.py') + " job build " + script_directory + " " + package + " " + type + " " + sequence_tag + " " + str(index) + " " + "out" + " " + nightly_release + " " + project + " " + platform + " " + nightly_tag)
- print "job ended", art_directory, sequence_tag, script_directory, package, type, index, test_name, nightly_release, project, platform, nightly_tag
+ print "job started", art_directory, sequence_tag, script_directory, package, job_type, index, test_name, nightly_release, project, platform, nightly_tag
+ (exit_code, out, err) = run_command(' '.join((os.path.join(art_directory, './art-internal.py'), "job", "build", script_directory, package, job_type, sequence_tag, str(index), "out", nightly_release, project, platform, nightly_tag)))
+ print "job ended", art_directory, sequence_tag, script_directory, package, job_type, index, test_name, nightly_release, project, platform, nightly_tag
+
return (test_name, exit_code, out, err)
@@ -30,6 +32,7 @@ class ArtBuild(ArtBase):
def __init__(self, art_directory, nightly_release, project, platform, nightly_tag, script_directory, max_jobs=0, ci=False):
"""TBD."""
+ super(ArtBuild, self).__init__(art_directory)
# print "ArtBuild", art_directory, script_directory, max_jobs
self.art_directory = art_directory
self.script_directory = script_directory.rstrip("/")
@@ -40,9 +43,9 @@ class ArtBuild(ArtBase):
self.max_jobs = multiprocessing.cpu_count() if max_jobs <= 0 else max_jobs
self.ci = ci
- def task_list(self, type, sequence_tag):
+ def task_list(self, job_type, sequence_tag):
"""TBD."""
- # print "task_list", type, sequence_tag
+ # print "task_list", job_type, sequence_tag
test_directories = self.get_test_directories(self.script_directory)
if not test_directories:
print 'WARNING: No tests found in directories ending in "test"'
@@ -50,11 +53,25 @@ class ArtBuild(ArtBase):
status = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.defaultdict()))
for package, root in test_directories.items():
- job_results = self.task(package, type, sequence_tag)
+ test_directory = os.path.abspath(test_directories[package])
+ job_results = self.task(package, job_type, sequence_tag)
for job_result in job_results:
- status[package][job_result[0]]['exit_code'] = job_result[1]
- status[package][job_result[0]]['out'] = job_result[2]
- status[package][job_result[0]]['err'] = job_result[3]
+ test_name = job_result[0]
+ status[package][test_name]['exit_code'] = job_result[1]
+ status[package][test_name]['out'] = job_result[2]
+ status[package][test_name]['err'] = job_result[3]
+ status[package][test_name]['test_directory'] = test_directory
+
+ # gather results
+ result = []
+ with open(os.path.join(sequence_tag, package, os.path.splitext(test_name)[0], 'stdout.txt'), 'r') as f:
+ for line in f:
+ match = re.search(r"art-result: (.*)", line)
+ if match:
+ result = json.loads(match.group(1))
+ break
+
+ status[package][job_result[0]]['result'] = result
mkdir_p(sequence_tag)
with open(os.path.join(sequence_tag, "status.json"), 'w') as outfile:
@@ -62,10 +79,10 @@ class ArtBuild(ArtBase):
return 0
- def task(self, package, type, sequence_tag):
+ def task(self, package, job_type, sequence_tag):
"""TBD."""
- # print "task", package, type, sequence_tag
- test_names = self.get_list(self.script_directory, package, type)
+ # print "task", package, job_type, sequence_tag
+ test_names = self.get_list(self.script_directory, package, job_type, "all")
scheduler = ParallelScheduler(self.max_jobs + 1)
index = 0
@@ -76,7 +93,7 @@ class ArtBuild(ArtBase):
branch_name = os.environ['AtlasBuildBranch']
cis = ArtHeader(fname).get('art-ci')
for ci in cis:
- if re.match(ci, branch_name):
+ if fnmatch.fnmatch(branch_name, ci):
schedule_test = True
break
else:
@@ -87,22 +104,23 @@ class ArtBuild(ArtBase):
print "job skipped, file not executable: ", fname
if schedule_test:
- scheduler.add_task(task_name="t" + str(index), dependencies=[], description="d", target_function=run_job, function_kwargs={'art_directory': self.art_directory, 'sequence_tag': sequence_tag, 'script_directory': self.script_directory, 'package': package, 'type': type, 'index': index, 'test_name': test_name, 'nightly_release': self.nightly_release, 'project': self.project, 'platform': self.platform, 'nightly_tag': self.nightly_tag})
+ scheduler.add_task(task_name="t" + str(index), dependencies=[], description="d", target_function=run_job, function_kwargs={'art_directory': self.art_directory, 'sequence_tag': sequence_tag, 'script_directory': self.script_directory, 'package': package, 'job_type': job_type, 'index': index, 'test_name': test_name, 'nightly_release': self.nightly_release, 'project': self.project, 'platform': self.platform, 'nightly_tag': self.nightly_tag})
index += 1
- return scheduler.run()
+ result = scheduler.run()
+ return result
- def job(self, package, type, sequence_tag, index, out):
+ def job(self, package, job_type, sequence_tag, index, out):
"""TBD."""
- # print "job", package, type, sequence_tag, index, out
+ # print "job", package, job_type, sequence_tag, index, out
test_directories = self.get_test_directories(self.script_directory)
test_directory = os.path.abspath(test_directories[package])
- test_name = self.get_files(test_directory, type)[int(index)]
+ test_name = self.get_files(test_directory, job_type)[int(index)]
work_directory = os.path.join(sequence_tag, package, os.path.splitext(test_name)[0])
mkdir_p(work_directory)
- (exit_code, output, err) = run_command(os.path.join(test_directory, test_name) + ' ' + '.' + ' ' + package + ' ' + type + ' ' + test_name + ' ' + self.nightly_release + ' ' + self.project + ' ' + self.platform + ' ' + self.nightly_tag, dir=work_directory)
+ (exit_code, output, err) = run_command(' '.join((os.path.join(test_directory, test_name), '.', package, job_type, test_name, self.nightly_release, self.project, self.platform, self.nightly_tag)), dir=work_directory)
with open(os.path.join(work_directory, "stdout.txt"), "w") as text_file:
text_file.write(output)
diff --git a/Tools/ART/python/ART/art_grid.py b/Tools/ART/python/ART/art_grid.py
index 4edb12a9ca7a37b0f0ae6ef0f59c7a4bc25e3c6b..35e032d525890ffacc3bfe4f2d15fbed8ff5ec16 100644
--- a/Tools/ART/python/ART/art_grid.py
+++ b/Tools/ART/python/ART/art_grid.py
@@ -18,7 +18,7 @@ import tempfile
from art_base import ArtBase
from art_header import ArtHeader
-from art_misc import mkdir_p, make_executable, check, run_command
+from art_misc import mkdir_p, make_executable, run_command, exit_on_failure
class ArtGrid(ArtBase):
@@ -26,7 +26,7 @@ class ArtGrid(ArtBase):
def __init__(self, art_directory, nightly_release, project, platform, nightly_tag, script_directory=None, skip_setup=False, submit_directory=None):
"""TBD."""
- self.art_directory = art_directory
+ super(ArtGrid, self).__init__(art_directory)
self.nightly_release = nightly_release
self.project = project
self.platform = platform
@@ -50,7 +50,7 @@ class ArtGrid(ArtBase):
"""Return true if the script directory is in cvmfs."""
return self.get_script_directory().startswith(self.cvmfs_directory)
- def task_list(self, type, sequence_tag):
+ def task_list(self, job_type, sequence_tag):
"""TBD."""
# job will be submitted from tmp directory
submit_directory = tempfile.mkdtemp(dir='.')
@@ -58,11 +58,9 @@ class ArtGrid(ArtBase):
# make sure tmp is removed afterwards
atexit.register(shutil.rmtree, submit_directory)
- config = None if self.skip_setup else self.get_config()
-
# make sure script directory exist
if not os.path.isdir(self.get_script_directory()):
- print 'ERROR: script directory does not exist: %s' % self.get_script_directory()
+ print 'ERROR: script directory does not exist:', self.get_script_directory()
print 'art-status: error'
sys.stdout.flush()
exit(1)
@@ -74,124 +72,192 @@ class ArtGrid(ArtBase):
sys.stdout.flush()
for package, root in test_directories.items():
- if self.excluded(config, package):
- print 'Excluding ' + package + ' for ' + self.nightly_release + ' project ' + self.project + ' on ' + self.platform
+ if package in ['TrigInDetValidation']:
+ continue
+
+ number_of_tests = len(self.get_files(root, job_type, "all", self.nightly_release, self.project, self.platform))
+ if number_of_tests > 0:
print 'art-package: ' + package
- print 'art-status: excluded'
+ print 'art-status: included'
+ print 'root' + root
+ print 'Handling', package, 'for', self.nightly_release, 'project', self.project, 'on', self.platform
+ print "Number of tests:", str(number_of_tests)
+ sys.stdout.flush()
+ submit_dir = os.path.join(submit_directory, package)
+ run = os.path.join(submit_dir, "run")
+ ART = os.path.join(run, "ART")
+ mkdir_p(ART)
+
+ # get the path of the python classes and support scripts
+ art_python_directory = os.path.join(self.art_directory, '..', 'python', 'ART')
+
+ shutil.copy(os.path.join(self.art_directory, 'art.py'), run)
+ shutil.copy(os.path.join(self.art_directory, 'art-get-input.sh'), run)
+ shutil.copy(os.path.join(self.art_directory, 'art-get-tar.sh'), run)
+ shutil.copy(os.path.join(self.art_directory, 'art-internal.py'), run)
+ shutil.copy(os.path.join(art_python_directory, '__init__.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_base.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_build.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_grid.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_header.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_misc.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'docopt.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'docopt_dispatch.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'parallelScheduler.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'serialScheduler.py'), ART)
+
+ make_executable(os.path.join(run, 'art.py'))
+ make_executable(os.path.join(run, 'art-get-input.sh'))
+ make_executable(os.path.join(run, 'art-get-tar.sh'))
+ make_executable(os.path.join(run, 'art-internal.py'))
+
+ # copy a local test directory if needed (only for 'art grid')
+ if not self.is_script_directory_in_cvmfs():
+ local_test_dir = os.path.join(run, os.path.basename(os.path.normpath(self.get_script_directory())))
+ print "Copying script directory for grid submission to ", local_test_dir
+ shutil.copytree(self.get_script_directory(), local_test_dir)
+
+ command = ' '.join((os.path.join(self.art_directory, 'art-internal.py'), 'task', 'grid', '--skip-setup' if self.skip_setup else '', submit_directory, self.get_script_directory(), package, job_type, sequence_tag, self.nightly_release, self.project, self.platform, self.nightly_tag))
+ print command
+ sys.stdout.flush()
+
+ env = os.environ.copy()
+ env['PATH'] = '.:' + env['PATH']
+ out = exit_on_failure(run_command(command, env=env))
+ print out
sys.stdout.flush()
- else:
- shell_files = self.get_files(root, type)
- number_of_tests = len(shell_files)
- if number_of_tests > 0:
- print 'art-package: ' + package
- print 'art-status: included'
- print 'root' + root
- print 'Handling ' + package + ' for ' + self.nightly_release + ' project ' + self.project + ' on ' + self.platform
- print "Number of tests: " + str(number_of_tests)
- sys.stdout.flush()
- submit_dir = os.path.join(submit_directory, package)
- run = os.path.join(submit_dir, "run")
- ART = os.path.join(run, "ART")
- mkdir_p(ART)
-
- # get the path of the python classes and support scripts
- art_python_directory = os.path.join(self.art_directory, '..', 'python', 'ART')
-
- shutil.copy(os.path.join(self.art_directory, 'art.py'), run)
- shutil.copy(os.path.join(self.art_directory, 'art-get-tar.sh'), run)
- shutil.copy(os.path.join(self.art_directory, 'art-internal.py'), run)
- shutil.copy(os.path.join(art_python_directory, '__init__.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'art_base.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'art_build.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'art_grid.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'art_header.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'art_misc.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'docopt.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'docopt_dispatch.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'parallelScheduler.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'serialScheduler.py'), ART)
-
- make_executable(os.path.join(run, 'art.py'))
- make_executable(os.path.join(run, 'art-get-tar.sh'))
- make_executable(os.path.join(run, 'art-internal.py'))
-
- # copy a local test directory if needed (only for 'art grid')
- if not self.is_script_directory_in_cvmfs():
- local_test_dir = os.path.join(run, os.path.basename(os.path.normpath(self.get_script_directory())))
- print "Copying script directory for grid submission to ", local_test_dir
- shutil.copytree(self.get_script_directory(), local_test_dir)
-
- command = os.path.join(self.art_directory, 'art-internal.py') + ' task grid ' + ('--skip-setup' if self.skip_setup else '') + ' ' + submit_directory + ' ' + self.get_script_directory() + ' ' + package + ' ' + type + ' ' + sequence_tag + ' ' + self.nightly_release + ' ' + self.project + ' ' + self.platform + ' ' + self.nightly_tag
- print command
- sys.stdout.flush()
-
- env = os.environ.copy()
- env['PATH'] = '.:' + env['PATH']
- out = check(run_command(command, env=env))
- print out
- sys.stdout.flush()
return 0
- def task(self, package, type, sequence_tag):
+ def task(self, package, job_type, sequence_tag):
"""TBD."""
print 'Running art task'
sys.stdout.flush()
- number_of_tests = len(self.get_list(self.get_script_directory(), package, type))
-
config = None if self.skip_setup else self.get_config()
grid_options = self.grid_option(config, package, 'grid-exclude-sites', '--excludedSite=')
grid_options += ' ' + self.grid_option(config, package, 'grid-sites', '--site=')
- print self.nightly_release + " " + self.project + " " + self.platform + " " + self.nightly_tag + " " + sequence_tag + " " + package + " " + type + " " + str(number_of_tests) + " " + grid_options
- sys.stdout.flush()
-
# run task from Bash Script as is needed in ATLAS setup
# FIXME we need to parse the output
env = os.environ.copy()
env['PATH'] = '.:' + env['PATH']
env['ART_GRID_OPTIONS'] = grid_options
- out = check(run_command(os.path.join(self.art_directory, 'art-task-grid.sh') + " " + ('--skip-setup' if self.skip_setup else '') + " " + self.submit_directory + " " + self.get_script_directory() + " " + package + " " + type + " " + sequence_tag + " " + str(number_of_tests) + " " + self.nightly_release + " " + self.project + " " + self.platform + " " + self.nightly_tag, env=env))
- print out
- sys.stdout.flush()
+
+ test_directories = self.get_test_directories(self.get_script_directory())
+ test_directory = test_directories[package]
+ number_of_batch_tests = len(self.get_files(test_directory, job_type, "batch", self.nightly_release, self.project, self.platform))
+
+ MAX_OUTFILE_LEN = 132
+ user = env['USER'] if self.skip_setup else 'artprod'
+ nightly_release_short = re.sub(r"-VAL-.*", "-VAL", self.nightly_release)
+ outfile = '.'.join(('user', user, 'atlas', nightly_release_short, self.project, self.platform, self.nightly_tag, sequence_tag, package))
+
+ # submit batch tests
+ if number_of_batch_tests > 0:
+ if len(outfile) > MAX_OUTFILE_LEN:
+ print "ERROR: OutFile string length >", MAX_OUTFILE_LEN, ": ", outfile
+ sys.stdout.flush()
+ return 1
+
+ print "batch:", nightly_release_short, self.project, self.platform, self.nightly_tag, sequence_tag, package, job_type, str(number_of_batch_tests), grid_options
+ sys.stdout.flush()
+
+ out = exit_on_failure(run_command(' '.join((os.path.join(self.art_directory, 'art-task-grid.sh'), '--skip-setup' if self.skip_setup else '', self.submit_directory, self.get_script_directory(), package, job_type, sequence_tag, str(number_of_batch_tests), nightly_release_short, self.project, self.platform, self.nightly_tag, outfile)), env=env))
+ print out
+ sys.stdout.flush()
+
+ # submit single tests
+ index = 1
+ for test_name in self.get_files(test_directory, job_type, "single", self.nightly_release, self.project, self.platform):
+ job = os.path.join(test_directory, test_name)
+ header = ArtHeader(job)
+ inds = header.get('art-input')
+ nFiles = header.get('art-input-nfiles')
+ nEvents = header.get('art-input-nevents')
+ files = ','.join(header.get('art-input-file'))
+ split = header.get('art-input-split')
+
+ outfile_test = '.'.join((outfile, str(index)))
+ if len(outfile_test) > MAX_OUTFILE_LEN:
+ print "ERROR: OutFile string length >", MAX_OUTFILE_LEN, ": ", outfile_test
+ sys.stdout.flush()
+ return 1
+
+ print "single:", nightly_release_short, self.project, self.platform, self.nightly_tag, sequence_tag, package, job_type, str(split), inds, str(nFiles), str(nEvents), grid_options
+ sys.stdout.flush()
+
+ out = exit_on_failure(run_command(' '.join((os.path.join(self.art_directory, 'art-task-grid.sh'), '--skip-setup' if self.skip_setup else '', '--test-name ' + test_name, '--inDS ' + inds, '--nFiles ' + str(nFiles) if nFiles > 0 else '', '--nEvents ' + str(nEvents) if nEvents > 0 else '', '--fileList ' + files if files != '' else '', self.submit_directory, self.get_script_directory(), package, job_type, sequence_tag, str(split), nightly_release_short, self.project, self.platform, self.nightly_tag, outfile_test)), env=env))
+ print out
+ sys.stdout.flush()
+
+ index += 1
+
return 0
- def job(self, package, type, sequence_tag, index, out):
+ def job(self, package, job_type, sequence_tag, index_type, index_or_name, out):
"""TBD."""
print 'Running art job grid'
sys.stdout.flush()
- index = int(index)
-
- print self.nightly_release + " " + self.project + " " + self.platform + " " + self.nightly_tag + " " + package + " " + type + " " + str(index) + " " + out
+ print self.nightly_release, self.project, self.platform, self.nightly_tag, package, job_type, str(index_or_name), out
sys.stdout.flush()
test_directories = self.get_test_directories(self.get_script_directory())
- test_dir = test_directories[package]
- test_list = self.get_files(test_dir, type)
+ test_directory = test_directories[package]
+ if index_type == "batch":
+ test_list = self.get_files(test_directory, job_type, "batch", self.nightly_release, self.project, self.platform)
+
+ # minus one for grid
+ index = int(index_or_name)
+ test_name = test_list[index - 1]
+ else:
+ test_name = index_or_name
- # minus one for grid
- test_name = test_list[index - 1]
- test_file = os.path.join(test_dir, test_name)
- com = '%s %s %s %s %s %s %s %s %s' % (test_file, self.get_script_directory(), package, type, test_name, self.nightly_release, self.project, self.platform, self.nightly_tag)
+ test_file = os.path.join(test_directory, test_name)
+ command = ' '.join((test_file, self.get_script_directory(), package, job_type, test_name, self.nightly_release, self.project, self.platform, self.nightly_tag))
print test_name
- print test_dir
- print com
+ print test_directory
+ print command
sys.stdout.flush()
# run the test
env = os.environ.copy()
env['PATH'] = '.:' + env['PATH']
- print check(run_command(com, env=env))
+ (exit_code, output, error) = run_command(command, env=env)
+ if (exit_code != 0):
+ print error
+ print output
sys.stdout.flush()
+ # gather results
+ result = {}
+ result['name'] = test_name
+ result['exit_code'] = exit_code
+ result['test_directory'] = test_directory
+
+ # find all 'art-result: x' or 'art-result: [x]' and append them to result list
+ result['result'] = []
+ matches = re.findall(r"art-result: (.*)", output)
+ for match in matches:
+ item = json.loads(match)
+ if isinstance(item, list):
+ result['result'].extend(item)
+ else:
+ result['result'].append(item)
+
+ # write out results
+ with open(os.path.join("art-job.json"), 'w') as jobfile:
+ json.dump(result, jobfile, sort_keys=True, indent=4, ensure_ascii=False)
+
# pick up the outputs
tar_file = tarfile.open(out, mode='w')
# pick up explicitly named output files
with open(test_file, "r") as f:
for line in f:
+ # remove comments
+ line = line.split('#', 1)[0]
out_names = re.findall(r"--output[^\s=]*[= ]*(\S*)", line)
print out_names
for out_name in out_names:
@@ -203,15 +269,16 @@ class ArtGrid(ArtBase):
# pick up art-header named outputs
for path_name in ArtHeader(test_file).get('art-output'):
for out_name in glob.glob(path_name):
- print 'Tar file contain: ', out_name
+ print 'Tar file contains:', out_name
tar_file.add(out_name)
tar_file.close()
+ # Always return 0
return 0
- def list(self, package, type, json_format=False):
+ def list(self, package, job_type, index_type, json_format=False):
"""TBD."""
- jobs = self.get_list(self.get_script_directory(), package, type)
+ jobs = self.get_list(self.get_script_directory(), package, job_type, index_type)
if json_format:
json.dump(jobs, sys.stdout, sort_keys=True, indent=4)
return 0
@@ -237,7 +304,7 @@ class ArtGrid(ArtBase):
def output(self, package, test_name, file_name):
"""TBD."""
- tar = self.get_tar(package, test_name, '_EXT0')
+ tar = self.get_tar(package, test_name, '_EXT1')
for member in tar.getmembers():
if file_name in member.name:
@@ -258,7 +325,7 @@ class ArtGrid(ArtBase):
ref_dir = os.path.join('.', 'ref-' + previous_nightly_tag)
mkdir_p(ref_dir)
- tar = self.get_tar(package, test_name, '_EXT0', previous_nightly_tag)
+ tar = self.get_tar(package, test_name, '_EXT1', previous_nightly_tag)
if tar is None:
print "ERROR: No comparison tar file found"
# do not flag as error, to make sure tar file gets uploaded
@@ -272,68 +339,31 @@ class ArtGrid(ArtBase):
for file_name in file_names:
ref_file = os.path.join(ref_dir, file_name)
if os.path.isfile(ref_file):
- print "art-compare: " + previous_nightly_tag + " " + file_name
+ print "art-compare:", previous_nightly_tag, file_name
self.compare_ref(file_name, ref_file, 10)
else:
print "ERROR:", ref_file, "not found in tar file"
# do not flag as error, to make sure tar file gets uploaded
return 0
- #
- # Protected Methods
- #
- def excluded(self, config, package):
- """Based on config, decide if a release is excluded from testing."""
- # NOTE changes in here should be reflected in art-submit/art-gitlab.py
- if config is None:
- return False
-
- if self.nightly_release not in config['releases'].keys():
- return True
-
- if self.project not in config['releases'][self.nightly_release].keys():
- return True
-
- if self.platform not in config['releases'][self.nightly_release][self.project].keys():
- return True
-
- # no packages listed -> included
- if config['releases'][self.nightly_release][self.project][self.platform] is None:
- return False
-
- # package not listed -> included
- if package not in config['releases'][self.nightly_release][self.project][self.platform].keys():
- return False
-
- return config['releases'][self.nightly_release][self.project][self.platform][package].get('excluded', False)
-
def grid_option(self, config, package, key, option_key):
- """Based on config, return value for key, or ''."""
- if config is None:
- return ''
-
- if self.nightly_release not in config['releases'].keys():
- return ''
-
- if self.project not in config['releases'][self.nightly_release].keys():
- return ''
+ """Based on config, return value for key, or ''.
- if self.platform not in config['releases'][self.nightly_release][self.project].keys():
- return ''
-
- if config['releases'][self.nightly_release][self.project][self.platform] is None:
+ A global value is pre-pended if found. If not local value is found only the global value is returned, or ''.
+ """
+ if config is None:
return ''
- prefix = config.get(key, None)
- if package not in config['releases'][self.nightly_release][self.project][self.platform].keys():
- return '' if prefix is None else option_key + prefix
+ global_value = config.get(key)
+ if package not in config.keys():
+ return '' if global_value is None else option_key + global_value
- value = config['releases'][self.nightly_release][self.project][self.platform][package].get(key, None)
+ value = config.get(package).get(key)
- if prefix is None:
+ if global_value is None:
return '' if value is None else option_key + value
else:
- return option_key + prefix + ('' if value is None else ', ' + value)
+ return option_key + global_value + ('' if value is None else ', ' + value)
def get_tar(self, package, test_name, extension, nightly_tag=None):
"""Open tar file for particular release."""
@@ -341,9 +371,9 @@ class ArtGrid(ArtBase):
nightly_tag = self.nightly_tag
try:
- type = self.get_type(self.get_test_directories(self.get_script_directory())[package], test_name)
- print "Type:", type
- files = self.get_list(self.get_script_directory(), package, type)
+ job_type = self.get_type(self.get_test_directories(self.get_script_directory())[package], test_name)
+ print "Job Type:", job_type
+ files = self.get_list(self.get_script_directory(), package, job_type, "batch")
number_of_tests = len(files)
index = files.index(test_name)
print "Index:", index
@@ -371,7 +401,7 @@ class ArtGrid(ArtBase):
try_index = (retry * number_of_tests) + index
print (retry + 1), ": Get tar for index ", try_index
# run art-get-tar.sh
- (code, out, err) = run_command(os.path.join(self.art_directory, "art-get-tar.sh") + " " + package + " " + str(try_index) + " _EXT0 " + self.nightly_release + " " + self.project + " " + self.platform + " " + nightly_tag, dir=tmpdir, env=env)
+ (code, out, err) = run_command(' '.join((os.path.join(self.art_directory, "art-get-tar.sh"), package, str(try_index), "_EXT1", self.nightly_release, self.project, self.platform, nightly_tag)), dir=tmpdir, env=env)
if code == 0 and out != '':
match = re.search(r"TAR_NAME=(.*)", out)
@@ -386,9 +416,9 @@ class ArtGrid(ArtBase):
retry += 1
if retry >= retries:
- print "Code: " + str(code)
- print "Err: " + err
- print "Out: " + out
+ print "Code:", str(code)
+ print "Err:", err
+ print "Out:", out
return None
return tarfile.open(os.path.join(tmpdir, tar_name.replace(':', '/', 1)))
diff --git a/Tools/ART/python/ART/art_header.py b/Tools/ART/python/ART/art_header.py
index 6bfecfeea510e6b01a23ba75670f200212a72874..046de1b53d7ae113a96cd6ffd266501063149acd 100644
--- a/Tools/ART/python/ART/art_header.py
+++ b/Tools/ART/python/ART/art_header.py
@@ -6,7 +6,7 @@ __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
import re
-from copy import copy
+from types import IntType
from types import ListType
from types import StringType
@@ -16,59 +16,78 @@ class ArtHeader(object):
def __init__(self, filename):
"""TBD."""
- self.header = re.compile(r'#\s(art-[\w-]+):\s+(.+)$')
+ self.header_format = re.compile(r'#\s(art-[\w-]+):\s+(.+)$')
self.header_format_error1 = re.compile(r'#(art-[\w-]+):\s*(.+)$')
self.header_format_error2 = re.compile(r'#\s\s+(art-[\w-]+):\s*(.+)$')
self.header_format_error3 = re.compile(r'#\s(art-[\w-]+):\S(.*)$')
self.filename = filename
- self.type = {}
- self.defaultValue = {}
- self.constraint = {}
- self.value = {}
+ self.header = {}
+ # general
self.add('art-description', StringType, '')
self.add('art-type', StringType, None, ['build', 'grid'])
+
+ # "build" type only
self.add('art-ci', ListType, [])
+
+ # "grid" type only
+ self.add('art-include', ListType, ['*'])
self.add('art-output', ListType, [])
+ self.add('art-input', StringType, None)
+ self.add('art-input-file', ListType, [])
+ self.add('art-input-nfiles', IntType, 0)
+ self.add('art-input-nevents', IntType, 0)
+ self.add('art-input-split', IntType, 0)
self.read(filename)
- def add(self, key, type, defaultValue=None, constraint=None):
+ def add(self, key, value_type, default_value=None, constraint=None):
"""TBD."""
- self.type[key] = type
- self.defaultValue[key] = defaultValue
- self.constraint[key] = constraint
- self.value[key] = copy(defaultValue)
+ self.header[key] = {}
+ self.header[key]['type'] = value_type
+ self.header[key]['default'] = default_value
+ self.header[key]['constraint'] = constraint
+ self.header[key]['value'] = None # never set
def is_list(self, key):
"""TBD."""
- return self.type[key] is ListType if key in self.type else False
+ return self.header[key]['type'] is ListType if key in self.header else False
def read(self, filename):
"""Read all headers from file."""
for line in open(filename, "r"):
- line_match = self.header.match(line)
+ line_match = self.header_format.match(line)
if line_match:
key = line_match.group(1)
value = line_match.group(2)
- if self.type[key] == StringType:
+ if key in self.header and self.header[key]['type'] == StringType:
value = value.strip()
if self.is_list(key):
- self.value[key].append(value)
+ if self.header[key]['value'] is None:
+ self.header[key]['value'] = []
+ self.header[key]['value'].append(value)
else:
- self.value[key] = value
+ if key not in self.header:
+ self.header[key] = {}
+ self.header[key]['value'] = value
def get(self, key):
"""TBD."""
- return self.value[key] if key in self.value else None
+ if key not in self.header:
+ return None
+
+ if self.header[key]['value'] is None:
+ return self.header[key]['default']
+
+ return self.header[key]['value']
def print_it(self):
"""TBD."""
- for key in self.type:
- print key, self.type[key], self.defaultValue[key], self.value[key], self.constraint[key]
+ for key in self.header:
+ print key, self.header[key]['type'], self.header[key]['default'], self.header[key]['value'], self.header[key]['constraint']
def validate(self):
"""
@@ -78,7 +97,7 @@ class ArtHeader(object):
- a header is spaced correctly (e.g. '#art-header: value')
- a value in a header is not spaced correctly (e.g. '# art-header:value')
- a key is found which is not defined
- - a value is found of the wrong type
+ - a value is found of the wrong value_type
- a value is found outside the constraint
"""
for line in open(self.filename, "r"):
@@ -95,20 +114,20 @@ class ArtHeader(object):
print "ERROR: Header Validation - invalid header format, use at least one space between ': and value' in file", self.filename
print
- for key in self.value:
- if key not in self.type:
- print "ERROR: Header Validation - Invalid key: '" + key + "' in file", self.filename
+ for key in self.header:
+ if 'type' not in self.header[key]:
+ print "ERROR: Header Validation - Invalid key:", key, "in file", self.filename
print
continue
- if type(self.value[key]) != self.type[key]:
- if not isinstance(self.value[key], type(None)):
- print "ERROR: Header Validation - type:", type(self.value[key]), "not valid for key:", key, "expected type:", self.type[key], "in file", self.filename
+ if type(self.header[key]['value']) != self.header[key]['type']:
+ if not isinstance(self.header[key]['value'], type(None)):
+ print "ERROR: Header Validation - value_type:", type(self.header[key]['value']), "not valid for key:", key, "expected value_type:", self.header[key]['type'], "in file", self.filename
print
- if self.constraint[key] is not None and self.value[key] not in self.constraint[key]:
- if self.value[key] is None:
- print "ERROR: Header Validation - missing key: '" + key + "' in file", self.filename
+ if self.header[key]['constraint'] is not None and self.header[key]['value'] not in self.header[key]['constraint']:
+ if self.header[key]['value'] is None:
+ print "ERROR: Header Validation - missing key:", key, "in file", self.filename
else:
- print "ERROR: Header Validation - value: '" + self.value[key] + "' for key: '" + key + "' not in constraints:", self.constraint[key], "in file", self.filename
+ print "ERROR: Header Validation - value:", self.header[key]['value'], "for key:", key, "not in constraints:", self.header[key]['constraint'], "in file", self.filename
print
return 0
diff --git a/Tools/ART/python/ART/art_misc.py b/Tools/ART/python/ART/art_misc.py
index 3b98072b024f697004b26ce07d025493f82089cc..edb77fe5ebe1783f121b46d7b5a40e240254b2db 100644
--- a/Tools/ART/python/ART/art_misc.py
+++ b/Tools/ART/python/ART/art_misc.py
@@ -33,39 +33,44 @@ def run_command(cmd, dir=None, shell=False, env=None):
return exit_code, str(output), str(err)
-def check((exitcode, out, err)):
+def exit_on_failure((exit_code, out, err)):
"""Check exitcode and print statement and exit if needed."""
- if exitcode == 0:
+ if exit_code == 0:
print err
return out
- print "Error:", exitcode
+ print "Error:", exit_code
print "StdOut:", out
print "StdErr:", err
print 'art-status: error'
- exit(exitcode)
+ exit(exit_code)
-def verify((exitcode, out, err)):
+def code_tbr((exit_code, out, err)):
"""Check exitcode and print statement."""
- if exitcode == 0:
+ if exit_code == 0:
print out
- return exitcode
+ return exit_code
- print "Error:", exitcode
+ print "Error:", exit_code
print "StdOut:", out
print "StdErr:", err
- return exitcode
+ return exit_code
-def redirect((exitcode, out, err)):
+def redirect_tbr((exitcode, out, err)):
"""Check exitcode."""
return exitcode
+def is_exe(fpath):
+ """Return True if fpath is executable."""
+ return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+
def make_executable(path):
"""Make file executable (chmod +x)."""
mode = os.stat(path).st_mode
diff --git a/Tools/ART/scripts/art-get-input.sh b/Tools/ART/scripts/art-get-input.sh
new file mode 100755
index 0000000000000000000000000000000000000000..5a6115e59381066240bf418ee362cfbfb0e6e544
--- /dev/null
+++ b/Tools/ART/scripts/art-get-input.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+#
+# NOTE do NOT run with /bin/bash -x as the output is too big for gitlab-ci
+# arguments: INPUTNAME
+#
+# author : Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>
+#
+# example: CONTAINER_ID:ENTRY_NAME
+
+if [ $# -ne 1 ]; then
+ echo 'Usage: art-get-input.sh INPUTNAME'
+ exit 1
+fi
+
+echo "Script executed by $(whoami) on $(date)"
+echo "User: ${USER}"
+
+INPUTNAME=$1
+shift
+
+export ATLAS_LOCAL_ROOT_BASE=/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase
+source $ATLAS_LOCAL_ROOT_BASE/user/atlasLocalSetup.sh
+
+unset ALRB_noGridMW
+
+lsetup -f rucio
+
+echo "InputName: ${INPUTNAME}"
+
+# Do not use: rucio delivers warnings as exit code 127
+#set -e
+
+rucio download ${INPUTNAME}
diff --git a/Tools/ART/scripts/art-get-tar.sh b/Tools/ART/scripts/art-get-tar.sh
index e5c1e302a2c852f2b2d58c4f5273371a3f86e754..23de65640703a42cc83f6497ea197d2092071917 100755
--- a/Tools/ART/scripts/art-get-tar.sh
+++ b/Tools/ART/scripts/art-get-tar.sh
@@ -3,7 +3,6 @@
#
# NOTE do NOT run with /bin/bash -x as the output is too big for gitlab-ci
# arguments: PACKAGE INDEX EXTENSION NIGHTLY_RELEASE PROJECT PLATFORM NIGHTLY_TAG
-# env: ART_GRID_OPTIONS
#
# author : Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>
#
diff --git a/Tools/ART/scripts/art-internal.py b/Tools/ART/scripts/art-internal.py
index 52984aa30306196fe0b2a7695aacbc4d00d542f2..029ac3b1e559863b12e0ec70079bacf03cdafbfe 100755
--- a/Tools/ART/scripts/art-internal.py
+++ b/Tools/ART/scripts/art-internal.py
@@ -5,7 +5,7 @@ ART-internal - ATLAS Release Tester (internal command).
Usage:
art-internal.py job build [-v] <script_directory> <package> <job_type> <sequence_tag> <index> <out> <nightly_release> <project> <platform> <nightly_tag>
- art-internal.py job grid [-v --skip-setup] <script_directory> <package> <job_type> <sequence_tag> <index> <out> <nightly_release> <project> <platform> <nightly_tag>
+ art-internal.py job grid [-v --skip-setup] <script_directory> <package> <job_type> <sequence_tag> <index_type> <index_or_name> <out> <nightly_release> <project> <platform> <nightly_tag>
art-internal.py task build [-v] <script_directory> <package> <job_type> <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
art-internal.py task grid [-v --skip-setup] <submit_directory> <script_directory> <package> <job_type> <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
@@ -20,7 +20,9 @@ Sub-commands:
task Runs a single task, consisting of given number of jobs
Arguments:
+ index_type Type of index used (e.g. batch or single)
index Index of the test inside the package
+ index_or_name Index of the test (batch), or its name (single)
nightly_release Name of the nightly release (e.g. 21.0)
nightly_tag Nightly tag (e.g. 2017-02-26T2119)
out Tar filename used for the output of the job
@@ -55,7 +57,7 @@ def job_build(script_directory, package, job_type, sequence_tag, index, out, nig
@dispatch.on('job', 'grid')
-def job_grid(script_directory, package, job_type, sequence_tag, index, out, nightly_release, project, platform, nightly_tag, **kwargs):
+def job_grid(script_directory, package, job_type, sequence_tag, index_type, index_or_name, out, nightly_release, project, platform, nightly_tag, **kwargs):
"""TBD.
Tests are called with the following parameters:
@@ -63,14 +65,14 @@ def job_grid(script_directory, package, job_type, sequence_tag, index, out, nigh
"""
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
skip_setup = kwargs['skip_setup']
- exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, skip_setup).job(package, job_type, sequence_tag, index, out))
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, skip_setup).job(package, job_type, sequence_tag, index_type, index_or_name, out))
@dispatch.on('task', 'build')
-def task_build(script_directory, job_type, sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
+def task_build(script_directory, package, job_type, sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
"""TBD."""
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
- exit(ArtBuild(art_directory, nightly_release, project, platform, nightly_tag, script_directory).task(job_type, sequence_tag))
+ exit(ArtBuild(art_directory, nightly_release, project, platform, nightly_tag, script_directory).task(package, job_type, sequence_tag))
@dispatch.on('task', 'grid')
diff --git a/Tools/ART/scripts/art-task-grid.sh b/Tools/ART/scripts/art-task-grid.sh
index 606056c0df311cdb81e382982e7466e2b7733b43..2c15ff91e7c0452447a7bc604218c9339c2eff00 100755
--- a/Tools/ART/scripts/art-task-grid.sh
+++ b/Tools/ART/scripts/art-task-grid.sh
@@ -2,12 +2,14 @@
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
#
# NOTE do NOT run with /bin/bash -x as the output is too big for gitlab-ci
-# arguments: [options] SUBMIT_DIRECTORY SCRIPT_DIRECTORY PACKAGE SEQUENCE_TAG NUMBER_OF_TESTS NIGHTLY_RELEASE PROJECT PLATFORM NIGHTLY_TAG
+# arguments: [options] SUBMIT_DIRECTORY SCRIPT_DIRECTORY PACKAGE SEQUENCE_TAG SPLIT NIGHTLY_RELEASE_SHORT PROJECT PLATFORM NIGHTLY_TAG OUT_FILE
# env: ART_GRID_OPTIONS
#
# author : Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>
#
-# example: [--skip-setup] tmp /cvmfs/atlas-nightlies.cern.ch/sw/... Tier0ChainTests grid 316236 32 21.0 Athena x86_64-slc6-gcc62-opt 2017-02-26T2119
+# options have to be in-order
+#
+# example: [--skip-setup --test-name TestName --inDS user.tcuhadar.SingleMuon... --nFiles 3 --nEventsPerFile 5] tmp /cvmfs/atlas-nightlies.cern.ch/sw/... Tier0ChainTests grid 316236 3 21.0 Athena x86_64-slc6-gcc62-opt 2017-02-26T2119 user.${USER}.atlas.${NIGHTLY_RELEASE_SHORT}.${PROJECT}.${PLATFORM}.${NIGHTLY_TAG}.${SEQUENCE_TAG}.${PACKAGE}[.${TEST_NUMBER}]
#set -e
echo "Script executed by $(whoami) on $(date)"
@@ -17,6 +19,38 @@ if [ $1 == "--skip-setup" ]; then
SKIP_SETUP=1
shift
fi
+TYPE_OPTION="batch %RNDM:0"
+PATHENA_OPTIONS="--destSE=CERN-PROD_SCRATCHDISK"
+if [ $1 == "--test-name" ]; then
+ TYPE_OPTION="single $2"
+ PATHENA_OPTIONS="--destSE=CERN-PROD_SCRATCHDISK --forceStaged"
+ shift
+ shift
+fi
+INDS=""
+if [ $1 == "--inDS" ]; then
+ INDS="--inDS $2"
+ shift
+ shift
+fi
+NFILES=""
+if [ $1 == "--nFiles" ]; then
+ NFILES="--nFiles $2"
+ shift
+ shift
+fi
+NEVENTS=""
+if [ $1 == "--nEvents" ]; then
+ NEVENTS="--nEventsPerFile $2"
+ shift
+ shift
+fi
+FILELIST=""
+if [ $1 == "--fileList" ]; then
+ FILELIST="--fileList $2"
+ shift
+ shift
+fi
SUBMIT_DIRECTORY=$1
shift
SCRIPT_DIRECTORY=$1
@@ -27,9 +61,9 @@ TYPE=$1
shift
SEQUENCE_TAG=$1
shift
-NUMBER_OF_TESTS=$1
+SPLIT=$1
shift
-NIGHTLY_RELEASE=$1
+NIGHTLY_RELEASE_SHORT=$1
shift
PROJECT=$1
shift
@@ -37,13 +71,12 @@ PLATFORM=$1
shift
NIGHTLY_TAG=$1
shift
+OUTFILE=$1
+shift
# we seem to have to copy the env variables locally
GRID_OPTIONS=$ART_GRID_OPTIONS
-# change -VAL-Prod and others into -VAL
-NIGHTLY_RELEASE_SHORT=${NIGHTLY_RELEASE/-VAL-*/-VAL}
-
if [ ${SKIP_SETUP} -eq 0 ]; then
echo "Setting up release: ${PLATFORM} ${NIGHTLY_RELEASE_SHORT} ${NIGHTLY_TAG} ${PROJECT}"
USER=artprod
@@ -59,14 +92,22 @@ if [ ${SKIP_SETUP} -eq 0 ]; then
fi
+if [ ${SPLIT} -eq 0 ]; then
+ SPLIT=""
+else
+ SPLIT="--split ${SPLIT}"
+fi
+
# NOTE: for art-internal.py the current dir can be used as it is copied there
cd ${SUBMIT_DIRECTORY}/${PACKAGE}/run
-OUTFILE="user.${USER}.atlas.${NIGHTLY_RELEASE_SHORT}.${PROJECT}.${PLATFORM}.${NIGHTLY_TAG}.${SEQUENCE_TAG}.${PACKAGE}"
-CMD="pathena ${GRID_OPTIONS} --noBuild --expertOnly_skipScout --trf \"./art-internal.py job grid ${SCRIPT_DIRECTORY} ${PACKAGE} ${TYPE} ${SEQUENCE_TAG} %RNDM:0 %OUT.tar ${NIGHTLY_RELEASE_SHORT} ${PROJECT} ${PLATFORM} ${NIGHTLY_TAG}\" --split ${NUMBER_OF_TESTS} --outDS ${OUTFILE}"
+SUBCOMMAND="./art-internal.py job grid ${SCRIPT_DIRECTORY} ${PACKAGE} ${TYPE} ${SEQUENCE_TAG} ${TYPE_OPTION} %OUT.tar ${NIGHTLY_RELEASE_SHORT} ${PROJECT} ${PLATFORM} ${NIGHTLY_TAG}"
+CMD="pathena ${GRID_OPTIONS} ${PATHENA_OPTIONS} --noBuild --expertOnly_skipScout --trf \"${SUBCOMMAND}\" ${SPLIT} --outDS ${OUTFILE} --extOutFile art-job.json ${INDS} ${NFILES} ${NEVENTS} ${FILELIST}"
+
#--disableAutoRetry
#--excludedSite=ANALY_TECHNION-HEP-CREAM
#--site=ANALY_NIKHEF-ELPROD_SHORT,ANALY_NIKHEF-ELPROD"
#--site=ANALY_FZK,ANALY_BNL,ANALY_RAL"
+
echo ${CMD}
RESULT=`eval "${CMD}"`
diff --git a/Tools/ART/scripts/art.py b/Tools/ART/scripts/art.py
index ac0fc5a1bbeea436a232f46f5999341f063ee786..9a0da14c933f88fded8f57ea55f36d1728e0a2d7 100755
--- a/Tools/ART/scripts/art.py
+++ b/Tools/ART/scripts/art.py
@@ -4,22 +4,25 @@
ART - ATLAS Release Tester.
Usage:
- art.py run [-v --type=<T> --max-jobs=<N> --ci] <script_directory> <sequence_tag>
- art.py grid [-v --type=<T>] <script_directory> <sequence_tag>
- art.py submit [-v --type=<T>] <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
- art.py validate [-v] <script_directory>
- art.py compare grid [-v --days=<D>] <nightly_release> <project> <platform> <nightly_tag> <package> <test_name> <file_name>...
+ art.py run [-v --type=<T> --max-jobs=<N> --ci] <script_directory> <sequence_tag>
+ art.py grid [-v --type=<T>] <script_directory> <sequence_tag>
+ art.py submit [-v --type=<T>] <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
+ art.py validate [-v] <script_directory>
+ art.py included [-v --type=<T> --test-type=<TT>] <script_directory> [<nightly_release> <project> <platform>]
+ art.py compare grid [-v --days=<D>] <nightly_release> <project> <platform> <nightly_tag> <package> <test_name> <file_name>...
art.py compare ref [-v] <file_name> <ref_file>
- art.py list grid [-v --json --type=<T>] <package> <nightly_release> <project> <platform> <nightly_tag>
- art.py log grid [-v] <package> <test_name> <nightly_release> <project> <platform> <nightly_tag>
- art.py output grid [-v] <package> <test_name> <file_name> <nightly_release> <project> <platform> <nightly_tag>
+ art.py download [-v] <input_file>
+ art.py list grid [-v --json --type=<T> --test-type=<TT>] <package> <nightly_release> <project> <platform> <nightly_tag>
+ art.py log grid [-v] <package> <test_name> <nightly_release> <project> <platform> <nightly_tag>
+ art.py output grid [-v] <package> <test_name> <file_name> <nightly_release> <project> <platform> <nightly_tag>
Options:
--ci Run Continuous Integration tests only (using env: AtlasBuildBranch)
--days=<D> Number of days ago to pick up reference for compare [default: 1]
--json Output in json format
- --max-jobs=<N> Maximum number of concurrent jobs to run
+ --max-jobs=<N> Maximum number of concurrent jobs to run [default: 0]
--type=<T> Type of job (e.g. grid, build)
+ --test-type=<TT> Type of test (e.g. all, batch or single)
-h --help Show this screen.
-v, --verbose Show details.
--version Show version.
@@ -29,7 +32,9 @@ Sub-commands:
grid Run jobs from a package on the grid (needs release and grid setup)
submit Submit nightly jobs to the grid (NOT for users)
validate Check headers in tests
+ included Shows list of files which will be included for art submit/art grid
compare Compare the output of a job
+ download Download a file from rucio
list Lists the jobs of a package
log Show the log of a job
output Get the output of a job
@@ -37,6 +42,7 @@ Sub-commands:
Arguments:
file_name Filename to save the output to
index Index of the test inside the package
+ input_file Input file to download (e.g. CONTAINER_ID:ENTRY_NAME)
nightly_release Name of the nightly release (e.g. 21.0)
nightly_tag Nightly tag (e.g. 2017-02-26T2119)
out Tar filename used for the output of the job
@@ -46,11 +52,10 @@ Arguments:
script_directory Directory containing the package(s) with tests
sequence_tag Sequence tag (e.g. 0 or PIPELINE_ID)
test_name Name of the test inside the package (e.g. test_q322.sh)
- job_type Type of job (e.g. grid, build)
"""
__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
-__version__ = '0.0.7'
+__version__ = '0.5.4'
import os
import sys
@@ -67,7 +72,8 @@ from ART import ArtBase, ArtGrid, ArtBuild
@dispatch.on('compare', 'ref')
def compare_ref(file_name, ref_file, **kwargs):
"""TBD."""
- exit(ArtBase().compare_ref(file_name, ref_file))
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+ exit(ArtBase(art_directory).compare_ref(file_name, ref_file))
@dispatch.on('compare', 'grid')
@@ -82,10 +88,11 @@ def compare_grid(package, test_name, nightly_release, project, platform, nightly
@dispatch.on('list', 'grid')
def list(package, nightly_release, project, platform, nightly_tag, **kwargs):
"""TBD."""
- json_format = kwargs['json']
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
job_type = 'grid' if kwargs['type'] is None else kwargs['type']
- exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).list(package, job_type, json_format))
+ index_type = 'all' if kwargs['test_type'] is None else kwargs['test_type']
+ json_format = kwargs['json']
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).list(package, job_type, index_type, json_format))
@dispatch.on('log', 'grid')
@@ -106,8 +113,8 @@ def output(package, test_name, file_name, nightly_release, project, platform, ni
def submit(sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
"""TBD."""
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
- type = 'grid' if kwargs['type'] is None else kwargs['type']
- exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).task_list(type, sequence_tag))
+ job_type = 'grid' if kwargs['type'] is None else kwargs['type']
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).task_list(job_type, sequence_tag))
@dispatch.on('grid')
@@ -122,8 +129,8 @@ def grid(script_directory, sequence_tag, **kwargs):
except KeyError, e:
print "Environment variable not set", e
sys.exit(1)
- type = 'grid' if kwargs['type'] is None else kwargs['type']
- exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, True).task_list(type, sequence_tag))
+ art_type = 'grid' if kwargs['type'] is None else kwargs['type']
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, True).task_list(art_type, sequence_tag))
@dispatch.on('run')
@@ -138,14 +145,34 @@ def run(script_directory, sequence_tag, **kwargs):
except KeyError, e:
print "Environment variable not set", e
sys.exit(1)
- type = 'build' if kwargs['type'] is None else kwargs['type']
- exit(ArtBuild(art_directory, nightly_release, project, platform, nightly_tag, script_directory, max_jobs=kwargs['max_jobs'], ci=kwargs['ci']).task_list(type, sequence_tag))
+ art_type = 'build' if kwargs['type'] is None else kwargs['type']
+ exit(ArtBuild(art_directory, nightly_release, project, platform, nightly_tag, script_directory, max_jobs=int(kwargs['max_jobs']), ci=kwargs['ci']).task_list(art_type, sequence_tag))
@dispatch.on('validate')
def validate(script_directory, **kwargs):
"""TBD."""
- exit(ArtBase().validate(script_directory))
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+ exit(ArtBase(art_directory).validate(script_directory))
+
+
+@dispatch.on('included')
+def included(script_directory, **kwargs):
+ """TBD."""
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+ nightly_release = os.environ['AtlasBuildBranch'] if kwargs['nightly_release'] is None else kwargs['nightly_release']
+ project = os.environ['AtlasProject'] if kwargs['project'] is None else kwargs['project']
+ platform = os.environ[project + '_PLATFORM'] if kwargs['platform'] is None else kwargs['platform']
+ art_type = 'grid' if kwargs['type'] is None else kwargs['type']
+ index_type = 'all' if kwargs['test_type'] is None else kwargs['test_type']
+ exit(ArtBase(art_directory).included(script_directory, art_type, index_type, nightly_release, project, platform))
+
+
+@dispatch.on('download')
+def download(input_file, **kwargs):
+ """TBD."""
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+ exit(ArtBase(art_directory).download(input_file))
if __name__ == '__main__':