diff --git a/Tools/ART/python/ART/__init__.py b/Tools/ART/python/ART/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..423193cc52bd18a9327792eafce9142b186b7bfc
--- /dev/null
+++ b/Tools/ART/python/ART/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+"""TBD."""
+
+from art_base import ArtBase
+from art_build import ArtBuild
+from art_grid import ArtGrid
diff --git a/Tools/ART/python/art_base.py b/Tools/ART/python/ART/art_base.py
similarity index 78%
rename from Tools/ART/python/art_base.py
rename to Tools/ART/python/ART/art_base.py
index 72658267b2b89f89226912b2f01b569b6f40c28c..3fdbab8190d57da992396481b2289b1d0bf29ed4 100755
--- a/Tools/ART/python/art_base.py
+++ b/Tools/ART/python/ART/art_base.py
@@ -1,16 +1,17 @@
#!/usr/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
"""TBD."""
+
__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
import fnmatch
import inspect
import os
-import re
import sys
import yaml
from art_misc import check, run_command
+from art_header import ArtHeader
class ArtBase(object):
@@ -48,13 +49,14 @@ class ArtBase(object):
"""TBD."""
self.not_implemented()
- def included(self):
+ def validate(self, script_directory):
"""TBD."""
- self.not_implemented()
-
- def wait_for(self):
- """TBD."""
- self.not_implemented()
+ directories = self.get_test_directories(script_directory)
+ for directory in directories.itervalues():
+ files = self.get_files(directory)
+ for fname in files:
+ ArtHeader(os.path.join(directory, fname)).validate()
+ return 0
#
# Default implementations
@@ -64,6 +66,7 @@ class ArtBase(object):
out = check(run_command("acmd.py diff-root " + file_name + " " + ref_file + " --error-mode resilient --ignore-leaves RecoTimingObj_p1_HITStoRDO_timings RecoTimingObj_p1_RAWtoESD_mems RecoTimingObj_p1_RAWtoESD_timings RAWtoESD_mems RAWtoESD_timings ESDtoAOD_mems ESDtoAOD_timings HITStoRDO_timings RAWtoALL_mems RAWtoALL_timings RecoTimingObj_p1_RAWtoALL_mems RecoTimingObj_p1_RAWtoALL_timings --entries " + str(entries)))
print out
sys.stdout.flush()
+ return 0
#
# Protected Methods
@@ -75,32 +78,25 @@ class ArtBase(object):
config_file.close()
return config
- def get_art_headers(self, filename):
- """Return dictionary with art headers."""
- result = {}
- for line in open(filename, "r"):
- line_match = re.match(r'#\s*art-(\w+):\s+(.+)$', line)
- if line_match:
- result[line_match.group(1)] = line_match.group(2)
- return result
+ def get_files(self, directory, type=None):
+ """
+ Return a list of all test files matching 'test_*.sh' of given 'type'.
- def get_files(self, directory, type):
- """Return a list of all test files matching 'test_*.sh' of given 'queue'."""
+ If type is None, all files are returned. Only the filenames are returned.
+ """
result = []
if directory is not None:
files = os.listdir(directory)
files.sort()
for fname in files:
if fnmatch.fnmatch(fname, 'test_*.sh') or fnmatch.fnmatch(fname, 'test_*.py'):
- headers = self.get_art_headers(os.path.join(directory, fname))
- if 'type' in headers and headers['type'] == type:
+ if type is None or ArtHeader(os.path.join(directory, fname)).get('art-type') == type:
result.append(fname)
return result
def get_type(self, directory, test_name):
"""Return the 'type' of a test."""
- headers = self.get_art_headers(os.path.join(directory, test_name))
- return None if 'type' not in headers else headers['type']
+ return ArtHeader(os.path.join(directory, test_name)).get('art-type')
def get_test_directories(self, directory):
"""
diff --git a/Tools/ART/python/ART/art_build.py b/Tools/ART/python/ART/art_build.py
new file mode 100644
index 0000000000000000000000000000000000000000..761b837d225dddc6ceb20019840a526e579b226d
--- /dev/null
+++ b/Tools/ART/python/ART/art_build.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+"""TBD."""
+
+__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
+
+import collections
+import json
+import multiprocessing
+import re
+import os
+
+from art_misc import run_command, mkdir_p
+from art_base import ArtBase
+from art_header import ArtHeader
+
+from parallelScheduler import ParallelScheduler
+
+
+def run_job(art_directory, sequence_tag, script_directory, package, type, index, test_name):
+ """TBD."""
+ # print "run_job", art_directory, sequence_tag, script_directory, package, type, index, test_name
+ (exit_code, out, err) = run_command(os.path.join(art_directory, './art-internal.py') + " job build " + script_directory + " " + package + " " + type + " " + sequence_tag + " " + str(index) + " " + "out")
+ return (test_name, exit_code, out, err)
+
+
+class ArtBuild(ArtBase):
+ """TBD."""
+
+ def __init__(self, art_directory, script_directory, max_jobs=0, ci=False):
+ """TBD."""
+ # print "ArtBuild", art_directory, script_directory, max_jobs
+ self.art_directory = art_directory
+ self.script_directory = script_directory
+ self.max_jobs = multiprocessing.cpu_count() if max_jobs <= 0 else max_jobs
+ self.ci = ci
+
+ def task_list(self, type, sequence_tag):
+ """TBD."""
+ # print "task_list", type, sequence_tag
+ test_directories = self.get_test_directories(self.script_directory)
+ status = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.defaultdict()))
+
+ for package, root in test_directories.items():
+ job_results = self.task(package, type, sequence_tag)
+ for job_result in job_results:
+ status[package][job_result[0]]['exit_code'] = job_result[1]
+ status[package][job_result[0]]['out'] = job_result[2]
+ status[package][job_result[0]]['err'] = job_result[3]
+
+ with open(os.path.join(sequence_tag, "status.json"), 'w') as outfile:
+ json.dump(status, outfile, sort_keys=True, indent=4, ensure_ascii=False)
+
+ return 0
+
+ def task(self, package, type, sequence_tag):
+ """TBD."""
+ # print "task", package, type, sequence_tag
+ test_names = self.get_list(self.script_directory, package, type)
+ scheduler = ParallelScheduler(self.max_jobs + 1)
+
+ index = 0
+ for test_name in test_names:
+ schedule_test = False
+ if self.ci:
+ branch_name = os.environ['AtlasBuildBranch']
+ fname = os.path.join(self.get_test_directories(self.script_directory)[package], test_name)
+ cis = ArtHeader(fname).get('art-ci')
+ for ci in cis:
+ if re.match(ci, branch_name):
+ schedule_test = True
+ break
+ else:
+ schedule_test = True
+
+ if schedule_test:
+ scheduler.add_task(task_name="t" + str(index), dependencies=[], description="d", target_function=run_job, function_kwargs={'art_directory': self.art_directory, 'sequence_tag': sequence_tag, 'script_directory': self.script_directory, 'package': package, 'type': type, 'index': index, 'test_name': test_name})
+ index += 1
+
+ return scheduler.run()
+
+ def job(self, package, type, sequence_tag, index, out):
+ """TBD."""
+ # print "job", package, type, sequence_tag, index, out
+ test_directories = self.get_test_directories(self.script_directory)
+ test_directory = os.path.abspath(test_directories[package])
+ test_name = self.get_files(test_directory, type)[int(index)]
+
+ work_directory = os.path.join(sequence_tag, package, os.path.splitext(test_name)[0])
+ mkdir_p(work_directory)
+
+ (exit_code, output, err) = run_command(os.path.join(test_directory, test_name) + ' ' + '.' + ' ' + package + ' ' + type + ' ' + test_name, dir=work_directory)
+
+ with open(os.path.join(work_directory, "stdout.txt"), "w") as text_file:
+ text_file.write(output)
+ with open(os.path.join(work_directory, "stderr.txt"), "w") as text_file:
+ text_file.write(err)
+
+ return exit_code
diff --git a/Tools/ART/python/art_grid.py b/Tools/ART/python/ART/art_grid.py
similarity index 76%
rename from Tools/ART/python/art_grid.py
rename to Tools/ART/python/ART/art_grid.py
index 9cc0166f013e44859a0378c7e7e806eae2b0c4ef..ec7bd2c50c020a80b672e46749c91b5f0b897597 100644
--- a/Tools/ART/python/art_grid.py
+++ b/Tools/ART/python/ART/art_grid.py
@@ -10,17 +10,17 @@ import re
import shutil
import sys
import tarfile
-import time
from art_base import ArtBase
-from art_misc import mkdir_p, make_executable, check, run_command, count_string_occurrence
+from art_misc import mkdir_p, make_executable, check, run_command
class ArtGrid(ArtBase):
"""TBD."""
- def __init__(self, nightly_release, project, platform, nightly_tag):
+ def __init__(self, art_directory, nightly_release, project, platform, nightly_tag):
"""TBD."""
+ self.art_directory = art_directory
self.nightly_release = nightly_release
self.project = project
self.platform = platform
@@ -39,9 +39,6 @@ class ArtGrid(ArtBase):
def task_list(self, type, sequence_tag):
"""TBD."""
- # get the path of the art.py script
- art_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
-
# job will be submitted from tmp directory
submit_directory = 'tmp'
@@ -82,38 +79,42 @@ class ArtGrid(ArtBase):
sys.stdout.flush()
submit_dir = os.path.join(submit_directory, package)
run = os.path.join(submit_dir, "run")
- mkdir_p(run)
-
- shutil.copy(os.path.join(art_dir, 'art.py'), run)
- shutil.copy(os.path.join(art_dir, 'art-internal.py'), run)
- shutil.copy(os.path.join(art_dir, 'art_base.py'), run)
- shutil.copy(os.path.join(art_dir, 'art_local.py'), run)
- shutil.copy(os.path.join(art_dir, 'art_grid.py'), run)
- shutil.copy(os.path.join(art_dir, 'art_batch.py'), run)
- shutil.copy(os.path.join(art_dir, 'art_misc.py'), run)
- shutil.copy(os.path.join(art_dir, 'serialScheduler.py'), run)
- shutil.copy(os.path.join(art_dir, 'parallelScheduler.py'), run)
- shutil.copy(os.path.join(art_dir, 'docopt.py'), run)
- shutil.copy(os.path.join(art_dir, 'docopt_dispatch.py'), run)
+ ART = os.path.join(run, "ART")
+ mkdir_p(ART)
+
+ # get the path of the python classes and support scripts
+ art_python_directory = os.path.join(self.art_directory, '..', 'python', 'ART')
+
+ # FIXME we need to know where all those files went in the Athena install
+ shutil.copy(os.path.join(self.art_directory, 'art.py'), run)
+ shutil.copy(os.path.join(self.art_directory, 'art-internal.py'), run)
+ shutil.copy(os.path.join(art_python_directory, '__init__.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_base.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_build.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_grid.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_header.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_misc.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'docopt.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'docopt_dispatch.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'parallelScheduler.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'serialScheduler.py'), ART)
make_executable(os.path.join(run, 'art.py'))
make_executable(os.path.join(run, 'art-internal.py'))
- command = os.path.join(art_dir, 'art-internal.py') + ' task grid ' + package + ' ' + type + ' ' + sequence_tag + ' ' + self.nightly_release + ' ' + self.project + ' ' + self.platform + ' ' + self.nightly_tag
+ command = os.path.join(self.art_directory, 'art-internal.py') + ' task grid ' + package + ' ' + type + ' ' + sequence_tag + ' ' + self.nightly_release + ' ' + self.project + ' ' + self.platform + ' ' + self.nightly_tag
print command
sys.stdout.flush()
out = check(run_command(command))
print out
sys.stdout.flush()
+ return 0
def task(self, package, type, sequence_tag):
"""TBD."""
print 'Running art task'
sys.stdout.flush()
- # get the path of the art.py script
- art_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
-
number_of_tests = len(self.get_list(self.get_script_directory(), package, type))
print self.nightly_release + " " + self.project + " " + self.platform + " " + self.nightly_tag + " " + sequence_tag + " " + package + " " + type + " " + str(number_of_tests)
@@ -121,9 +122,10 @@ class ArtGrid(ArtBase):
# run task from Bash Script as is needed in ATLAS setup
# FIXME we need to parse the output
- out = check(run_command(os.path.join(art_dir, 'art-task-grid.sh') + " " + package + " " + type + " " + sequence_tag + " " + str(number_of_tests) + " " + self.nightly_release + " " + self.project + " " + self.platform + " " + self.nightly_tag))
+ out = check(run_command(os.path.join(self.art_directory, 'art-task-grid.sh') + " " + package + " " + type + " " + sequence_tag + " " + str(number_of_tests) + " " + self.nightly_release + " " + self.project + " " + self.platform + " " + self.nightly_tag))
print out
sys.stdout.flush()
+ return 0
def job(self, package, type, sequence_tag, index, out):
"""TBD."""
@@ -164,6 +166,7 @@ class ArtGrid(ArtBase):
if os.path.exists(out_name[0]):
tar_file.add(out_name[0])
tar_file.close()
+ return 0
def list(self, package, type):
"""TBD."""
@@ -173,6 +176,7 @@ class ArtGrid(ArtBase):
print str(i) + ' ' + job
sys.stdout.flush()
i += 1
+ return 0
def log(self, package, test_name):
"""TBD."""
@@ -185,6 +189,7 @@ class ArtGrid(ArtBase):
print content
break
tar.close()
+ return 0
def output(self, package, test_name, file_name):
"""TBD."""
@@ -195,46 +200,6 @@ class ArtGrid(ArtBase):
tar.extractall(path='.', members=[member])
break
tar.close()
-
- def included(self):
- """TBD."""
- package_name = "__name_never_used__"
-
- if self.excluded(self.get_config(), package_name):
- print 'Excluding ' + 'all' + ' for ' + self.nightly_release + ' project ' + self.project + ' on ' + self.platform
- print 'art-status: excluded'
- sys.stdout.flush()
- return 1
- else:
- print 'art-status: included'
- return 0
-
- def wait_for(self):
- """TBD."""
- directory = os.path.join(self.cvmfs_directory, self.nightly_release, self.nightly_tag)
- path = os.path.join(directory, self.nightly_release + "__" + self.project + "__" + self.platform + "*" + self.nightly_tag + "__*.ayum.log")
-
- count = 0
- needed = 1
- value = count_string_occurrence(path, "Install looks to have been successful")
- print "art-status: waiting"
- print path
- print "count: " + str(value) + " mins: " + str(count)
- sys.stdout.flush()
- while (value < needed) and (count < 30):
- time.sleep(60)
- count += 1
- value = count_string_occurrence(path, "Install looks to have been successful")
- print "count: " + str(value) + " mins: " + str(count)
- sys.stdout.flush()
-
- if value < needed:
- print "art-status: no release"
- sys.stdout.flush()
- return -2
-
- print "art-status: setup"
- sys.stdout.flush()
return 0
def compare(self, package, test_name, days, file_names):
@@ -255,6 +220,7 @@ class ArtGrid(ArtBase):
ref_file = os.path.join(ref_dir, file_name)
self.compare_ref(file_name, ref_file, 10)
+ return 0
#
# Protected Methods
diff --git a/Tools/ART/python/ART/art_header.py b/Tools/ART/python/ART/art_header.py
new file mode 100644
index 0000000000000000000000000000000000000000..bca71469d31df2fbfbf77d3d0834bb4f1feb7f50
--- /dev/null
+++ b/Tools/ART/python/ART/art_header.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+"""TBD."""
+
+__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
+
+import re
+
+from copy import copy
+from types import ListType
+from types import StringType
+
+
+class ArtHeader(object):
+ """TBD."""
+
+ def __init__(self, filename):
+ """TBD."""
+ self.header = re.compile(r'#\s(art-[\w-]+):\s+(.+)$')
+ self.header_format_error1 = re.compile(r'#(art-[\w-]+):\s*(.+)$')
+ self.header_format_error2 = re.compile(r'#\s\s+(art-[\w-]+):\s*(.+)$')
+ self.header_format_error3 = re.compile(r'#\s(art-[\w-]+):\S(.*)$')
+
+ self.filename = filename
+
+ self.type = {}
+ self.defaultValue = {}
+ self.constraint = {}
+ self.value = {}
+
+ self.add('art-description', StringType, '')
+ self.add('art-type', StringType, None, ['build', 'grid'])
+ self.add('art-ci', ListType, [])
+
+ self.read(filename)
+
+ def add(self, key, type, defaultValue=None, constraint=None):
+ """TBD."""
+ self.type[key] = type
+ self.defaultValue[key] = defaultValue
+ self.constraint[key] = constraint
+ self.value[key] = copy(defaultValue)
+
+ def is_list(self, key):
+ """TBD."""
+ return self.type[key] is ListType if key in self.type else False
+
+ def read(self, filename):
+ """Read all headers from file."""
+ for line in open(filename, "r"):
+ line_match = self.header.match(line)
+ if line_match:
+ key = line_match.group(1)
+ value = line_match.group(2)
+ if self.is_list(key):
+ self.value[key].append(value)
+ else:
+ self.value[key] = value
+
+ def get(self, key):
+ """TBD."""
+ return self.value[key] if key in self.value else None
+
+ def print_it(self):
+ """TBD."""
+ for key in self.type:
+ print key, self.type[key], self.defaultValue[key], self.value[key], self.constraint[key]
+
+ def validate(self):
+ """
+ Validate the '# art-*' headers in the file.
+
+ Validation fails if:
+ - a header is spaced correctly (e.g. '#art-header: value')
+ - a value in a header is not spaced correctly (e.g. '# art-header:value')
+ - a key is found which is not defined
+ - a value is found of the wrong type
+ - a value is found outside the constraint
+ """
+ for line in open(self.filename, "r"):
+ if self.header_format_error1.match(line):
+ print "LINE: ", line.rstrip()
+ print "ERROR: Header Validation - invalid header format, use space between '# and art-xxx' in file", self.filename
+ print
+ if self.header_format_error2.match(line):
+ print "LINE: ", line.rstrip()
+ print "ERROR: Header Validation - invalid header format, too many spaces between '# and art-xxx' in file", self.filename
+ print
+ if self.header_format_error3.match(line):
+ print "LINE: ", line.rstrip()
+ print "ERROR: Header Validation - invalid header format, use at least one space between ': and value' in file", self.filename
+ print
+
+ for key in self.value:
+ if key not in self.type:
+ print "ERROR: Header Validation - Invalid key: '" + key + "' in file", self.filename
+ print
+ continue
+ if type(self.value[key]) != self.type[key]:
+ if not isinstance(self.value[key], type(None)):
+ print "ERROR: Header Validation - type:", type(self.value[key]), "not valid for key:", key, "expected type:", self.type[key], "in file", self.filename
+ print
+ if self.constraint[key] is not None and self.value[key] not in self.constraint[key]:
+ if self.value[key] is None:
+ print "ERROR: Header Validation - missing key: '" + key + "' in file", self.filename
+ else:
+ print "ERROR: Header Validation - value: '" + self.value[key] + "' for key: '" + key + "' not in constraints:", self.constraint[key], "in file", self.filename
+ print
+
+ return 0
diff --git a/Tools/ART/python/art_misc.py b/Tools/ART/python/ART/art_misc.py
similarity index 78%
rename from Tools/ART/python/art_misc.py
rename to Tools/ART/python/ART/art_misc.py
index e32aabee0fe33ad5d5763db19f110fba99133c65..c055089fa2e2472d5f4bc563b724bd0938a672fa 100644
--- a/Tools/ART/python/art_misc.py
+++ b/Tools/ART/python/ART/art_misc.py
@@ -5,14 +5,12 @@
__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
import errno
-import glob
import os
import shlex
import subprocess
-import sys
-def run_command(cmd, dir=None, shell=False, redirect=False):
+def run_command(cmd, dir=None, shell=False):
"""Run the given command locally and returns the output, err and exit_code."""
print "Execute: " + cmd
if "|" in cmd:
@@ -32,12 +30,6 @@ def run_command(cmd, dir=None, shell=False, redirect=False):
(output, err) = p[i - 1].communicate()
exit_code = p[0].wait()
- if redirect:
- with open(os.path.join(dir, "stdout.txt"), "w") as text_file:
- text_file.write(str(output))
- with open(os.path.join(dir, "stderr.txt"), "w") as text_file:
- text_file.write(str(err))
-
return exit_code, str(output), str(err)
@@ -60,13 +52,13 @@ def verify((exitcode, out, err)):
"""Check exitcode and print statement."""
if exitcode == 0:
print out
- return out
+ return exitcode
print "Error:", exitcode
print "StdOut:", out
print "StdErr:", err
- print 'art-status: error'
+ # print 'art-status: error'
return exitcode
@@ -114,15 +106,3 @@ def which(program):
return exe_file
return None
-
-
-def count_string_occurrence(path, string):
- """Count number of occurences of 'string' inside file 'path'. Returns count or 0."""
- for file in glob.iglob(path):
- print file
- sys.stdout.flush()
- f = open(file)
- contents = f.read()
- f.close()
- return contents.count(string)
- return 0
diff --git a/Tools/ART/python/docopt.py b/Tools/ART/python/ART/docopt.py
similarity index 100%
rename from Tools/ART/python/docopt.py
rename to Tools/ART/python/ART/docopt.py
diff --git a/Tools/ART/python/docopt_dispatch.py b/Tools/ART/python/ART/docopt_dispatch.py
similarity index 100%
rename from Tools/ART/python/docopt_dispatch.py
rename to Tools/ART/python/ART/docopt_dispatch.py
diff --git a/Tools/ART/python/parallelScheduler.py b/Tools/ART/python/ART/parallelScheduler.py
similarity index 99%
rename from Tools/ART/python/parallelScheduler.py
rename to Tools/ART/python/ART/parallelScheduler.py
index 5471652b80e1fa1984f474cab5c5556d66a53fb6..42a1d1cfc42a5f7f7a4e1a2c8cea3c05d36293c7 100644
--- a/Tools/ART/python/parallelScheduler.py
+++ b/Tools/ART/python/ART/parallelScheduler.py
@@ -3,7 +3,7 @@ Created on 16/05/2012
* Repository : https://github.com/victor-gil-sepulveda/pyScheduler
* Licensed under the MIT license (see LICENSE-MIT)
- * Copyright (C) 2013 VÃctor Alejandro Gil Sepúlveda
+ * Copyright (C) 2013 Victor Alejandro Gil Sepulveda
@author: victor
'''
diff --git a/Tools/ART/python/serialScheduler.py b/Tools/ART/python/ART/serialScheduler.py
similarity index 99%
rename from Tools/ART/python/serialScheduler.py
rename to Tools/ART/python/ART/serialScheduler.py
index e11248fb6b83fc856bc132debf1bcc75dc55e6a4..9272f8eff6f325652d262d0f60d81795369e581a 100644
--- a/Tools/ART/python/serialScheduler.py
+++ b/Tools/ART/python/ART/serialScheduler.py
@@ -3,7 +3,7 @@ Created on 16/08/2012
* Repository : https://github.com/victor-gil-sepulveda/pyScheduler
* Licensed under the MIT license (see LICENSE-MIT)
- * Copyright (C) 2013 VÃctor Alejandro Gil Sepúlveda
+ * Copyright (C) 2013 Victor Alejandro Gil Sepulveda
@author: victor
'''
diff --git a/Tools/ART/python/art_local.py b/Tools/ART/python/art_local.py
deleted file mode 100644
index 4a0a723d86ffb1eb2cec545c37a2a39bccf61eeb..0000000000000000000000000000000000000000
--- a/Tools/ART/python/art_local.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python
-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
-"""TBD."""
-
-__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
-
-import os
-import sys
-import multiprocessing
-
-from art_misc import run_command, verify, redirect, mkdir_p
-from art_base import ArtBase
-
-from parallelScheduler import ParallelScheduler
-
-
-def run_job(sequence_tag, script_directory, package, type, index, test_name):
- """TBD."""
- print "run_job", sequence_tag, script_directory, package, type, index, test_name
- art_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
- verify(run_command(os.path.join(art_dir, './art-internal.py') + " job local " + script_directory + " " + package + " " + type + " " + sequence_tag + " " + str(index) + " " + "out"))
- # print out
-
-
-class ArtLocal(ArtBase):
- """TBD."""
-
- def __init__(self, script_directory, max_jobs=0):
- """TBD."""
- print "ArtLocal", script_directory, max_jobs
- self.script_directory = script_directory
- self.max_jobs = multiprocessing.cpu_count() if max_jobs <= 0 else max_jobs
-
- def task_list(self, type, sequence_tag):
- """TBD."""
- print "task_list", type, sequence_tag
- test_directories = self.get_test_directories(self.script_directory)
- for package, root in test_directories.items():
- self.task(package, type, sequence_tag)
-
- def task(self, package, type, sequence_tag):
- """TBD."""
- print "task", package, type, sequence_tag
- test_names = self.get_list(self.script_directory, package, type)
- scheduler = ParallelScheduler(self.max_jobs + 1)
-
- index = 0
- for test_name in test_names:
- scheduler.add_task(task_name="t" + str(index), dependencies=[], description="d", target_function=run_job, function_kwargs={'sequence_tag': sequence_tag, 'script_directory': self.script_directory, 'package': package, 'type': type, 'index': index, 'test_name': test_name})
- index += 1
-
- scheduler.run()
-
- def job(self, package, type, sequence_tag, index, out):
- """TBD."""
- print "job", package, type, sequence_tag, index, out
- test_directories = self.get_test_directories(self.script_directory)
- test_directory = os.path.abspath(test_directories[package])
- test_name = self.get_files(test_directory, type)[int(index)]
-
- work_directory = os.path.join(sequence_tag, package, os.path.splitext(test_name)[0])
- mkdir_p(work_directory)
-
- output = redirect(run_command(os.path.join(test_directory, test_name) + ' ' + '.' + ' ' + package + ' ' + type + ' ' + test_name, dir=work_directory, redirect=True))
- print output
diff --git a/Tools/ART/scripts/art-internal.py b/Tools/ART/scripts/art-internal.py
index 4100948da0ddb67351879aa33ca7c3a0fc1aa129..f604394f54e81bb19dc77df12a633033e347ce01 100755
--- a/Tools/ART/scripts/art-internal.py
+++ b/Tools/ART/scripts/art-internal.py
@@ -4,12 +4,10 @@
ART-internal - ATLAS Release Tester (internal command).
Usage:
- art.py included [-v] <nightly_release> <project> <platform>
- art.py job local [-v] <script_directory> <package> <job_type> <sequence_tag> <index> <out>
- art.py job (grid|batch) [-v] <package> <job_type> <sequence_tag> <index> <out> <nightly_release> <project> <platform> <nightly_tag>
- art.py task local [-v] <script_directory> <package> <job_type> <sequence_tag>
- art.py task (grid|batch) [-v] <package> <job_type> <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
- art.py wait_for [-v] <nightly_release> <project> <platform> <nightly_tag>
+ art-internal.py job build [-v] <script_directory> <package> <job_type> <sequence_tag> <index> <out>
+ art-internal.py job grid [-v] <package> <job_type> <sequence_tag> <index> <out> <nightly_release> <project> <platform> <nightly_tag>
+ art-internal.py task build [-v] <script_directory> <package> <job_type> <sequence_tag>
+ art-internal.py task grid [-v] <package> <job_type> <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
Options:
-h --help Show this screen.
@@ -17,10 +15,8 @@ Options:
-v, --verbose Show details.
Sub-commands:
- included Check if a release and platform is included
job Runs a single job, given a particular index
task Runs a single task, consisting of given number of jobs
- wait_for Wait for the release to be available
Arguments:
index Index of the test inside the package
@@ -39,28 +35,21 @@ __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
import os
import sys
-from docopt_dispatch import dispatch
-from art_local import ArtLocal
-from art_grid import ArtGrid
-from art_batch import ArtBatch
+from ART.docopt_dispatch import dispatch
-
-@dispatch.on('included')
-def included(nightly_release, project, platform, **kwargs):
- """TBD."""
- sys.exit(ArtGrid(nightly_release, project, platform, None).included())
+from ART import ArtGrid, ArtBuild
-@dispatch.on('job', 'local')
-def job_local(script_directory, package, job_type, sequence_tag, index, out, **kwargs):
+@dispatch.on('job', 'build')
+def job_build(script_directory, package, job_type, sequence_tag, index, out, **kwargs):
"""TBD.
Tests are called with the following parameters:
SCRIPT_DIRECTORY, PACKAGE, TYPE, TEST_NAME
"""
- print "job_local", script_directory, package, job_type, sequence_tag, index, out, kwargs
- ArtLocal(script_directory).job(package, job_type, sequence_tag, index, out)
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+ exit(ArtBuild(art_directory, script_directory).job(package, job_type, sequence_tag, index, out))
@dispatch.on('job', 'grid')
@@ -70,44 +59,26 @@ def job_grid(package, job_type, sequence_tag, index, out, nightly_release, proje
Tests are called with the following parameters:
SCRIPT_DIRECTORY, PACKAGE, TYPE, TEST_NAME, NIGHTLY_RELEASE, PROJECT, PLATFORM, NIGHTLY_TAG
"""
- ArtGrid(nightly_release, project, platform, nightly_tag).job(package, job_type, sequence_tag, index, out)
-
-
-@dispatch.on('job', 'batch')
-def job_batch(package, job_type, sequence_tag, index, out, nightly_release, project, platform, nightly_tag, **kwargs):
- """TBD.
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).job(package, job_type, sequence_tag, index, out))
- Tests are called with the following parameters:
- SCRIPT_DIRECTORY, PACKAGE, TYPE, TEST_NAME, NIGHTLY_RELEASE, PROJECT, PLATFORM, NIGHTLY_TAG
- """
- ArtBatch(nightly_release, project, platform, nightly_tag).job(package, job_type, sequence_tag, index, out)
-
-@dispatch.on('task', 'local')
-def task_local(script_directory, job_type, sequence_tag, **kwargs):
+@dispatch.on('task', 'build')
+def task_build(script_directory, job_type, sequence_tag, **kwargs):
"""TBD."""
- ArtLocal(script_directory).task(job_type, sequence_tag)
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+ exit(ArtBuild(art_directory, script_directory).task(job_type, sequence_tag))
@dispatch.on('task', 'grid')
def task_grid(package, job_type, sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
"""TBD."""
- ArtGrid(nightly_release, project, platform, nightly_tag).task(package, job_type, sequence_tag)
-
-
-@dispatch.on('task', 'batch')
-def task_batch(package, job_type, sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
- """TBD."""
- ArtBatch(nightly_release, project, platform, nightly_tag).task(package, sequence_tag)
-
-
-@dispatch.on('wait_for')
-def wait_for(nightly_release, project, platform, nightly_tag, **kwargs):
- """TBD."""
- sys.exit(ArtGrid(nightly_release, project, platform, nightly_tag).wait_for())
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).task(package, job_type, sequence_tag))
if __name__ == '__main__':
# NOTE: import should be here, to keep the order of the decorators (module first, art last and unused)
from art import __version__
+ print "ART_PATH", os.path.dirname(os.path.realpath(sys.argv[0]))
dispatch(__doc__, version=os.path.splitext(os.path.basename(__file__))[0] + ' ' + __version__)
diff --git a/Tools/ART/scripts/art-task-build.sh b/Tools/ART/scripts/art-task-build.sh
new file mode 100755
index 0000000000000000000000000000000000000000..c7e3520f05275cdbdd95cb992c84878c3e6e1fcf
--- /dev/null
+++ b/Tools/ART/scripts/art-task-build.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+# arguments: RELEASE_BASE, PROJECT, PLATFORM, DATESTAMP
+# author : Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>, Emil Obreshkov <Emil.Obreshkov@cern.ch>
+
+whoami
+date
+
+RELEASE_BASE=$1
+PROJECT=$2
+PLATFORM=$3
+DATESTAMP=$4
+
+echo Release base $RELEASE_BASE
+echo Project $PROJECT
+echo Platform $PLATFORM
+echo Date $DATESTAMP
+
+BRANCH=`echo $RELEASE_BASE |tr "/" " " |awk '{print $5}'`
+
+# setup for the build
+[[ "${ATLAS_LOCAL_ROOT_BASE}" = "" ]] && export ATLAS_LOCAL_ROOT_BASE="/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase"
+source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh --quiet
+lsetup asetup
+asetup none,cmakesetup --platform ${PLATFORM}
+source ${RELEASE_BASE}/build/install/${PROJECT}/*/InstallArea/${PLATFORM}/setup.sh
+
+# run build tests
+SUBDIR=${BRANCH}/${PROJECT}/${PLATFORM}/${DATESTAMP}
+OUTDIR="art-build/${SUBDIR}"
+CMD="art.py run ${RELEASE_BASE} ${OUTDIR}"
+echo ${CMD}
+RESULT=`eval "${CMD}"`
+echo ${RESULT}
+
+# copy the test results to EOS area
+TARGETDIR=/eos/atlas/atlascerngroupdisk/art-build/${SUBDIR}
+if [[ ! -e ${TARGETDIR} ]]; then
+ eos mkdir -p ${TARGET}
+ xrdcp -vr $OUTDIR $TARGETDIR
+fi
diff --git a/Tools/ART/scripts/art-task-grid.sh b/Tools/ART/scripts/art-task-grid.sh
index 8fe0d302c2c0bd093134dfe65be330da96ed8e20..4e669aa92636ac65c1faf4dbcf762ebff2bcaf77 100755
--- a/Tools/ART/scripts/art-task-grid.sh
+++ b/Tools/ART/scripts/art-task-grid.sh
@@ -1,8 +1,11 @@
#!/bin/bash
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+#
# NOTE do NOT run with /bin/bash -x as the output is too big for gitlab-ci
# arguments: PACKAGE, SEQUENCE_TAG, NUMBER_OF_TESTS, NIGHTLY_RELEASE, PROJECT, PLATFORM, NIGHTLY_TAG
#
+# author : Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>
+#
# example: Tier0ChainTests grid 316236 32 21.0 Athena x86_64-slc6-gcc62-opt 2017-02-26T2119
#set -e
@@ -23,6 +26,8 @@ NIGHTLY_TAG=$8
export ATLAS_LOCAL_ROOT_BASE=/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase
source $ATLAS_LOCAL_ROOT_BASE/user/atlasLocalSetup.sh
+export RUCIO_ACCOUNT=artprod
+
lsetup panda
voms-proxy-init --rfc -noregen -cert ./grid.proxy -voms atlas
@@ -32,6 +37,7 @@ NIGHTLY_RELEASE_SHORT=${NIGHTLY_RELEASE/-VAL-*/-VAL}
asetup --platform=${PLATFORM} ${NIGHTLY_RELEASE_SHORT},${NIGHTLY_TAG},${PROJECT}
+# NOTE: for art-internal.py the current dir can be used as it is copied there
cd ./tmp/${PACKAGE}/run
OUTFILE="user.${USER}.atlas.${NIGHTLY_RELEASE_SHORT}.${PROJECT}.${PLATFORM}.${NIGHTLY_TAG}.${SEQUENCE_TAG}.${PACKAGE}"
CMD="pathena --noBuild --skipScout --trf \"./art-internal.py job grid ${PACKAGE} ${TYPE} ${SEQUENCE_TAG} %RNDM:0 %OUT.tar ${NIGHTLY_RELEASE_SHORT} ${PROJECT} ${PLATFORM} ${NIGHTLY_TAG}\" --split ${NUMBER_OF_TESTS} --outDS ${OUTFILE}"
diff --git a/Tools/ART/scripts/art.py b/Tools/ART/scripts/art.py
index dd6de771080d805fa298477dadda7d67527e9d2a..489d799877698e4cd1b01f4f9951474d7af3042b 100755
--- a/Tools/ART/scripts/art.py
+++ b/Tools/ART/scripts/art.py
@@ -4,8 +4,9 @@
ART - ATLAS Release Tester.
Usage:
- art.py run [-v --type=<T> --max-jobs=<N>] <script_directory> <sequence_tag>
+ art.py run [-v --type=<T> --max-jobs=<N> --ci] <script_directory> <sequence_tag>
art.py submit [-v --type=<T>] <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
+ art.py validate [-v] <script_directory>
art.py compare grid [-v --days=<D>] <nightly_release> <project> <platform> <nightly_tag> <package> <test_name> <file_name>...
art.py compare ref [-v] <file_name> <ref_file>
art.py list grid [-v] <package> <job_type> <nightly_release> <project> <platform> <nightly_tag>
@@ -16,13 +17,15 @@ Options:
-h --help Show this screen.
--version Show version.
-v, --verbose Show details.
- --type=<T> Type of job (e.g. grid, ci, build)
+ --type=<T> Type of job (e.g. grid, build)
--days=<D> Number of days ago to pick up reference for compare [default: 1]
--max-jobs=<N> Maximum number of concurrent jobs to run
+ --ci Run Continuous Integration tests only (using env: AtlasBuildBranch)
Sub-commands:
- run Run tests from a package locally
+ run Run tests from a package in a local build
submit Submit tests to the grid
+ validate Check headers in tests
compare Compare the output of a job
list Lists the jobs of a package
log Show the log of a job
@@ -40,7 +43,7 @@ Arguments:
script_directory Directory containing the packages with tests
sequence_tag Sequence tag (e.g. 0 or PIPELINE_ID)
test_name Name of the test inside the package (e.g. test_q322.sh)
- job_type Type of job (e.g. grid, ci, build)
+ job_type Type of job (e.g. grid, build)
"""
__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
@@ -51,57 +54,67 @@ import requests
import os
import sys
+from ART.docopt_dispatch import dispatch
-from docopt_dispatch import dispatch
-from art_base import ArtBase
-from art_local import ArtLocal
-from art_grid import ArtGrid
+from ART import ArtBase, ArtGrid, ArtBuild
@dispatch.on('submit')
def submit_grid(sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
"""TBD."""
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
type = 'grid' if kwargs['type'] is None else kwargs['type']
- ArtGrid(nightly_release, project, platform, nightly_tag).task_list(type, sequence_tag)
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).task_list(type, sequence_tag))
@dispatch.on('run')
def run(script_directory, sequence_tag, **kwargs):
"""TBD."""
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
type = 'build' if kwargs['type'] is None else kwargs['type']
- ArtLocal(script_directory, max_jobs=kwargs['max_jobs']).task_list(type, sequence_tag)
+ exit(ArtBuild(art_directory, script_directory, max_jobs=kwargs['max_jobs'], ci=kwargs['ci']).task_list(type, sequence_tag))
+
+
+@dispatch.on('validate')
+def validate(script_directory, **kwargs):
+ """TBD."""
+ exit(ArtBase().validate(script_directory))
@dispatch.on('compare', 'ref')
-def compare_local(file_name, ref_file, **kwargs):
+def compare_ref(file_name, ref_file, **kwargs):
"""TBD."""
- ArtBase().compare_ref(file_name, ref_file)
+ exit(ArtBase().compare_ref(file_name, ref_file))
@dispatch.on('compare', 'grid')
def compare_grid(package, test_name, nightly_release, project, platform, nightly_tag, **kwargs):
"""TBD."""
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
days = int(kwargs['days'])
file_names = kwargs['file_name']
- ArtGrid(nightly_release, project, platform, nightly_tag).compare(package, test_name, days, file_names)
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).compare(package, test_name, days, file_names))
@dispatch.on('list', 'grid')
def list(package, job_type, nightly_release, project, platform, nightly_tag, **kwargs):
"""TBD."""
- ArtGrid(nightly_release, project, platform, nightly_tag).list(package, job_type)
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).list(package, job_type))
@dispatch.on('log', 'grid')
def log(package, test_name, nightly_release, project, platform, nightly_tag, **kwargs):
"""TBD."""
- ArtGrid(nightly_release, project, platform, nightly_tag).log(package, test_name)
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).log(package, test_name))
@dispatch.on('output', 'grid')
def output(package, test_name, file_name, nightly_release, project, platform, nightly_tag, **kwargs):
"""TBD."""
- ArtGrid(nightly_release, project, platform, nightly_tag).output(package, test_name, file_name)
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).output(package, test_name, file_name))
@dispatch.on('retrieve')
@@ -158,4 +171,5 @@ def retrieve(job_id, **kwargs):
if __name__ == '__main__':
+ print "ART_PATH", os.path.dirname(os.path.realpath(sys.argv[0]))
dispatch(__doc__, version=os.path.splitext(os.path.basename(__file__))[0] + ' ' + __version__)
diff --git a/Tools/ART/share/localSetupART.sh b/Tools/ART/share/localSetupART.sh
new file mode 100644
index 0000000000000000000000000000000000000000..94d9403da6132a59fa2c2d840ffb24b380053c83
--- /dev/null
+++ b/Tools/ART/share/localSetupART.sh
@@ -0,0 +1,2 @@
+SCRIPTPATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+export PYTHONPATH=${SCRIPTPATH}/python:${PYTHONPATH}