diff --git a/Tools/ART/python/ART/art_base.py b/Tools/ART/python/ART/art_base.py
index c9032de7cb6e744b42b8688111c4024d781d69c2..6a683081840fcbfb25eaf13f37ff3ad8f5488451 100755
--- a/Tools/ART/python/ART/art_base.py
+++ b/Tools/ART/python/ART/art_base.py
@@ -6,13 +6,20 @@ __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
import fnmatch
import inspect
+import logging
import os
-import sys
import yaml
+try:
+ import scandir as scan
+except ImportError:
+ import os as scan
+
from art_misc import is_exe, run_command
from art_header import ArtHeader
+MODULE = "art.base"
+
class ArtBase(object):
"""TBD."""
@@ -51,26 +58,37 @@ class ArtBase(object):
def validate(self, script_directory):
"""TBD."""
+ log = logging.getLogger(MODULE)
directories = self.get_test_directories(script_directory.rstrip("/"))
+
+ found_test = False
for directory in directories.itervalues():
files = self.get_files(directory)
for fname in files:
test_name = os.path.join(directory, fname)
- print test_name
+ found_test = True
+ log.debug(test_name)
if not is_exe(test_name):
- print "ERROR: ", test_name, "is not executable."
+ log.error("%s is not executable.", test_name)
ArtHeader(test_name).validate()
+
+ if not found_test:
+ log.warning('No scripts found in %s directory', directories.values()[0])
+ return 0
+
+ log.info("Scripts in %s directory are validated", script_directory)
return 0
def included(self, script_directory, job_type, index_type, nightly_release, project, platform):
"""TBD."""
+ log = logging.getLogger(MODULE)
directories = self.get_test_directories(script_directory.rstrip("/"))
for directory in directories.itervalues():
files = self.get_files(directory, job_type, index_type)
for fname in files:
test_name = os.path.join(directory, fname)
if self.is_included(test_name, nightly_release, project, platform):
- print test_name, ArtHeader(test_name).get('art-include')
+ log.info("%s %s", test_name, ArtHeader(test_name).get('art-include'))
return 0
def download(self, input_file):
@@ -84,6 +102,8 @@ class ArtBase(object):
"""TBD."""
import PyUtils.PoolFile as PF
+ log = logging.getLogger(MODULE)
+
# diff-pool
df = PF.DiffFiles(refFileName=ref_file, chkFileName=file_name, ignoreList=['RecoTimingObj_p1_RAWtoESD_timings', 'RecoTimingObj_p1_ESDtoAOD_timings'])
df.printSummary()
@@ -94,22 +114,24 @@ class ArtBase(object):
# diff-root
(code, out, err) = run_command("acmd.py diff-root " + file_name + " " + ref_file + " --error-mode resilient --ignore-leaves RecoTimingObj_p1_HITStoRDO_timings RecoTimingObj_p1_RAWtoESD_mems RecoTimingObj_p1_RAWtoESD_timings RAWtoESD_mems RAWtoESD_timings ESDtoAOD_mems ESDtoAOD_timings HITStoRDO_timings RAWtoALL_mems RAWtoALL_timings RecoTimingObj_p1_RAWtoALL_mems RecoTimingObj_p1_RAWtoALL_timings RecoTimingObj_p1_EVNTtoHITS_timings --entries " + str(entries))
if code != 0:
- print "Error:", code
- print "StdErr:", err
+ log.error("Error: %d", code)
+ print(err)
- print out
- sys.stdout.flush()
- return err
+ log.info(out)
+ return code
#
# Protected Methods
#
def get_config(self):
- """Retrieve dictionary of ART configuration file."""
- config_file = open("art-configuration.yml", "r")
- config = yaml.load(config_file)
- config_file.close()
- return config
+ """Retrieve dictionary of ART configuration file, or None if file does not exist."""
+ try:
+ config_file = open("art-configuration.yml", "r")
+ config = yaml.load(config_file)
+ config_file.close()
+ return config
+ except IOError:
+ return None
def get_files(self, directory, job_type=None, index_type="all", nightly_release=None, project=None, platform=None):
"""
@@ -163,7 +185,7 @@ class ArtBase(object):
A dictionary key=<package>, value=<directory> is returned
"""
result = {}
- for root, dirs, files in os.walk(directory):
+ for root, dirs, files in scan.walk(directory):
if root.endswith('/test'):
package = os.path.basename(os.path.dirname(root))
result[package] = root
diff --git a/Tools/ART/python/ART/art_build.py b/Tools/ART/python/ART/art_build.py
index 3c62cb3b11b5396ea66bea42f6b105dd9f4c13e0..0c58f19886db14f6bbbb07d5858a0dbeb8bd3977 100644
--- a/Tools/ART/python/ART/art_build.py
+++ b/Tools/ART/python/ART/art_build.py
@@ -7,6 +7,7 @@ __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
import collections
import fnmatch
import json
+import logging
import multiprocessing
import os
import re
@@ -17,12 +18,16 @@ from art_header import ArtHeader
from parallelScheduler import ParallelScheduler
+MODULE = "art.build"
+
def run_job(art_directory, sequence_tag, script_directory, package, job_type, index, test_name, nightly_release, project, platform, nightly_tag):
"""TBD."""
- print "job started", art_directory, sequence_tag, script_directory, package, job_type, index, test_name, nightly_release, project, platform, nightly_tag
+ log = logging.getLogger(MODULE)
+
+ log.info("job started %s %s %s %s %s %d %s %s %s %s %s", art_directory, sequence_tag, script_directory, package, job_type, index, test_name, nightly_release, project, platform, nightly_tag)
(exit_code, out, err) = run_command(' '.join((os.path.join(art_directory, './art-internal.py'), "job", "build", script_directory, package, job_type, sequence_tag, str(index), "out", nightly_release, project, platform, nightly_tag)))
- print "job ended", art_directory, sequence_tag, script_directory, package, job_type, index, test_name, nightly_release, project, platform, nightly_tag
+ log.info("job ended %s %s %s %s %s %d %s %s %s %s %s", art_directory, sequence_tag, script_directory, package, job_type, index, test_name, nightly_release, project, platform, nightly_tag)
return (test_name, exit_code, out, err)
@@ -33,7 +38,8 @@ class ArtBuild(ArtBase):
def __init__(self, art_directory, nightly_release, project, platform, nightly_tag, script_directory, max_jobs=0, ci=False):
"""TBD."""
super(ArtBuild, self).__init__(art_directory)
- # print "ArtBuild", art_directory, script_directory, max_jobs
+ log = logging.getLogger(MODULE)
+ log.debug("ArtBuild %s %s %d", art_directory, script_directory, max_jobs)
self.art_directory = art_directory
self.script_directory = script_directory.rstrip("/")
self.nightly_release = nightly_release
@@ -45,10 +51,11 @@ class ArtBuild(ArtBase):
def task_list(self, job_type, sequence_tag):
"""TBD."""
- # print "task_list", job_type, sequence_tag
+ log = logging.getLogger(MODULE)
+ log.debug("task_list %s %s", job_type, sequence_tag)
test_directories = self.get_test_directories(self.script_directory)
if not test_directories:
- print 'WARNING: No tests found in directories ending in "test"'
+ log.warning('No tests found in directories ending in "test"')
status = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.defaultdict()))
@@ -81,7 +88,8 @@ class ArtBuild(ArtBase):
def task(self, package, job_type, sequence_tag):
"""TBD."""
- # print "task", package, job_type, sequence_tag
+ log = logging.getLogger(MODULE)
+ log.debug("task %s %s %s", package, job_type, sequence_tag)
test_names = self.get_list(self.script_directory, package, job_type, "all")
scheduler = ParallelScheduler(self.max_jobs + 1)
@@ -101,7 +109,7 @@ class ArtBuild(ArtBase):
if not os.access(fname, os.X_OK):
schedule_test = False
- print "job skipped, file not executable: ", fname
+ log.warning("job skipped, file not executable: %s", fname)
if schedule_test:
scheduler.add_task(task_name="t" + str(index), dependencies=[], description="d", target_function=run_job, function_kwargs={'art_directory': self.art_directory, 'sequence_tag': sequence_tag, 'script_directory': self.script_directory, 'package': package, 'job_type': job_type, 'index': index, 'test_name': test_name, 'nightly_release': self.nightly_release, 'project': self.project, 'platform': self.platform, 'nightly_tag': self.nightly_tag})
@@ -112,7 +120,8 @@ class ArtBuild(ArtBase):
def job(self, package, job_type, sequence_tag, index, out):
"""TBD."""
- # print "job", package, job_type, sequence_tag, index, out
+ log = logging.getLogger(MODULE)
+ log.debug("job %s %s %s %d %s", package, job_type, sequence_tag, index, out)
test_directories = self.get_test_directories(self.script_directory)
test_directory = os.path.abspath(test_directories[package])
test_name = self.get_files(test_directory, job_type)[int(index)]
diff --git a/Tools/ART/python/ART/art_grid.py b/Tools/ART/python/ART/art_grid.py
index 35e032d525890ffacc3bfe4f2d15fbed8ff5ec16..6c74cf0e38eaa956856f46d81b4c57fccdc19ae8 100644
--- a/Tools/ART/python/ART/art_grid.py
+++ b/Tools/ART/python/ART/art_grid.py
@@ -8,6 +8,7 @@ import atexit
import datetime
import glob
import json
+import logging
import os
import re
import shutil
@@ -15,15 +16,38 @@ import sys
import tarfile
import tempfile
+try:
+ import rucio.client
+ RUCIO = True
+except ImportError:
+ # NOTE: defer logging as level is not set yet
+ RUCIO = False
from art_base import ArtBase
from art_header import ArtHeader
-from art_misc import mkdir_p, make_executable, run_command, exit_on_failure
+from art_misc import mkdir_p, make_executable, run_command
+
+MODULE = "art.grid"
class ArtGrid(ArtBase):
"""TBD."""
+ CVMFS_DIRECTORY = '/cvmfs/atlas-nightlies.cern.ch/repo/sw'
+
+ LOG = '.log'
+ JSON = '_EXT0'
+ OUTPUT = '_EXT1'
+
+ ARTPROD = 'artprod'
+ ART_JOB = 'art-job.json'
+ LOG_TGZ = 'log.tgz'
+ JOB_TAR = 'job.tar'
+ JOB_REPORT = 'jobReport.json'
+ JOB_REPORT_ART_KEY = 'art'
+
+ ATHENA_STDOUT = 'athena_stdout.txt'
+
def __init__(self, art_directory, nightly_release, project, platform, nightly_tag, script_directory=None, skip_setup=False, submit_directory=None):
"""TBD."""
super(ArtGrid, self).__init__(art_directory)
@@ -34,104 +58,478 @@ class ArtGrid(ArtBase):
self.script_directory = script_directory
self.skip_setup = skip_setup
self.submit_directory = submit_directory
+ self.rucio_cache = os.path.join(tempfile.gettempdir(), "rucio-cache")
- self.cvmfs_directory = '/cvmfs/atlas-nightlies.cern.ch/repo/sw'
+ def status(self, status):
+ """Print status for usage in gitlab-ci."""
+ print 'art-status:', status
def get_script_directory(self):
"""On demand script directory, only to be called if directory exists."""
if self.script_directory is None:
- self.script_directory = self.cvmfs_directory
- self.script_directory = os.path.join(self.script_directory, self.nightly_release, self.nightly_tag, self.project)
- self.script_directory = os.path.join(self.script_directory, os.listdir(self.script_directory)[0]) # e.g. 21.0.3
- self.script_directory = os.path.join(self.script_directory, os.listdir(self.script_directory)[0], self.platform) # InstallArea/x86_64-slc6-gcc62-opt
+ self.script_directory = ArtGrid.CVMFS_DIRECTORY
+ self.script_directory = os.path.join(self.script_directory, self.nightly_release) # e.g. 21.0
+ self.script_directory = os.path.join(self.script_directory, self.nightly_tag) # e.g. 2017-10-25T2150
+ self.script_directory = os.path.join(self.script_directory, self.project) # e.g. Athena
+ try:
+ self.script_directory = os.path.join(self.script_directory, os.listdir(self.script_directory)[0]) # e.g. 21.0.3
+ self.script_directory = os.path.join(self.script_directory, os.listdir(self.script_directory)[0]) # InstallArea
+ except OSError:
+ self.script_directory = os.path.join(self.script_directory, '*', '*')
+ self.script_directory = os.path.join(self.script_directory, self.platform) # x86_64-slc6-gcc62-opt
return self.script_directory
def is_script_directory_in_cvmfs(self):
"""Return true if the script directory is in cvmfs."""
- return self.get_script_directory().startswith(self.cvmfs_directory)
+ return self.get_script_directory().startswith(ArtGrid.CVMFS_DIRECTORY)
+
+ def exit_if_no_script_directory(self):
+ """Exit with ERROR is script directory does not exist."""
+ log = logging.getLogger(MODULE)
+ if not os.path.isdir(self.get_script_directory()):
+ log.critical('Script directory does not exist: %s', self.get_script_directory())
+ self.status('error')
+ exit(1)
+
+ def copy_art(self, run_dir):
+ """Copy all art files to the the run directory. Returns final script directory to be used."""
+ log = logging.getLogger(MODULE)
+ ART = os.path.join(run_dir, "ART")
+ mkdir_p(ART)
+
+ # get the path of the python classes and support scripts
+ art_python_directory = os.path.join(self.art_directory, '..', 'python', 'ART')
+
+ shutil.copy(os.path.join(self.art_directory, 'art.py'), run_dir)
+ shutil.copy(os.path.join(self.art_directory, 'art-get-input.sh'), run_dir)
+ shutil.copy(os.path.join(self.art_directory, 'art-get-tar.sh'), run_dir)
+ shutil.copy(os.path.join(self.art_directory, 'art-internal.py'), run_dir)
+ shutil.copy(os.path.join(art_python_directory, '__init__.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_base.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_build.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_grid.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_header.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'art_misc.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'docopt.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'docopt_dispatch.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'parallelScheduler.py'), ART)
+ shutil.copy(os.path.join(art_python_directory, 'serialScheduler.py'), ART)
+
+ make_executable(os.path.join(run_dir, 'art.py'))
+ make_executable(os.path.join(run_dir, 'art-get-input.sh'))
+ make_executable(os.path.join(run_dir, 'art-get-tar.sh'))
+ make_executable(os.path.join(run_dir, 'art-internal.py'))
+
+ script_directory = self.get_script_directory()
+
+ # copy a local test directory if needed (only for 'art grid')
+ if not self.is_script_directory_in_cvmfs():
+ script_directory = os.path.basename(os.path.normpath(self.get_script_directory()))
+ target_directory = os.path.join(run_dir, script_directory)
+ log.info("Copying script directory for grid submission to %s", target_directory)
+ shutil.copytree(self.get_script_directory(), target_directory)
+
+ return script_directory
+
+ def get_jedi_id(self, text):
+ """Return Jedi Task Id or 0."""
+ match = re.search(r"jediTaskID=(\d+)", text)
+ return match.group(1) if match else 0
+
+ def get_nightly_release_short(self):
+ """Return a short version of the nightly release."""
+ return re.sub(r"-VAL-.*", "-VAL", self.nightly_release)
+
+ def get_outfile(self, user, package, sequence_tag=0, test_name=None, nightly_tag=None):
+ """Create outfile from parameters."""
+ log = logging.getLogger(MODULE)
+
+ if nightly_tag is None:
+ nightly_tag = self.nightly_tag
+
+ if sequence_tag == 0:
+ if not RUCIO:
+ log.critical("RUCIO not available")
+ exit(1)
+
+ scope = '.'.join(('user', user))
+ outfile = '.'.join(('user', user, 'atlas', self.get_nightly_release_short(), self.project, self.platform, nightly_tag, '*', package, 'log'))
+ rucio_client = rucio.client.Client()
+ for out in rucio_client.list_dids(scope, {'name': outfile}):
+ outfile = os.path.splitext(out)[0]
+ else:
+ outfile = '.'.join(('user', user, 'atlas', self.get_nightly_release_short(), self.project, self.platform, nightly_tag, sequence_tag, package))
+ return outfile if test_name is None else '.'.join((outfile, test_name))
+
+ def copy(self, sequence_tag, package, dst, user):
+ """Copy output from scratch area to eos area."""
+ log = logging.getLogger(MODULE)
+ real_user = os.getenv('USER', ArtGrid.ARTPROD)
+ user = real_user if user is None else user
+ default_dst = '/eos/atlas/atlascerngroupdisk/data-art/grid-output' if real_user == ArtGrid.ARTPROD else '.'
+ dst = default_dst if dst is None else dst
+
+ if package is not None:
+ log.info("Copy %s", package)
+ outfile = self.get_outfile(user, package, sequence_tag)
+
+ return self.copy_output(outfile, dst)
+
+ # make sure script directory exist
+ self.exit_if_no_script_directory()
+
+ # get the test_*.sh from the test directory
+ test_directories = self.get_test_directories(self.get_script_directory())
+ if not test_directories:
+ log.warning('No tests found in directories ending in "test"')
+
+ # copy results for all packages
+ result = 0
+ for package, root in test_directories.items():
+ number_of_tests = len(self.get_files(root, "grid", "all", self.nightly_release, self.project, self.platform))
+ if number_of_tests > 0:
+ # FIXME limited
+ if self.nightly_release == '21.0' and package in ['TriggerTest', 'Tier0ChainTests']:
+ log.info("Copy %s", package)
+ outfile = self.get_outfile(user, package, sequence_tag)
+
+ result |= self.copy_output(outfile, dst)
+ return result
+
+ # Not used yet
+ def download(self, did):
+ """Download did into temp directory."""
+ log = logging.getLogger(MODULE)
+ if not RUCIO:
+ log.critical("RUCIO not available")
+ exit(1)
+
+ # rucio downloads cache properly
+ (exit_code, out, err) = run_command("rucio download --dir " + self.rucio_cache + " " + did)
+ if (exit_code != 0):
+ log.error(err)
+ log.info(out)
+ return exit_code
+
+ # Not used yet
+ def get_job_name(self, user, index, package, sequence_tag, nightly_tag):
+ """
+ Return job name for index.
- def task_list(self, job_type, sequence_tag):
+ job_name is without .sh or .py
+ """
+ log = logging.getLogger(MODULE)
+ if not RUCIO:
+ log.critical("RUCIO not available")
+ exit(1)
+
+ outfile = self.get_outfile(user, package, sequence_tag=sequence_tag, nightly_tag=nightly_tag)
+ log.debug("outfile %s", outfile)
+
+ container_json = outfile + ArtGrid.JSON
+ container_log = outfile + ArtGrid.LOG
+ log.info("Downloading json")
+ self.download(container_json)
+ log.info("Downloading log")
+ self.download(container_log)
+
+ index_formatted = index
+ indexed_json = os.path.join(container_json, '.'.join((container_json, sequence_tag, index_formatted, ArtGrid.JSON)))
+ log.debug("Looking for json")
+ if os.path.exists(indexed_json):
+ with open(indexed_json) as json_file:
+ info = json.load(json_file)
+ test_name = os.path.splitext(info['name'])[0]
+ return test_name
+
+ indexed_log = os.path.join(container_log, '.'.join((container_log, sequence_tag, index_formatted, ArtGrid.LOG_TGZ)))
+ log.debug("Looking for log")
+ if os.path.exists(indexed_log):
+ tar = tarfile.open(indexed_log)
+ for name in tar.getnames():
+ if ArtGrid.ATHENA_STDOUT in name:
+ log.debug("Found %s", ArtGrid.ATHENA_STDOUT)
+ info = tar.extractfile(name).read()
+ # try art-job-name
+ match = re.search(r"art-job-name:\s(\S+)", info)
+ if match:
+ log.debug("Found 'art-job-name'")
+ return os.path.splitext(match.group(1))[0]
+
+ # try Job Name
+ match = re.search(r"Job Name:\s(\S+)", info)
+ if match:
+ log.debug("Found 'Job Name:'")
+ return os.path.splitext(match.group(1))[0]
+
+ log.error("Cannot retrieve job_name from art-job.json or logfile")
+ return None
+
+ def get_test_name(self, rucio_name, rucio_log_name):
+ """Return test_name for log rucio_name."""
+ log = logging.getLogger(MODULE)
+ if not RUCIO:
+ log.critical("RUCIO not available")
+ exit(1)
+
+ tmp_dir = tempfile.mkdtemp()
+ atexit.register(shutil.rmtree, tmp_dir)
+
+ tmp_json = os.path.join(tmp_dir, ArtGrid.ART_JOB)
+
+ if rucio_name is not None:
+ (exit_code, out, err) = run_command(' '.join(('xrdcp -N -f ', rucio_name, tmp_json)))
+ if exit_code == 0:
+ log.debug("copied json %s", rucio_name)
+ with open(tmp_json) as json_file:
+ info = json.load(json_file)
+ test_name = os.path.splitext(info['name'])[0]
+ return test_name
+
+ tmp_log = os.path.join(tmp_dir, ArtGrid.LOG_TGZ)
+
+ if rucio_log_name is not None:
+ (exit_code, out, err) = run_command(' '.join(('xrdcp -N -f ', rucio_log_name, tmp_log)))
+ if exit_code == 0:
+ log.debug("copied log %s %s", rucio_log_name, tmp_log)
+ tar = tarfile.open(tmp_log)
+ for name in tar.getnames():
+ if ArtGrid.ATHENA_STDOUT in name:
+ log.debug("Found %s", ArtGrid.ATHENA_STDOUT)
+ info = tar.extractfile(name).read()
+ # try art-job-name
+ match = re.search(r"art-job-name:\s(\S+)", info)
+ if match:
+ log.debug("Found 'art-job-name'")
+ return os.path.splitext(match.group(1))[0]
+
+ # try Job Name
+ match = re.search(r"Job Name:\s(\S+)", info)
+ if match:
+ log.debug("Found 'Job Name:'")
+ return os.path.splitext(match.group(1))[0]
+
+ log.debug("Cannot retrieve job_name from art-job.json or logfile")
+ return None
+
+ def copy_output(self, outfile, dst):
+ """Copy outfile to dst."""
+ log = logging.getLogger(MODULE)
+ if not RUCIO:
+ log.critical("RUCIO not available")
+ exit(1)
+
+ result = 0
+ outfile_pattern = r"([^\.]+)\.([^\.]+)\.([^\.]+)\.(.+)\.([^\.]+)\.([^\.]+)\.([^\.]+)\.([^\.]+)\.([^\.\n]+)"
+ match = re.search(outfile_pattern, outfile)
+ if not match:
+ log.error("%s does not match pattern", outfile)
+ return 1
+ (user_type, user, experiment, nightly_release, project, platform, nightly_tag, sequence_tag, package) = match.groups()
+ dst_dir = os.path.join(dst, nightly_release, nightly_tag, project, platform, package)
+ log.info(dst_dir)
+
+ scope = '.'.join((user_type, user))
+
+ tmp_dir = tempfile.mkdtemp()
+ atexit.register(shutil.rmtree, tmp_dir)
+
+ tmp_json = os.path.join(tmp_dir, ArtGrid.ART_JOB)
+ tmp_log = os.path.join(tmp_dir, ArtGrid.LOG_TGZ)
+ tmp_tar = os.path.join(tmp_dir, ArtGrid.JOB_TAR)
+
+ jsons = self.get_rucio_map(scope, outfile, ArtGrid.JSON)
+ logs = self.get_rucio_map(scope, outfile, ArtGrid.LOG)
+ tars = self.get_rucio_map(scope, outfile, ArtGrid.OUTPUT)
+
+ # log.debug(jsons)
+ # log.debug(logs)
+
+ for number in tars:
+ # get the test name
+ rucio_name = jsons[number]['rucio_name'] if number in jsons else None
+ rucio_log_name = logs[number]['rucio_name'] if number in logs else None
+ test_name = self.get_test_name(rucio_name, rucio_log_name)
+ if test_name is None:
+ log.error("JSON Lookup Error for test %s", rucio_name)
+ result = 1
+ continue
+
+ # create tmp test directory
+ test_dir = os.path.join(tmp_dir, test_name)
+ mkdir_p(test_dir)
+
+ # copy art-job in, ignore error
+ run_command(' '.join(('xrdcp -N -f', rucio_name, tmp_json)))
+ shutil.copyfile(tmp_json, os.path.join(test_dir, ArtGrid.ART_JOB))
+
+ # copy and unpack log
+ log_source = logs[number]['source']
+ (exit_code, out, err) = run_command(' '.join(('xrdcp -N -f', rucio_log_name, tmp_log)))
+ if exit_code != 0:
+ log.error("Log Unpack Error: %d %s %s", exit_code, out, err)
+ result = 1
+ else:
+ tar = tarfile.open(tmp_log)
+ for member in tar.getmembers():
+ tar.extract(member, path=test_dir)
+ # does not work: tar.extractall()
+ tar.close()
+
+ log.info("Copying: %d %s", number, test_name)
+ log.info("- json: %s", jsons[number]['source'])
+ log.info("- log: %s", log_source)
+ log.info("- tar: %s", tars[number]['source'])
+
+ # copy results and unpack
+ (exit_code, out, err) = run_command(' '.join(('xrdcp -N -f', tars[number]['rucio_name'], tmp_tar)))
+ if exit_code != 0:
+ log.error("TAR Error: %d %s %s", exit_code, out, err)
+ result = 1
+ else:
+ tar = tarfile.open(tmp_tar)
+ tar.extractall(path=test_dir)
+ tar.close()
+
+ # copy to eos
+ dst_target = os.path.join(dst_dir, test_name)
+ log.info("to: %s", dst_target)
+ if dst_target.startswith('/eos'):
+ mkdir_cmd = 'eos mkdir -p'
+ xrdcp_target = 'root://eosatlas.cern.ch/' + dst_target
+ else:
+ mkdir_cmd = 'mkdir -p'
+ xrdcp_target = dst_target
+
+ (exit_code, out, err) = run_command(' '.join((mkdir_cmd, dst_target)))
+ if exit_code != 0:
+ log.error("Mkdir Error: %d %s %s", exit_code, out, err)
+ result = 1
+ else:
+ (exit_code, out, err) = run_command(' '.join(('xrdcp -N -r -v', test_dir, xrdcp_target)))
+ if exit_code not in [0, 51, 54]:
+ # 0 all is ok
+ # 51 File exists
+ # 54 is already copied
+ log.error("XRDCP to EOS Error: %d %s %s", exit_code, out, err)
+ result = 1
+
+ # cleanup
+ shutil.rmtree(test_dir)
+
+ return result
+
+ def get_rucio_map(self, scope, outfile, extension):
+ """Return map of entries by grid_index into { source, rucio_name }."""
+ log = logging.getLogger(MODULE)
+ if not RUCIO:
+ log.critical("RUCIO not available")
+ exit(1)
+
+ CERN = 'CERN-PROD_SCRATCHDISK'
+
+ LOG_PATTERN = r"\.(\d{6})\.log\.tgz"
+ JSON_PATTERN = r"\._(\d{6})\.art-job\.json"
+ OUTPUT_PATTERN = r"\._(\d{6})\.tar"
+ table = {}
+ rucio_client = rucio.client.Client()
+ log.debug("Looking for %s", outfile + extension)
+ for rep in rucio_client.list_replicas([{'scope': scope, 'name': outfile + extension}], schemes=['root']):
+ source = None
+ rucio_name = None
+ log.debug("Found in %s", rep['states'].keys())
+ # first look at CERN
+ if CERN in rep['states'].keys() and rep['states'][CERN] == 'AVAILABLE':
+ source = CERN
+ rucio_name = rep['rses'][CERN][0]
+ else:
+ for rse in rep['states'].keys():
+ if rep['states'][rse] == 'AVAILABLE' and len(rep['rses'][rse]) >= 1:
+ source = rse
+ rucio_name = rep['rses'][rse][0]
+ break
+
+ # maybe not found at all
+ if rucio_name is not None:
+ log.debug("Found rucio name %s in %s", rucio_name, source)
+ pattern = JSON_PATTERN if extension == ArtGrid.JSON else LOG_PATTERN if extension == ArtGrid.LOG else OUTPUT_PATTERN
+ match = re.search(pattern, rucio_name)
+ if match:
+ number = int(match.group(1))
+ else:
+ log.warning("%s does not contain test number using pattern %s skipped...", rucio_name, pattern)
+ continue
+
+ table[number] = {'source': source, 'rucio_name': rucio_name}
+
+ if not table:
+ log.warning("Outfile %s not found or empty", outfile + extension)
+ return table
+
+ def task_package(self, root, package, job_type, sequence_tag, no_action):
+ """TBD."""
+ log = logging.getLogger(MODULE)
+ result = {}
+ number_of_tests = len(self.get_files(root, job_type, "all", self.nightly_release, self.project, self.platform))
+ if number_of_tests > 0:
+ print 'art-package:', package
+ self.status('included')
+ log.info('root %s', root)
+ log.info('Handling %s for %s project %s on %s', package, self.nightly_release, self.project, self.platform)
+ log.info("Number of tests: %d", number_of_tests)
+ submit_dir = os.path.join(self.submit_directory, package)
+ run_dir = os.path.join(submit_dir, "run")
+
+ script_directory = self.copy_art(run_dir)
+
+ result = self.task(script_directory, package, job_type, sequence_tag, no_action)
+ return result
+
+ def task_list(self, job_type, sequence_tag, package=None, no_action=False):
"""TBD."""
+ log = logging.getLogger(MODULE)
# job will be submitted from tmp directory
- submit_directory = tempfile.mkdtemp(dir='.')
+ self.submit_directory = tempfile.mkdtemp(dir='.')
# make sure tmp is removed afterwards
- atexit.register(shutil.rmtree, submit_directory)
+ atexit.register(shutil.rmtree, self.submit_directory)
# make sure script directory exist
- if not os.path.isdir(self.get_script_directory()):
- print 'ERROR: script directory does not exist:', self.get_script_directory()
- print 'art-status: error'
- sys.stdout.flush()
- exit(1)
+ self.exit_if_no_script_directory()
# get the test_*.sh from the test directory
test_directories = self.get_test_directories(self.get_script_directory())
if not test_directories:
- print 'WARNING: No tests found in directories ending in "test"'
- sys.stdout.flush()
+ log.warning('No tests found in directories ending in "test"')
- for package, root in test_directories.items():
- if package in ['TrigInDetValidation']:
- continue
+ all_results = {}
+
+ if package is None:
+ config = None if self.skip_setup else self.get_config()
+ excluded_packages = config.get('excluded-packages', []) if config is not None else []
+
+ # submit tasks for all packages
+ for package, root in test_directories.items():
+ if package in excluded_packages:
+ log.warning("Package %s is excluded", package)
+ else:
+ all_results.update(self.task_package(root, package, job_type, sequence_tag, no_action))
+ else:
+ # Submit single package
+ root = test_directories[package]
+ all_results.update(self.task_package(root, package, job_type, sequence_tag, no_action))
- number_of_tests = len(self.get_files(root, job_type, "all", self.nightly_release, self.project, self.platform))
- if number_of_tests > 0:
- print 'art-package: ' + package
- print 'art-status: included'
- print 'root' + root
- print 'Handling', package, 'for', self.nightly_release, 'project', self.project, 'on', self.platform
- print "Number of tests:", str(number_of_tests)
- sys.stdout.flush()
- submit_dir = os.path.join(submit_directory, package)
- run = os.path.join(submit_dir, "run")
- ART = os.path.join(run, "ART")
- mkdir_p(ART)
-
- # get the path of the python classes and support scripts
- art_python_directory = os.path.join(self.art_directory, '..', 'python', 'ART')
-
- shutil.copy(os.path.join(self.art_directory, 'art.py'), run)
- shutil.copy(os.path.join(self.art_directory, 'art-get-input.sh'), run)
- shutil.copy(os.path.join(self.art_directory, 'art-get-tar.sh'), run)
- shutil.copy(os.path.join(self.art_directory, 'art-internal.py'), run)
- shutil.copy(os.path.join(art_python_directory, '__init__.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'art_base.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'art_build.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'art_grid.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'art_header.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'art_misc.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'docopt.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'docopt_dispatch.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'parallelScheduler.py'), ART)
- shutil.copy(os.path.join(art_python_directory, 'serialScheduler.py'), ART)
-
- make_executable(os.path.join(run, 'art.py'))
- make_executable(os.path.join(run, 'art-get-input.sh'))
- make_executable(os.path.join(run, 'art-get-tar.sh'))
- make_executable(os.path.join(run, 'art-internal.py'))
-
- # copy a local test directory if needed (only for 'art grid')
- if not self.is_script_directory_in_cvmfs():
- local_test_dir = os.path.join(run, os.path.basename(os.path.normpath(self.get_script_directory())))
- print "Copying script directory for grid submission to ", local_test_dir
- shutil.copytree(self.get_script_directory(), local_test_dir)
-
- command = ' '.join((os.path.join(self.art_directory, 'art-internal.py'), 'task', 'grid', '--skip-setup' if self.skip_setup else '', submit_directory, self.get_script_directory(), package, job_type, sequence_tag, self.nightly_release, self.project, self.platform, self.nightly_tag))
- print command
- sys.stdout.flush()
-
- env = os.environ.copy()
- env['PATH'] = '.:' + env['PATH']
- out = exit_on_failure(run_command(command, env=env))
- print out
- sys.stdout.flush()
return 0
- def task(self, package, job_type, sequence_tag):
- """TBD."""
- print 'Running art task'
- sys.stdout.flush()
+ def task(self, script_directory, package, job_type, sequence_tag, no_action=False):
+ """
+ Submit a task, consisting of multiple jobs.
+
+ For 'single' jobs each task contains exactly one job.
+ Returns a map of JediIds to tuples of (package, test_name)
+ """
+ log = logging.getLogger(MODULE)
+ log.info('Running art task')
config = None if self.skip_setup else self.get_config()
grid_options = self.grid_option(config, package, 'grid-exclude-sites', '--excludedSite=')
@@ -143,28 +541,37 @@ class ArtGrid(ArtBase):
env['PATH'] = '.:' + env['PATH']
env['ART_GRID_OPTIONS'] = grid_options
- test_directories = self.get_test_directories(self.get_script_directory())
+ test_directories = self.get_test_directories(script_directory)
test_directory = test_directories[package]
number_of_batch_tests = len(self.get_files(test_directory, job_type, "batch", self.nightly_release, self.project, self.platform))
MAX_OUTFILE_LEN = 132
- user = env['USER'] if self.skip_setup else 'artprod'
- nightly_release_short = re.sub(r"-VAL-.*", "-VAL", self.nightly_release)
- outfile = '.'.join(('user', user, 'atlas', nightly_release_short, self.project, self.platform, self.nightly_tag, sequence_tag, package))
+ user = env['USER'] if self.skip_setup else ArtGrid.ARTPROD
+ outfile = self.get_outfile(user, package, sequence_tag)
+
+ result = {}
# submit batch tests
if number_of_batch_tests > 0:
if len(outfile) > MAX_OUTFILE_LEN:
- print "ERROR: OutFile string length >", MAX_OUTFILE_LEN, ": ", outfile
- sys.stdout.flush()
+ log.error("OutFile string length > %d: ", MAX_OUTFILE_LEN, outfile)
return 1
- print "batch:", nightly_release_short, self.project, self.platform, self.nightly_tag, sequence_tag, package, job_type, str(number_of_batch_tests), grid_options
- sys.stdout.flush()
-
- out = exit_on_failure(run_command(' '.join((os.path.join(self.art_directory, 'art-task-grid.sh'), '--skip-setup' if self.skip_setup else '', self.submit_directory, self.get_script_directory(), package, job_type, sequence_tag, str(number_of_batch_tests), nightly_release_short, self.project, self.platform, self.nightly_tag, outfile)), env=env))
- print out
- sys.stdout.flush()
+ # Batch
+ cmd = ' '.join((os.path.join(self.art_directory, 'art-task-grid.sh'), '--skip-setup' if self.skip_setup else '', self.submit_directory, script_directory, package, job_type, sequence_tag, str(number_of_batch_tests), self.get_nightly_release_short(), self.project, self.platform, self.nightly_tag, outfile))
+ log.info("batch: %s", cmd)
+
+ if not no_action:
+ (exit_code, out, err) = run_command(cmd, env=env)
+ if exit_code != 0:
+ log.error("art-task-grid failed %d", exit_code)
+ print out
+ print err
+ else:
+ jediID = self.get_jedi_id(err)
+ if jediID > 0:
+ result[jediID] = (package, "", outfile)
+ log.info(out)
# submit single tests
index = 1
@@ -173,34 +580,40 @@ class ArtGrid(ArtBase):
header = ArtHeader(job)
inds = header.get('art-input')
nFiles = header.get('art-input-nfiles')
- nEvents = header.get('art-input-nevents')
- files = ','.join(header.get('art-input-file'))
split = header.get('art-input-split')
- outfile_test = '.'.join((outfile, str(index)))
+ outfile_test = self.get_outfile(user, package, sequence_tag, str(index))
if len(outfile_test) > MAX_OUTFILE_LEN:
- print "ERROR: OutFile string length >", MAX_OUTFILE_LEN, ": ", outfile_test
- sys.stdout.flush()
+ log.error("ERROR: OutFile string length > %d : %s ", MAX_OUTFILE_LEN, outfile_test)
return 1
- print "single:", nightly_release_short, self.project, self.platform, self.nightly_tag, sequence_tag, package, job_type, str(split), inds, str(nFiles), str(nEvents), grid_options
- sys.stdout.flush()
+ # Single
+ cmd = ' '.join((os.path.join(self.art_directory, 'art-task-grid.sh'), '--skip-setup' if self.skip_setup else '', '--test-name ' + test_name, '--inDS ' + inds, '--nFiles ' + str(nFiles) if nFiles > 0 else '', self.submit_directory, script_directory, package, job_type, sequence_tag, str(split), self.get_nightly_release_short(), self.project, self.platform, self.nightly_tag, outfile_test))
+ log.info("single: %s", cmd)
- out = exit_on_failure(run_command(' '.join((os.path.join(self.art_directory, 'art-task-grid.sh'), '--skip-setup' if self.skip_setup else '', '--test-name ' + test_name, '--inDS ' + inds, '--nFiles ' + str(nFiles) if nFiles > 0 else '', '--nEvents ' + str(nEvents) if nEvents > 0 else '', '--fileList ' + files if files != '' else '', self.submit_directory, self.get_script_directory(), package, job_type, sequence_tag, str(split), nightly_release_short, self.project, self.platform, self.nightly_tag, outfile_test)), env=env))
- print out
- sys.stdout.flush()
+ if not no_action:
+ (exit_code, out, err) = run_command(cmd, env=env)
+ if exit_code != 0:
+ log.error("art-task-grid failed %d", exit_code)
+ print out
+ print err
+ else:
+ jediID = self.get_jedi_id(err)
+ if jediID > 0:
+ result[jediID] = (package, test_name, outfile_test)
+
+ log.info(out)
index += 1
- return 0
+ return result
def job(self, package, job_type, sequence_tag, index_type, index_or_name, out):
"""TBD."""
- print 'Running art job grid'
- sys.stdout.flush()
+ log = logging.getLogger(MODULE)
+ log.info('Running art job grid')
- print self.nightly_release, self.project, self.platform, self.nightly_tag, package, job_type, str(index_or_name), out
- sys.stdout.flush()
+ log.info("%s %s %s %s %s %s %s %s", self.nightly_release, self.project, self.platform, self.nightly_tag, package, job_type, str(index_or_name), out)
test_directories = self.get_test_directories(self.get_script_directory())
test_directory = test_directories[package]
@@ -213,22 +626,26 @@ class ArtGrid(ArtBase):
else:
test_name = index_or_name
+ log.info("art-job-name: %s", test_name)
+
test_file = os.path.join(test_directory, test_name)
+ # arguments are SCRIPT_DIRECTORY, PACKAGE, TYPE, TEST_NAME, NIGHTLY_RELEASE, PROJECT, PLATFORM, NIGHTLY_TAG
command = ' '.join((test_file, self.get_script_directory(), package, job_type, test_name, self.nightly_release, self.project, self.platform, self.nightly_tag))
- print test_name
- print test_directory
- print command
- sys.stdout.flush()
+ log.debug(test_name)
+ log.debug(test_directory)
+ log.debug(command)
# run the test
env = os.environ.copy()
env['PATH'] = '.:' + env['PATH']
(exit_code, output, error) = run_command(command, env=env)
+ print output
if (exit_code != 0):
+ log.error("Test %s failed %d", str(index_or_name), exit_code)
print error
- print output
- sys.stdout.flush()
+ # NOTE: exit_code always 0
+ print error
# gather results
result = {}
@@ -247,8 +664,20 @@ class ArtGrid(ArtBase):
result['result'].append(item)
# write out results
- with open(os.path.join("art-job.json"), 'w') as jobfile:
+ with open(os.path.join(ArtGrid.ART_JOB), 'w') as jobfile:
json.dump(result, jobfile, sort_keys=True, indent=4, ensure_ascii=False)
+ log.info("Wrote %s", ArtGrid.ART_JOB)
+
+ # grab the content of "jobReport.json", add the art dictionary and write it back
+ if os.path.isfile(ArtGrid.JOB_REPORT):
+ with open(ArtGrid.JOB_REPORT, 'r+') as json_file:
+ info = json.load(json_file)
+ info[ArtGrid.JOB_REPORT_ART_KEY] = result
+ # write out results
+ json_file.seek(0)
+ json.dump(info, json_file, sort_keys=True, indent=4, ensure_ascii=False)
+ json_file.truncate()
+ log.info("Updated %s", ArtGrid.JOB_REPORT)
# pick up the outputs
tar_file = tarfile.open(out, mode='w')
@@ -259,42 +688,97 @@ class ArtGrid(ArtBase):
# remove comments
line = line.split('#', 1)[0]
out_names = re.findall(r"--output[^\s=]*[= ]*(\S*)", line)
- print out_names
+ log.debug(out_names)
for out_name in out_names:
out_name = out_name.strip('\'"')
if os.path.exists(out_name):
- print 'Tar file contain: ', out_name
+ log.info('Tar file contain: %s', out_name)
tar_file.add(out_name)
# pick up art-header named outputs
for path_name in ArtHeader(test_file).get('art-output'):
for out_name in glob.glob(path_name):
- print 'Tar file contains:', out_name
+ log.info('Tar file contains: %s', out_name)
tar_file.add(out_name)
tar_file.close()
# Always return 0
return 0
- def list(self, package, job_type, index_type, json_format=False):
+ def get_grid_map(self, user, package, sequence_tag=0, nightly_tag=None):
+ """Return grid map of test_name to grid_index."""
+ log = logging.getLogger(MODULE)
+ scope = '.'.join(('user', user))
+
+ outfile = self.get_outfile(user, package, sequence_tag=sequence_tag, nightly_tag=nightly_tag)
+ log.debug("outfile %s", outfile)
+ jsons = self.get_rucio_map(scope, outfile, ArtGrid.JSON)
+ logs = self.get_rucio_map(scope, outfile, ArtGrid.LOG)
+
+ result = {}
+ for grid_index in logs:
+ rucio_name = jsons[grid_index]['rucio_name'] if grid_index in jsons else None
+ rucio_log_name = logs[grid_index]['rucio_name'] if grid_index in logs else None
+ test_name = self.get_test_name(rucio_name, rucio_log_name)
+ if test_name is None:
+ # log.warning("JSON Lookup failed for test %s", rucio_log_name if rucio_name is None else rucio_name)
+ continue
+
+ result[test_name] = int(grid_index)
+ return result
+
+ def list(self, package, job_type, index_type, json_format, user):
"""TBD."""
- jobs = self.get_list(self.get_script_directory(), package, job_type, index_type)
+ log = logging.getLogger(MODULE)
+ user = ArtGrid.ARTPROD if user is None else user
+
+ # make sure script directory exist
+ self.exit_if_no_script_directory()
+
+ log.info("Getting grid map...")
+ grid_map = self.get_grid_map(user, package)
+
+ log.info("Getting test names...")
+ test_names = self.get_list(self.get_script_directory(), package, job_type, index_type)
+ json_array = []
+ for test_name in test_names:
+ name = os.path.splitext(test_name)[0]
+ json_array.append({
+ 'name': name,
+ 'grid_index': str(grid_map[name]) if name in grid_map else '-1'
+ })
+
if json_format:
- json.dump(jobs, sys.stdout, sort_keys=True, indent=4)
+ json.dump(json_array, sys.stdout, sort_keys=True, indent=4)
return 0
- i = 1
- for job in jobs:
- print str(i) + ' ' + job
+
+ i = 0
+ for entry in json_array:
+ print str(i) + ' ' + entry['name'] + (' ' + entry['grid_index'])
i += 1
- sys.stdout.flush()
+
+ # print warnings
+ for entry in json_array:
+ if entry['grid_index'] < 0:
+ log.warning('test %s could not be found in json or log', entry['name'])
+
return 0
- def log(self, package, test_name):
+ def log(self, package, test_name, user):
"""TBD."""
- tar = self.get_tar(package, test_name, '.log')
+ log = logging.getLogger(MODULE)
+ user = ArtGrid.ARTPROD if user is None else user
+
+ # make sure script directory exist
+ self.exit_if_no_script_directory()
+
+ tar = self.open_tar(user, package, test_name, ArtGrid.LOG)
+ if tar is None:
+ log.error("No log tar file found")
+ return 1
for name in tar.getnames():
- if 'athena_stdout.txt' in name:
+ if ArtGrid.ATHENA_STDOUT in name:
f = tar.extractfile(name)
content = f.read()
print content
@@ -302,49 +786,64 @@ class ArtGrid(ArtBase):
tar.close()
return 0
- def output(self, package, test_name, file_name):
+ def output(self, package, test_name, user):
"""TBD."""
- tar = self.get_tar(package, test_name, '_EXT1')
+ log = logging.getLogger(MODULE)
+ user = ArtGrid.ARTPROD if user is None else user
- for member in tar.getmembers():
- if file_name in member.name:
- tar.extractall(path='.', members=[member])
- break
+ # make sure script directory exist
+ self.exit_if_no_script_directory()
+
+ outfile = self.get_outfile(user, package)
+ tar_dir = os.path.join(tempfile.gettempdir(), outfile + ArtGrid.OUTPUT)
+ mkdir_p(tar_dir)
+
+ tar = self.open_tar(user, package, test_name, ArtGrid.OUTPUT)
+ if tar is None:
+ log.error("No output tar file found")
+ return 1
+
+ tar.extractall(path=tar_dir)
tar.close()
+ print "Output extracted in", tar_dir
return 0
- def compare(self, package, test_name, days, file_names):
+ def compare(self, package, test_name, days, file_names, user):
"""TBD."""
+ log = logging.getLogger(MODULE)
+ user = ArtGrid.ARTPROD if user is None else user
+
previous_nightly_tag = self.get_previous_nightly_tag(days)
- print "Previous Nightly Tag:", str(previous_nightly_tag)
+ log.info("LOG Previous Nightly Tag: %s", str(previous_nightly_tag))
+ print "PRINT Previous Nightly Tag", str(previous_nightly_tag)
+
if previous_nightly_tag is None:
- print "ERROR: No previous nightly tag found"
- # do not flag as error, to make sure tar file gets uploaded
- return 0
+ log.error("No previous nightly tag found")
+ return 1
ref_dir = os.path.join('.', 'ref-' + previous_nightly_tag)
mkdir_p(ref_dir)
- tar = self.get_tar(package, test_name, '_EXT1', previous_nightly_tag)
+ tar = self.open_tar(user, package, test_name, ArtGrid.OUTPUT, previous_nightly_tag)
if tar is None:
- print "ERROR: No comparison tar file found"
- # do not flag as error, to make sure tar file gets uploaded
- return 0
+ log.error("No comparison tar file found")
+ return 1
for member in tar.getmembers():
if member.name in file_names:
tar.extractall(path=ref_dir, members=[member])
tar.close()
+ result = 0
for file_name in file_names:
ref_file = os.path.join(ref_dir, file_name)
if os.path.isfile(ref_file):
print "art-compare:", previous_nightly_tag, file_name
- self.compare_ref(file_name, ref_file, 10)
+ result |= self.compare_ref(file_name, ref_file, 10)
else:
- print "ERROR:", ref_file, "not found in tar file"
- # do not flag as error, to make sure tar file gets uploaded
- return 0
+ log.error("%s not found in tar file", ref_file)
+ result = 1
+ return result
def grid_option(self, config, package, key, option_key):
"""Based on config, return value for key, or ''.
@@ -365,67 +864,49 @@ class ArtGrid(ArtBase):
else:
return option_key + global_value + ('' if value is None else ', ' + value)
- def get_tar(self, package, test_name, extension, nightly_tag=None):
+ def open_tar(self, user, package, test_name, extension, nightly_tag=None):
"""Open tar file for particular release."""
- if nightly_tag is None:
- nightly_tag = self.nightly_tag
-
- try:
- job_type = self.get_type(self.get_test_directories(self.get_script_directory())[package], test_name)
- print "Job Type:", job_type
- files = self.get_list(self.get_script_directory(), package, job_type, "batch")
- number_of_tests = len(files)
- index = files.index(test_name)
- print "Index:", index
- except KeyError:
- print package, "does not exist in tests of ", self.get_script_directory()
- return None
+ log = logging.getLogger(MODULE)
+ if not RUCIO:
+ log.critical("RUCIO not available")
+ exit(1)
- try:
- tmpdir = os.environ['TMPDIR']
- except KeyError:
- tmpdir = '.'
- print "Using", tmpdir, "for tar file download"
+ log.info("Getting grid map...")
+ grid_map = self.get_grid_map(user, package, nightly_tag=nightly_tag)
- # run in correct environment
- env = os.environ.copy()
- env['PATH'] = '.:' + env['PATH']
+ name = os.path.splitext(test_name)[0]
+ if name not in grid_map:
+ log.error("No log or tar found for package %s or test %s", package, test_name)
+ return None
- # Grid counts from 1, retries will up that number every time by total number of jobs
- index += 1
- retries = 3
- retry = 0
+ grid_index = grid_map[name]
+ log.info("Grid Index: %d", grid_index)
- (code, out, err) = (0, "", "")
- while retry < retries:
- try_index = (retry * number_of_tests) + index
- print (retry + 1), ": Get tar for index ", try_index
- # run art-get-tar.sh
- (code, out, err) = run_command(' '.join((os.path.join(self.art_directory, "art-get-tar.sh"), package, str(try_index), "_EXT1", self.nightly_release, self.project, self.platform, nightly_tag)), dir=tmpdir, env=env)
- if code == 0 and out != '':
+ scope = '.'.join(('user', user))
+ outfile = self.get_outfile(user, package, nightly_tag=nightly_tag)
+ rucio_map = self.get_rucio_map(scope, outfile, extension)
+ if grid_index not in rucio_map:
+ log.error("No entry in rucio map for %d", grid_index)
+ return None
- match = re.search(r"TAR_NAME=(.*)", out)
- if match:
- tar_name = match.group(1)
- print "Matched TAR_NAME ", tar_name
+ rucio_name = rucio_map[grid_index]['rucio_name']
+ log.info("RUCIO: %s", rucio_name)
- if tar_name != "":
- print "Tar Name:", tar_name
- break
+ tmp_dir = tempfile.mkdtemp()
+ atexit.register(shutil.rmtree, tmp_dir)
- retry += 1
+ tmp_tar = os.path.join(tmp_dir, os.path.basename(rucio_name))
- if retry >= retries:
- print "Code:", str(code)
- print "Err:", err
- print "Out:", out
+ (exit_code, out, err) = run_command(' '.join(('xrdcp -N -f', rucio_name, tmp_dir)))
+ if exit_code != 0:
+ log.error("TAR Error: %s %d %s %s", rucio_name, exit_code, out, err)
return None
- return tarfile.open(os.path.join(tmpdir, tar_name.replace(':', '/', 1)))
+ return tarfile.open(tmp_tar)
def get_previous_nightly_tag(self, days):
"""TBD. 21:00 is cutoff time."""
- directory = os.path.join(self.cvmfs_directory, self.nightly_release)
+ directory = os.path.join(ArtGrid.CVMFS_DIRECTORY, self.nightly_release)
tags = os.listdir(directory)
tags.sort(reverse=True)
tags = [x for x in tags if re.match(r'\d{4}-\d{2}-\d{2}T\d{2}\d{2}', x)]
diff --git a/Tools/ART/python/ART/art_header.py b/Tools/ART/python/ART/art_header.py
index 046de1b53d7ae113a96cd6ffd266501063149acd..372cff4983283e1a84f02bd170b155ffc3cb8c47 100644
--- a/Tools/ART/python/ART/art_header.py
+++ b/Tools/ART/python/ART/art_header.py
@@ -4,12 +4,15 @@
__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
+import logging
import re
from types import IntType
from types import ListType
from types import StringType
+MODULE = "art.header"
+
class ArtHeader(object):
"""TBD."""
@@ -36,9 +39,7 @@ class ArtHeader(object):
self.add('art-include', ListType, ['*'])
self.add('art-output', ListType, [])
self.add('art-input', StringType, None)
- self.add('art-input-file', ListType, [])
- self.add('art-input-nfiles', IntType, 0)
- self.add('art-input-nevents', IntType, 0)
+ self.add('art-input-nfiles', IntType, 1)
self.add('art-input-split', IntType, 0)
self.read(filename)
@@ -49,7 +50,7 @@ class ArtHeader(object):
self.header[key]['type'] = value_type
self.header[key]['default'] = default_value
self.header[key]['constraint'] = constraint
- self.header[key]['value'] = None # never set
+ self.header[key]['value'] = None # e.g. the value was never set
def is_list(self, key):
"""TBD."""
@@ -57,26 +58,38 @@ class ArtHeader(object):
def read(self, filename):
"""Read all headers from file."""
+ log = logging.getLogger(MODULE)
for line in open(filename, "r"):
line_match = self.header_format.match(line)
if line_match:
- key = line_match.group(1)
- value = line_match.group(2)
- if key in self.header and self.header[key]['type'] == StringType:
- value = value.strip()
-
- if self.is_list(key):
- if self.header[key]['value'] is None:
- self.header[key]['value'] = []
- self.header[key]['value'].append(value)
- else:
- if key not in self.header:
- self.header[key] = {}
- self.header[key]['value'] = value
+ try:
+ key = line_match.group(1)
+ value = line_match.group(2)
+ if key in self.header:
+ if self.header[key]['type'] == StringType:
+ value = value.strip()
+ elif self.header[key]['type'] == IntType:
+ value = int(value)
+
+ if self.is_list(key):
+ # handle list types
+ if self.header[key]['value'] is None:
+ self.header[key]['value'] = []
+ self.header[key]['value'].append(value)
+ else:
+ # handle values
+ if key not in self.header:
+ log.warning("Unknown art-header %s: %s in file %s", key, value, filename)
+ self.header[key] = {}
+ self.header[key]['value'] = value
+ except ValueError:
+ log.error("Invalid value in art-header %s: %s in file %s", key, value, filename)
def get(self, key):
"""TBD."""
+ log = logging.getLogger(MODULE)
if key not in self.header:
+ log.warning("Art seems to look for a header key %s which is not in the list of defined headers.", key)
return None
if self.header[key]['value'] is None:
@@ -86,8 +99,9 @@ class ArtHeader(object):
def print_it(self):
"""TBD."""
+ log = logging.getLogger(MODULE)
for key in self.header:
- print key, self.header[key]['type'], self.header[key]['default'], self.header[key]['value'], self.header[key]['constraint']
+ log.info("%s: %s %s %s %s", key, self.header[key]['type'], self.header[key]['default'], self.header[key]['value'], self.header[key]['constraint'])
def validate(self):
"""
@@ -100,34 +114,35 @@ class ArtHeader(object):
- a value is found of the wrong value_type
- a value is found outside the constraint
"""
+ log = logging.getLogger(MODULE)
for line in open(self.filename, "r"):
if self.header_format_error1.match(line):
- print "LINE: ", line.rstrip()
- print "ERROR: Header Validation - invalid header format, use space between '# and art-xxx' in file", self.filename
- print
+ log.error("LINE: %s", line.rstrip())
+ log.error("Header Validation - invalid header format, use space between '# and art-xxx' in file %s", self.filename)
+ log.error("")
if self.header_format_error2.match(line):
- print "LINE: ", line.rstrip()
- print "ERROR: Header Validation - invalid header format, too many spaces between '# and art-xxx' in file", self.filename
- print
+ log.error("LINE: %s", line.rstrip())
+ log.error("Header Validation - invalid header format, too many spaces between '# and art-xxx' in file %s", self.filename)
+ log.error("")
if self.header_format_error3.match(line):
- print "LINE: ", line.rstrip()
- print "ERROR: Header Validation - invalid header format, use at least one space between ': and value' in file", self.filename
- print
+ log.error("LINE: %s", line.rstrip())
+ log.error("Header Validation - invalid header format, use at least one space between ': and value' in file %s", self.filename)
+ log.error("")
for key in self.header:
if 'type' not in self.header[key]:
- print "ERROR: Header Validation - Invalid key:", key, "in file", self.filename
- print
+ log.error("Header Validation - Invalid key: %s in file %s", key, self.filename)
+ log.error("")
continue
if type(self.header[key]['value']) != self.header[key]['type']:
if not isinstance(self.header[key]['value'], type(None)):
- print "ERROR: Header Validation - value_type:", type(self.header[key]['value']), "not valid for key:", key, "expected value_type:", self.header[key]['type'], "in file", self.filename
- print
+ log.error("Header Validation - value_type: %s not valid for key: %s, expected value_type: %s in file %s", type(self.header[key]['value']), key, self.header[key]['type'], self.filename)
+ log.error("")
if self.header[key]['constraint'] is not None and self.header[key]['value'] not in self.header[key]['constraint']:
if self.header[key]['value'] is None:
- print "ERROR: Header Validation - missing key:", key, "in file", self.filename
+ log.error("Header Validation - missing key: %s in file %s", key, self.filename)
else:
- print "ERROR: Header Validation - value:", self.header[key]['value'], "for key:", key, "not in constraints:", self.header[key]['constraint'], "in file", self.filename
- print
+ log.error("Header Validation - value: %s for key: %s not in constraints: %s in file %s", self.header[key]['value'], key, self.header[key]['constraint'], self.filename)
+ log.error("")
return 0
diff --git a/Tools/ART/python/ART/art_misc.py b/Tools/ART/python/ART/art_misc.py
index edb77fe5ebe1783f121b46d7b5a40e240254b2db..d81f3440599e4219e12ac519863f43bc10c3bc95 100644
--- a/Tools/ART/python/ART/art_misc.py
+++ b/Tools/ART/python/ART/art_misc.py
@@ -5,14 +5,40 @@
__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
import errno
+import logging
import os
import shlex
import subprocess
+import sys
+
+MODULE = "art.misc"
+
+
+def set_log(kwargs):
+ """TBD."""
+ level = logging.DEBUG if kwargs['verbose'] else logging.WARN if kwargs['quiet'] else logging.INFO
+ log = logging.getLogger("art")
+ log.setLevel(level)
+
+ # create and attach new handler, disable propagation to root logger to avoid double messages
+ handler = logging.StreamHandler(sys.stdout)
+ format_string = "%(asctime)s %(name)15s.%(funcName)-15s %(levelname)8s %(message)s"
+ date_format_string = None
+ formatter = logging.Formatter(format_string, date_format_string)
+ handler.setFormatter(formatter)
+ log.addHandler(handler)
+ log.propagate = False
def run_command(cmd, dir=None, shell=False, env=None):
- """Run the given command locally and returns the output, err and exit_code."""
- # print "Execute: " + cmd
+ """
+ Run the given command locally.
+
+ The command runs as separate subprocesses for every piped command.
+ Returns tuple of exit_code, output and err.
+ """
+ log = logging.getLogger(MODULE)
+ log.debug("Execute: %s", cmd)
if "|" in cmd:
cmd_parts = cmd.split('|')
else:
@@ -33,39 +59,6 @@ def run_command(cmd, dir=None, shell=False, env=None):
return exit_code, str(output), str(err)
-def exit_on_failure((exit_code, out, err)):
- """Check exitcode and print statement and exit if needed."""
- if exit_code == 0:
- print err
- return out
-
- print "Error:", exit_code
- print "StdOut:", out
- print "StdErr:", err
-
- print 'art-status: error'
-
- exit(exit_code)
-
-
-def code_tbr((exit_code, out, err)):
- """Check exitcode and print statement."""
- if exit_code == 0:
- print out
- return exit_code
-
- print "Error:", exit_code
- print "StdOut:", out
- print "StdErr:", err
-
- return exit_code
-
-
-def redirect_tbr((exitcode, out, err)):
- """Check exitcode."""
- return exitcode
-
-
def is_exe(fpath):
"""Return True if fpath is executable."""
return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
diff --git a/Tools/ART/scripts/art-get-tar.sh b/Tools/ART/scripts/art-get-tar.sh
index 23de65640703a42cc83f6497ea197d2092071917..ec552a5ad34239fe87beda432299b7d1bf9cd9bb 100755
--- a/Tools/ART/scripts/art-get-tar.sh
+++ b/Tools/ART/scripts/art-get-tar.sh
@@ -2,14 +2,14 @@
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
#
# NOTE do NOT run with /bin/bash -x as the output is too big for gitlab-ci
-# arguments: PACKAGE INDEX EXTENSION NIGHTLY_RELEASE PROJECT PLATFORM NIGHTLY_TAG
+# arguments: PACKAGE GRID_INDEX EXTENSION NIGHTLY_RELEASE PROJECT PLATFORM NIGHTLY_TAG
#
# author : Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>
#
# example: Tier0ChainTests 4 _EXT0 21.0 Athena x86_64-slc6-gcc62-opt 2017-07-24T2151
if [ $# -ne 7 ]; then
- echo 'Usage: art-get-tar.sh PACKAGE INDEX EXTENSION NIGHTLY_RELEASE PROJECT PLATFORM NIGHTLY_TAG'
+ echo 'Usage: art-get-tar.sh PACKAGE GRID_INDEX EXTENSION NIGHTLY_RELEASE PROJECT PLATFORM NIGHTLY_TAG'
exit 1
fi
@@ -19,7 +19,7 @@ ART_USER='artprod'
PACKAGE=$1
shift
-INDEX=$1
+GRID_INDEX=$1
shift
EXTENSION=$1
shift
@@ -54,7 +54,7 @@ echo "Tar files in the Container: ${FILELIST}"
CONTAINER=`rucio list-dids ${CONTAINER_LIST} --filter type=container | grep ${NIGHTLY_TAG} | sort -r | cut -d ' ' -f 2 | head -n 1`
echo "Container: ${CONTAINER}"
-printf -v INDEX_FORMAT '_%06d.tar' ${INDEX}
+printf -v INDEX_FORMAT '_%06d.tar' ${GRID_INDEX}
TAR_NAME=`rucio list-files --csv ${CONTAINER} | grep ${INDEX_FORMAT} | cut -d ',' -f 1`
echo "Tar Name: ${TAR_NAME}"
diff --git a/Tools/ART/scripts/art-internal.py b/Tools/ART/scripts/art-internal.py
index 029ac3b1e559863b12e0ec70079bacf03cdafbfe..8c700482a7b965bf8a674ee24b0fab2bcd65c025 100755
--- a/Tools/ART/scripts/art-internal.py
+++ b/Tools/ART/scripts/art-internal.py
@@ -4,20 +4,19 @@
ART-internal - ATLAS Release Tester (internal command).
Usage:
- art-internal.py job build [-v] <script_directory> <package> <job_type> <sequence_tag> <index> <out> <nightly_release> <project> <platform> <nightly_tag>
- art-internal.py job grid [-v --skip-setup] <script_directory> <package> <job_type> <sequence_tag> <index_type> <index_or_name> <out> <nightly_release> <project> <platform> <nightly_tag>
- art-internal.py task build [-v] <script_directory> <package> <job_type> <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
- art-internal.py task grid [-v --skip-setup] <submit_directory> <script_directory> <package> <job_type> <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
+ art-internal.py job build [-v -q] <script_directory> <package> <job_type> <sequence_tag> <index> <out> <nightly_release> <project> <platform> <nightly_tag>
+ art-internal.py job grid [-v -q --skip-setup] <script_directory> <package> <job_type> <sequence_tag> <index_type> <index_or_name> <out> <nightly_release> <project> <platform> <nightly_tag>
Options:
--skip-setup Do not run atlas setup or voms
-h --help Show this screen.
- -v, --verbose Show details.
+ -q --quiet Show less information, only warnings and errors
+ -v --verbose Show more information, debug level
--version Show version.
Sub-commands:
- job Runs a single job, given a particular index
- task Runs a single task, consisting of given number of jobs
+ job Run a single job, given a particular index
+ copy Copy outputs to eos area
Arguments:
index_type Type of index used (e.g. batch or single)
@@ -37,6 +36,7 @@ Arguments:
__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
+import logging
import os
import sys
@@ -44,47 +44,45 @@ from ART.docopt_dispatch import dispatch
from ART import ArtGrid, ArtBuild
+from ART.art_misc import set_log
+
+MODULE = "art.internal"
+
@dispatch.on('job', 'build')
def job_build(script_directory, package, job_type, sequence_tag, index, out, nightly_release, project, platform, nightly_tag, **kwargs):
- """TBD.
+ """Run a single job, given a particular index.
Tests are called with the following parameters:
SCRIPT_DIRECTORY, PACKAGE, TYPE, TEST_NAME
"""
+ set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
exit(ArtBuild(art_directory, nightly_release, project, platform, nightly_tag, script_directory).job(package, job_type, sequence_tag, index, out))
@dispatch.on('job', 'grid')
def job_grid(script_directory, package, job_type, sequence_tag, index_type, index_or_name, out, nightly_release, project, platform, nightly_tag, **kwargs):
- """TBD.
+ """Run a single job, given a particular index.
Tests are called with the following parameters:
SCRIPT_DIRECTORY, PACKAGE, TYPE, TEST_NAME, NIGHTLY_RELEASE, PROJECT, PLATFORM, NIGHTLY_TAG
"""
+ set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
skip_setup = kwargs['skip_setup']
exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, skip_setup).job(package, job_type, sequence_tag, index_type, index_or_name, out))
-@dispatch.on('task', 'build')
-def task_build(script_directory, package, job_type, sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
- """TBD."""
- art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
- exit(ArtBuild(art_directory, nightly_release, project, platform, nightly_tag, script_directory).task(package, job_type, sequence_tag))
-
-
-@dispatch.on('task', 'grid')
-def task_grid(submit_directory, script_directory, package, job_type, sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
- """TBD."""
- art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
- skip_setup = kwargs['skip_setup']
- exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, skip_setup, submit_directory).task(package, job_type, sequence_tag))
-
-
if __name__ == '__main__':
+ if sys.version_info < (2, 7, 0):
+ sys.stderr.write("You need python 2.7 or later to run this script\n")
+ exit(1)
+
# NOTE: import should be here, to keep the order of the decorators (module first, art last and unused)
from art import __version__
- print "ART_PATH", os.path.dirname(os.path.realpath(sys.argv[0]))
+ logging.basicConfig()
+ log = logging.getLogger(MODULE)
+ log.setLevel(logging.INFO)
+ log.info("ART_PATH %s", os.path.dirname(os.path.realpath(sys.argv[0])))
dispatch(__doc__, version=os.path.splitext(os.path.basename(__file__))[0] + ' ' + __version__)
diff --git a/Tools/ART/scripts/art-share.py b/Tools/ART/scripts/art-share.py
new file mode 100755
index 0000000000000000000000000000000000000000..77a97f00d1dc9b65cfb7a0974d9854f6a90b3e17
--- /dev/null
+++ b/Tools/ART/scripts/art-share.py
@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+"""
+ART - ATLAS Release Tester - Share.
+
+Usage:
+ art-share.py [options] <data_directory>
+
+Options:
+ -h --help Show this screen
+ --version Show version
+ --q --quiet Show less information, only warnings and errors
+ --write Write to directory
+ -v --verbose Show more information, debug level
+
+Arguments:
+ data_directory directory to scan for shared files
+
+"""
+
+__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
+
+import collections
+import hashlib
+import logging
+import os
+import sys
+
+try:
+ import scandir as scan
+except ImportError:
+ import os as scan
+
+from ART.docopt import docopt
+
+MODULE = "art.share"
+
+
+class ArtShare(object):
+ """Class for copying input files.
+
+ Files are copies to the .art area under their SHA1 sum. The orignal file is replaced
+ by a symbolic link. Any duplicates will result in the same SHA1 sum and thus just
+ be replaced by their symbiolic link. Removing a file results in removing a link.
+ If the .art directory has files without links pointing to them, these files are also
+ removed.
+ """
+
+ def __init__(self, data_directory, write):
+ """Constructor of ArtShare."""
+ log = logging.getLogger(MODULE)
+ self.data_directory = data_directory
+ self.write = write
+
+ self.art_directory = '.art'
+ self.extension = '.art'
+ self.count = collections.defaultdict(int) # key is sha1
+
+ if not os.path.isdir(self.data_directory):
+ log.critical("data_directory does not exist: %s", self.data_directory)
+ sys.exit(1)
+
+ if self.write:
+ log.warning("NOTE - Changing File System")
+ else:
+ log.warning("NOT Changing File System, use --write to change the File System")
+
+ self.share()
+
+ def create_sha1sum(self, path):
+ """Calculate SHA1 from file on path."""
+ BUF_SIZE = 65536
+ sha1 = hashlib.sha1()
+ with open(path, 'rb') as f:
+ while True:
+ data = f.read(BUF_SIZE)
+ if not data:
+ break
+ sha1.update(data)
+ return sha1.hexdigest()
+
+ def sha1sum(self, artpath):
+ """Retrieve SHA1 from artpath specification (in the filename)."""
+ f = os.path.basename(artpath)
+ return os.path.splitext(f)[0]
+
+ def share(self):
+ """Share the files by copying."""
+ log = logging.getLogger(MODULE)
+ art_root = os.path.join(self.data_directory, self.art_directory)
+ if not os.path.isdir(art_root):
+ log.info("NOTE - art_directory does not exist.")
+ log.info(" creating... %s", art_root)
+ if self.write:
+ os.makedirs(art_root)
+
+ if os.path.isdir(art_root):
+ for f in os.listdir(art_root):
+ sha1art = os.path.join(art_root, f)
+ if os.path.isfile(sha1art):
+ sha1 = self.sha1sum(sha1art)
+ self.count[sha1] = 0
+
+ i = 0
+ for root, dirs, files in scan.walk(self.data_directory):
+ for f in files:
+ if os.path.basename(root) == self.art_directory:
+ continue
+
+ path = os.path.join(root, f)
+ i += 1
+ if os.path.islink(path):
+ # link
+ if not os.path.exists(path):
+ log.warning("WARNING - Stale link/file, skipping")
+ log.warning(" path: %s", path)
+ continue
+ sha1 = self.sha1sum(os.path.realpath(path))
+ log.debug("Link %d path %s", i, path)
+ log.debug("SHA1 %s", sha1)
+ self.count[sha1] += 1
+ else:
+ # file
+ byte_size = os.path.getsize(path)
+ if byte_size <= 0:
+ log.warning("WARNING - zero sized file, skipping")
+ log.warning(" path: %s", path)
+ continue
+
+ megabyte_size = byte_size / 1024 / 1024
+ log.debug("File %d %s", i, path)
+ log.debug("File size %d", megabyte_size)
+ sha1 = self.create_sha1sum(path) if self.write or megabyte_size < 100 else "????????????????????????????????????????"
+ log.debug("SHA1 %s", sha1)
+
+ art_path = os.path.join(art_root, sha1 + self.extension)
+ art_relpath = os.path.relpath(art_path, os.path.dirname(path))
+
+ if sha1 not in self.count.keys():
+ log.info(" Moving file from %s", path)
+ log.info(" to %s", art_path)
+ if self.write:
+ os.rename(path, art_path)
+ self.count[sha1] = 0
+ else:
+ log.info(" Removing file from %s", path)
+ if self.write:
+ os.remove(path)
+
+ log.info(" Creating link from %s", path)
+ log.info(" to %s", art_relpath)
+ if self.write:
+ os.symlink(art_relpath, path)
+ self.count[sha1] += 1
+
+ for sha1, count in self.count.iteritems():
+ if count <= 0:
+ art_path = os.path.join(art_root, sha1 + self.extension)
+ log.info(" Removing file %s", art_path)
+ if self.write:
+ os.remove(art_path)
+
+
+if __name__ == '__main__':
+ if sys.version_info < (2, 7, 0):
+ sys.stderr.write("You need python 2.7 or later to run this script\n")
+ exit(1)
+
+ # NOTE: import should be here, to keep the order of the decorators (module first, art last and unused)
+ from art import __version__
+
+ logging.basicConfig()
+ log = logging.getLogger(MODULE)
+
+ arguments = docopt(__doc__, version=os.path.splitext(os.path.basename(__file__))[0] + ' ' + __version__)
+ level = logging.DEBUG if arguments['--verbose'] else logging.WARN if arguments['--quiet'] else logging.INFO
+ log.setLevel(level)
+ ArtShare(arguments['<data_directory>'], arguments['--write'])
diff --git a/Tools/ART/scripts/art-task-grid.sh b/Tools/ART/scripts/art-task-grid.sh
index 2c15ff91e7c0452447a7bc604218c9339c2eff00..ad7be6ed95e0a2b1d4e9657e1a35d5ce89c0b790 100755
--- a/Tools/ART/scripts/art-task-grid.sh
+++ b/Tools/ART/scripts/art-task-grid.sh
@@ -21,9 +21,10 @@ if [ $1 == "--skip-setup" ]; then
fi
TYPE_OPTION="batch %RNDM:0"
PATHENA_OPTIONS="--destSE=CERN-PROD_SCRATCHDISK"
+PATHENA_TYPE_OPTIONS=""
if [ $1 == "--test-name" ]; then
TYPE_OPTION="single $2"
- PATHENA_OPTIONS="--destSE=CERN-PROD_SCRATCHDISK --forceStaged"
+ PATHENA_TYPE_OPTIONS="--forceStaged"
shift
shift
fi
@@ -39,18 +40,6 @@ if [ $1 == "--nFiles" ]; then
shift
shift
fi
-NEVENTS=""
-if [ $1 == "--nEvents" ]; then
- NEVENTS="--nEventsPerFile $2"
- shift
- shift
-fi
-FILELIST=""
-if [ $1 == "--fileList" ]; then
- FILELIST="--fileList $2"
- shift
- shift
-fi
SUBMIT_DIRECTORY=$1
shift
SCRIPT_DIRECTORY=$1
@@ -101,7 +90,7 @@ fi
# NOTE: for art-internal.py the current dir can be used as it is copied there
cd ${SUBMIT_DIRECTORY}/${PACKAGE}/run
SUBCOMMAND="./art-internal.py job grid ${SCRIPT_DIRECTORY} ${PACKAGE} ${TYPE} ${SEQUENCE_TAG} ${TYPE_OPTION} %OUT.tar ${NIGHTLY_RELEASE_SHORT} ${PROJECT} ${PLATFORM} ${NIGHTLY_TAG}"
-CMD="pathena ${GRID_OPTIONS} ${PATHENA_OPTIONS} --noBuild --expertOnly_skipScout --trf \"${SUBCOMMAND}\" ${SPLIT} --outDS ${OUTFILE} --extOutFile art-job.json ${INDS} ${NFILES} ${NEVENTS} ${FILELIST}"
+CMD="pathena ${GRID_OPTIONS} ${PATHENA_OPTIONS} ${PATHENA_TYPE_OPTIONS} --noBuild --expertOnly_skipScout --trf \"${SUBCOMMAND}\" ${SPLIT} --outDS ${OUTFILE} --extOutFile art-job.json ${INDS} ${NFILES}"
#--disableAutoRetry
#--excludedSite=ANALY_TECHNION-HEP-CREAM
diff --git a/Tools/ART/scripts/art.py b/Tools/ART/scripts/art.py
index 9a0da14c933f88fded8f57ea55f36d1728e0a2d7..54d8d37a47c6fa3a5689c820863bbb41a6038431 100755
--- a/Tools/ART/scripts/art.py
+++ b/Tools/ART/scripts/art.py
@@ -4,38 +4,44 @@
ART - ATLAS Release Tester.
Usage:
- art.py run [-v --type=<T> --max-jobs=<N> --ci] <script_directory> <sequence_tag>
- art.py grid [-v --type=<T>] <script_directory> <sequence_tag>
- art.py submit [-v --type=<T>] <sequence_tag> <nightly_release> <project> <platform> <nightly_tag>
- art.py validate [-v] <script_directory>
- art.py included [-v --type=<T> --test-type=<TT>] <script_directory> [<nightly_release> <project> <platform>]
- art.py compare grid [-v --days=<D>] <nightly_release> <project> <platform> <nightly_tag> <package> <test_name> <file_name>...
- art.py compare ref [-v] <file_name> <ref_file>
- art.py download [-v] <input_file>
- art.py list grid [-v --json --type=<T> --test-type=<TT>] <package> <nightly_release> <project> <platform> <nightly_tag>
- art.py log grid [-v] <package> <test_name> <nightly_release> <project> <platform> <nightly_tag>
- art.py output grid [-v] <package> <test_name> <file_name> <nightly_release> <project> <platform> <nightly_tag>
+ art.py run [-v -q --type=<T> --max-jobs=<N> --ci] <script_directory> <sequence_tag>
+ art.py grid [-v -q --type=<T> -n] <script_directory> <sequence_tag>
+ art.py submit [-v -q --type=<T> -n] <sequence_tag> <nightly_release> <project> <platform> <nightly_tag> [<package>]
+ art.py copy [-v -q --user=<user> --dst=<dir>] <sequence_tag> <nightly_release> <project> <platform> <nightly_tag> [<package>]
+ art.py validate [-v -q] <script_directory>
+ art.py included [-v -q --type=<T> --test-type=<TT>] <script_directory> [<nightly_release> <project> <platform>]
+ art.py compare grid [-v -q --days=<D> --user=<user>] <nightly_release> <project> <platform> <nightly_tag> <package> <test_name> <file_name>...
+ art.py compare ref [-v -q] <file_name> <ref_file>
+ art.py download [-v -q] <input_file>
+ art.py list grid [-v -q --user=<user> --json --type=<T> --test-type=<TT>] <package> <nightly_release> <project> <platform> <nightly_tag>
+ art.py log grid [-v -q --user=<user>] <package> <test_name> <nightly_release> <project> <platform> <nightly_tag>
+ art.py output grid [-v -q --user=<user>] <package> <test_name> <nightly_release> <project> <platform> <nightly_tag>
Options:
--ci Run Continuous Integration tests only (using env: AtlasBuildBranch)
--days=<D> Number of days ago to pick up reference for compare [default: 1]
+ --dst=<dir> Destination directory for downloaded files
+ -h --help Show this screen.
--json Output in json format
--max-jobs=<N> Maximum number of concurrent jobs to run [default: 0]
+ -n --no-action No real submit will be done
+ -q --quiet Show less information, only warnings and errors
+ --test-type=<TT> Type of test (e.g. all, batch or single) [default: all]
--type=<T> Type of job (e.g. grid, build)
- --test-type=<TT> Type of test (e.g. all, batch or single)
- -h --help Show this screen.
- -v, --verbose Show details.
+ --user=<user> User to use for RUCIO
+ -v --verbose Show more information, debug level
--version Show version.
Sub-commands:
run Run jobs from a package in a local build (needs release and grid setup)
grid Run jobs from a package on the grid (needs release and grid setup)
submit Submit nightly jobs to the grid (NOT for users)
+ copy Copy outputs and logs from RUCIO
validate Check headers in tests
- included Shows list of files which will be included for art submit/art grid
+ included Show list of files which will be included for art submit/art grid
compare Compare the output of a job
download Download a file from rucio
- list Lists the jobs of a package
+ list List the jobs of a package
log Show the log of a job
output Get the output of a job
@@ -55,8 +61,9 @@ Arguments:
"""
__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>"
-__version__ = '0.5.4'
+__version__ = '0.6.5'
+import logging
import os
import sys
@@ -64,62 +71,77 @@ from ART.docopt_dispatch import dispatch
from ART import ArtBase, ArtGrid, ArtBuild
+from ART.art_misc import set_log
#
# First list the double commands
#
+
@dispatch.on('compare', 'ref')
def compare_ref(file_name, ref_file, **kwargs):
- """TBD."""
+ """Compare the output of a job."""
+ set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
exit(ArtBase(art_directory).compare_ref(file_name, ref_file))
@dispatch.on('compare', 'grid')
def compare_grid(package, test_name, nightly_release, project, platform, nightly_tag, **kwargs):
- """TBD."""
+ """Compare the output of a job."""
+ set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
days = int(kwargs['days'])
file_names = kwargs['file_name']
- exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).compare(package, test_name, days, file_names))
+ user = kwargs['user']
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).compare(package, test_name, days, file_names, user))
@dispatch.on('list', 'grid')
def list(package, nightly_release, project, platform, nightly_tag, **kwargs):
- """TBD."""
+ """List the jobs of a package."""
+ set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
job_type = 'grid' if kwargs['type'] is None else kwargs['type']
- index_type = 'all' if kwargs['test_type'] is None else kwargs['test_type']
+ index_type = kwargs['test_type']
json_format = kwargs['json']
- exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).list(package, job_type, index_type, json_format))
+ user = kwargs['user']
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).list(package, job_type, index_type, json_format, user))
@dispatch.on('log', 'grid')
def log(package, test_name, nightly_release, project, platform, nightly_tag, **kwargs):
- """TBD."""
+ """Show the log of a job."""
+ set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
- exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).log(package, test_name))
+ user = kwargs['user']
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).log(package, test_name, user))
@dispatch.on('output', 'grid')
-def output(package, test_name, file_name, nightly_release, project, platform, nightly_tag, **kwargs):
- """TBD."""
+def output(package, test_name, nightly_release, project, platform, nightly_tag, **kwargs):
+ """Get the output of a job."""
+ set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
- exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).output(package, test_name, file_name))
+ user = kwargs['user']
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).output(package, test_name, user))
@dispatch.on('submit')
def submit(sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
- """TBD."""
+ """Submit nightly jobs to the grid, NOT for users."""
+ set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
- job_type = 'grid' if kwargs['type'] is None else kwargs['type']
- exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).task_list(job_type, sequence_tag))
+ job_type = kwargs['type']
+ package = kwargs['package']
+ no_action = kwargs['no_action']
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).task_list(job_type, sequence_tag, package, no_action))
@dispatch.on('grid')
def grid(script_directory, sequence_tag, **kwargs):
- """TBD."""
+ """Run jobs from a package on the grid, needs release and grid setup."""
+ set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
try:
nightly_release = os.environ['AtlasBuildBranch']
@@ -127,15 +149,18 @@ def grid(script_directory, sequence_tag, **kwargs):
platform = os.environ[project + '_PLATFORM']
nightly_tag = os.environ['AtlasBuildStamp']
except KeyError, e:
- print "Environment variable not set", e
+ log.critical("Environment variable not set %s", e)
sys.exit(1)
art_type = 'grid' if kwargs['type'] is None else kwargs['type']
- exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, True).task_list(art_type, sequence_tag))
+ package = None
+ no_action = kwargs['no_action']
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, True).task_list(art_type, sequence_tag, package, no_action))
@dispatch.on('run')
def run(script_directory, sequence_tag, **kwargs):
- """TBD."""
+ """Run jobs from a package in a local build, needs release and grid setup."""
+ set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
try:
nightly_release = os.environ['AtlasBuildBranch']
@@ -143,37 +168,57 @@ def run(script_directory, sequence_tag, **kwargs):
platform = os.environ[project + '_PLATFORM']
nightly_tag = os.environ['AtlasBuildStamp']
except KeyError, e:
- print "Environment variable not set", e
+ log.critical("Environment variable not set %s", e)
sys.exit(1)
art_type = 'build' if kwargs['type'] is None else kwargs['type']
exit(ArtBuild(art_directory, nightly_release, project, platform, nightly_tag, script_directory, max_jobs=int(kwargs['max_jobs']), ci=kwargs['ci']).task_list(art_type, sequence_tag))
+@dispatch.on('copy')
+def copy(sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs):
+ """Copy outputs to eos area."""
+ set_log(kwargs)
+ art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+ package = kwargs['package']
+ # NOTE: default depends on USER, not set it here but in ArtGrid.copy
+ dst = kwargs['dst']
+ user = kwargs['user']
+ exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).copy(sequence_tag, package, dst, user))
+
+
@dispatch.on('validate')
def validate(script_directory, **kwargs):
- """TBD."""
+ """Check headers in tests."""
+ set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
exit(ArtBase(art_directory).validate(script_directory))
@dispatch.on('included')
def included(script_directory, **kwargs):
- """TBD."""
+ """Show list of files which will be included for art submit/art grid."""
+ set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
nightly_release = os.environ['AtlasBuildBranch'] if kwargs['nightly_release'] is None else kwargs['nightly_release']
project = os.environ['AtlasProject'] if kwargs['project'] is None else kwargs['project']
platform = os.environ[project + '_PLATFORM'] if kwargs['platform'] is None else kwargs['platform']
art_type = 'grid' if kwargs['type'] is None else kwargs['type']
- index_type = 'all' if kwargs['test_type'] is None else kwargs['test_type']
+ index_type = kwargs['test_type']
exit(ArtBase(art_directory).included(script_directory, art_type, index_type, nightly_release, project, platform))
@dispatch.on('download')
def download(input_file, **kwargs):
- """TBD."""
+ """Download a file from rucio."""
+ set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
exit(ArtBase(art_directory).download(input_file))
if __name__ == '__main__':
+ if sys.version_info < (2, 7, 0):
+ sys.stderr.write("You need python 2.7 or later to run this script\n")
+ exit(1)
+
+ logging.basicConfig()
dispatch(__doc__, version=os.path.splitext(os.path.basename(__file__))[0] + ' ' + __version__)