diff --git a/Tools/ART/python/ART/art_base.py b/Tools/ART/python/ART/art_base.py index 6a683081840fcbfb25eaf13f37ff3ad8f5488451..5812f8759aa2473951f8d6828231afe8d4f810fe 100755 --- a/Tools/ART/python/ART/art_base.py +++ b/Tools/ART/python/ART/art_base.py @@ -6,8 +6,10 @@ __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>" import fnmatch import inspect +import json import logging import os +import re import yaml try: @@ -88,22 +90,17 @@ class ArtBase(object): for fname in files: test_name = os.path.join(directory, fname) if self.is_included(test_name, nightly_release, project, platform): - log.info("%s %s", test_name, ArtHeader(test_name).get('art-include')) + log.info("%s %s", test_name, ArtHeader(test_name).get(ArtHeader.ART_INCLUDE)) return 0 def download(self, input_file): """TBD.""" return self.get_input(input_file) - # - # Default implementations - # - def compare_ref(self, file_name, ref_file, entries=-1): + def diff_pool(self, file_name, ref_file): """TBD.""" import PyUtils.PoolFile as PF - log = logging.getLogger(MODULE) - # diff-pool df = PF.DiffFiles(refFileName=ref_file, chkFileName=file_name, ignoreList=['RecoTimingObj_p1_RAWtoESD_timings', 'RecoTimingObj_p1_ESDtoAOD_timings']) df.printSummary() @@ -111,6 +108,12 @@ class ArtBase(object): print stat del df + return stat + + def diff_root(self, file_name, ref_file, entries=-1): + """TBD.""" + log = logging.getLogger(MODULE) + # diff-root (code, out, err) = run_command("acmd.py diff-root " + file_name + " " + ref_file + " --error-mode resilient --ignore-leaves RecoTimingObj_p1_HITStoRDO_timings RecoTimingObj_p1_RAWtoESD_mems RecoTimingObj_p1_RAWtoESD_timings RAWtoESD_mems RAWtoESD_timings ESDtoAOD_mems ESDtoAOD_timings HITStoRDO_timings RAWtoALL_mems RAWtoALL_timings RecoTimingObj_p1_RAWtoALL_mems RecoTimingObj_p1_RAWtoALL_timings RecoTimingObj_p1_EVNTtoHITS_timings --entries " + str(entries)) if code != 0: @@ -120,9 +123,44 @@ class ArtBase(object): log.info(out) return code + # + # Default implementations + # + def compare_ref(self, file_name, ref_file, entries=-1): + """TBD.""" + result = 0 + result |= self.diff_pool(file_name, ref_file) + + result |= self.diff_root(file_name, ref_file, entries) + return result + # # Protected Methods # + def get_art_results(self, output): + """ + Extract art-results. + + find all + 'art-result: x' or 'art-result: x name' or 'art-result: [x]' + and append them to result list + """ + result = [] + for line in output.splitlines(): + match = re.search(r"art-result: (\d+)\s*(.*)", line) + if match: + item = json.loads(match.group(1)) + name = match.group(2) + result.append({'name': name, 'result': item}) + else: + match = re.search(r"art-result: (\[.*\])", line) + if match: + array = json.loads(match.group(1)) + for item in array: + result.append({'name': '', 'result': item}) + + return result + def get_config(self): """Retrieve dictionary of ART configuration file, or None if file does not exist.""" try: @@ -155,7 +193,7 @@ class ArtBase(object): test_name = os.path.join(directory, fname) # is not of correct type - if job_type is not None and ArtHeader(test_name).get('art-type') != job_type: + if job_type is not None and ArtHeader(test_name).get(ArtHeader.ART_TYPE) != job_type: continue # is not included in nightly_release, project, platform @@ -163,11 +201,11 @@ class ArtBase(object): continue # batch and does specify art-input - if index_type == "batch" and ArtHeader(test_name).get('art-input') is not None: + if index_type == "batch" and ArtHeader(test_name).get(ArtHeader.ART_INPUT) is not None: continue # single and does not specify art-input - if index_type == "single" and ArtHeader(test_name).get('art-input') is None: + if index_type == "single" and ArtHeader(test_name).get(ArtHeader.ART_INPUT) is None: continue result.append(fname) @@ -176,7 +214,7 @@ class ArtBase(object): def get_type(self, directory, test_name): """Return the 'job_type' of a test.""" - return ArtHeader(os.path.join(directory, test_name)).get('art-type') + return ArtHeader(os.path.join(directory, test_name)).get(ArtHeader.ART_TYPE) def get_test_directories(self, directory): """ @@ -199,7 +237,7 @@ class ArtBase(object): def is_included(self, test_name, nightly_release, project, platform): """Return true if a match is found for test_name in nightly_release, project, platform.""" - patterns = ArtHeader(test_name).get('art-include') + patterns = ArtHeader(test_name).get(ArtHeader.ART_INCLUDE) for pattern in patterns: nightly_release_pattern = "*" diff --git a/Tools/ART/python/ART/art_build.py b/Tools/ART/python/ART/art_build.py index 0c58f19886db14f6bbbb07d5858a0dbeb8bd3977..ddd0a73f243e95c27a59e470b61f4dcef407828a 100644 --- a/Tools/ART/python/ART/art_build.py +++ b/Tools/ART/python/ART/art_build.py @@ -10,7 +10,6 @@ import json import logging import multiprocessing import os -import re from art_misc import run_command, mkdir_p from art_base import ArtBase @@ -59,7 +58,7 @@ class ArtBuild(ArtBase): status = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.defaultdict())) - for package, root in test_directories.items(): + for package, directory in test_directories.items(): test_directory = os.path.abspath(test_directories[package]) job_results = self.task(package, job_type, sequence_tag) for job_result in job_results: @@ -71,12 +70,10 @@ class ArtBuild(ArtBase): # gather results result = [] + log.debug("Looking for results for test %s", test_name) with open(os.path.join(sequence_tag, package, os.path.splitext(test_name)[0], 'stdout.txt'), 'r') as f: - for line in f: - match = re.search(r"art-result: (.*)", line) - if match: - result = json.loads(match.group(1)) - break + output = f.read() + result = self.get_art_results(output) status[package][job_result[0]]['result'] = result @@ -90,16 +87,18 @@ class ArtBuild(ArtBase): """TBD.""" log = logging.getLogger(MODULE) log.debug("task %s %s %s", package, job_type, sequence_tag) - test_names = self.get_list(self.script_directory, package, job_type, "all") + test_directories = self.get_test_directories(self.script_directory) + test_directory = os.path.abspath(test_directories[package]) + test_names = self.get_files(test_directory, job_type, "all", self.nightly_release, self.project, self.platform) scheduler = ParallelScheduler(self.max_jobs + 1) index = 0 for test_name in test_names: schedule_test = False - fname = os.path.join(self.get_test_directories(self.script_directory)[package], test_name) + fname = os.path.join(test_directory, test_name) if self.ci: branch_name = os.environ['AtlasBuildBranch'] - cis = ArtHeader(fname).get('art-ci') + cis = ArtHeader(fname).get(ArtHeader.ART_CI) for ci in cis: if fnmatch.fnmatch(branch_name, ci): schedule_test = True @@ -124,7 +123,7 @@ class ArtBuild(ArtBase): log.debug("job %s %s %s %d %s", package, job_type, sequence_tag, index, out) test_directories = self.get_test_directories(self.script_directory) test_directory = os.path.abspath(test_directories[package]) - test_name = self.get_files(test_directory, job_type)[int(index)] + test_name = self.get_files(test_directory, job_type, "all", self.nightly_release, self.project, self.platform)[int(index)] work_directory = os.path.join(sequence_tag, package, os.path.splitext(test_name)[0]) mkdir_p(work_directory) diff --git a/Tools/ART/python/ART/art_grid.py b/Tools/ART/python/ART/art_grid.py index 6c74cf0e38eaa956856f46d81b4c57fccdc19ae8..43d034d81b3e708f0cdaba3476adf23c8c26ae90 100644 --- a/Tools/ART/python/ART/art_grid.py +++ b/Tools/ART/python/ART/art_grid.py @@ -15,6 +15,8 @@ import shutil import sys import tarfile import tempfile +import time +import urllib2 try: import rucio.client @@ -34,6 +36,8 @@ class ArtGrid(ArtBase): """TBD.""" CVMFS_DIRECTORY = '/cvmfs/atlas-nightlies.cern.ch/repo/sw' + EOS_MGM_URL = 'root://eosatlas.cern.ch/' + EOS_OUTPUT_DIR = '/eos/atlas/atlascerngroupdisk/data-art/grid-output' LOG = '.log' JSON = '_EXT0' @@ -47,6 +51,7 @@ class ArtGrid(ArtBase): JOB_REPORT_ART_KEY = 'art' ATHENA_STDOUT = 'athena_stdout.txt' + RESULT_WAIT_INTERVAL = 300 def __init__(self, art_directory, nightly_release, project, platform, nightly_tag, script_directory=None, skip_setup=False, submit_directory=None): """TBD.""" @@ -161,17 +166,18 @@ class ArtGrid(ArtBase): outfile = '.'.join(('user', user, 'atlas', self.get_nightly_release_short(), self.project, self.platform, nightly_tag, sequence_tag, package)) return outfile if test_name is None else '.'.join((outfile, test_name)) - def copy(self, sequence_tag, package, dst, user): + def copy(self, package, dst=None, user=None): """Copy output from scratch area to eos area.""" log = logging.getLogger(MODULE) real_user = os.getenv('USER', ArtGrid.ARTPROD) user = real_user if user is None else user - default_dst = '/eos/atlas/atlascerngroupdisk/data-art/grid-output' if real_user == ArtGrid.ARTPROD else '.' + default_dst = ArtGrid.EOS_OUTPUT_DIR if real_user == ArtGrid.ARTPROD else '.' dst = default_dst if dst is None else dst if package is not None: log.info("Copy %s", package) - outfile = self.get_outfile(user, package, sequence_tag) + outfile = self.get_outfile(user, package) + log.info("Copying from %s", outfile) return self.copy_output(outfile, dst) @@ -188,12 +194,11 @@ class ArtGrid(ArtBase): for package, root in test_directories.items(): number_of_tests = len(self.get_files(root, "grid", "all", self.nightly_release, self.project, self.platform)) if number_of_tests > 0: - # FIXME limited - if self.nightly_release == '21.0' and package in ['TriggerTest', 'Tier0ChainTests']: - log.info("Copy %s", package) - outfile = self.get_outfile(user, package, sequence_tag) + log.info("Copy %s", package) + outfile = self.get_outfile(user, package) + log.info("Copying from %s", outfile) - result |= self.copy_output(outfile, dst) + result |= self.copy_output(outfile, dst) return result # Not used yet @@ -395,8 +400,8 @@ class ArtGrid(ArtBase): dst_target = os.path.join(dst_dir, test_name) log.info("to: %s", dst_target) if dst_target.startswith('/eos'): - mkdir_cmd = 'eos mkdir -p' - xrdcp_target = 'root://eosatlas.cern.ch/' + dst_target + mkdir_cmd = 'eos ' + ArtGrid.EOS_MGM_URL + ' mkdir -p' + xrdcp_target = ArtGrid.EOS_MGM_URL + dst_target else: mkdir_cmd = 'mkdir -p' xrdcp_target = dst_target @@ -485,7 +490,7 @@ class ArtGrid(ArtBase): result = self.task(script_directory, package, job_type, sequence_tag, no_action) return result - def task_list(self, job_type, sequence_tag, package=None, no_action=False): + def task_list(self, job_type, sequence_tag, package=None, no_action=False, wait_and_copy=True): """TBD.""" log = logging.getLogger(MODULE) # job will be submitted from tmp directory @@ -519,8 +524,48 @@ class ArtGrid(ArtBase): root = test_directories[package] all_results.update(self.task_package(root, package, job_type, sequence_tag, no_action)) + # wait for all results + if wait_and_copy: + while len(all_results) > 0: + time.sleep(ArtGrid.RESULT_WAIT_INTERVAL) + # force a cpy as we are modifying all_results + for jedi_id in list(all_results): + status = self.task_status(jedi_id) + if status is not None: + log.info("JediID %s finished with status %s", str(jedi_id), status) + if status == 'done': + package = all_results[jedi_id][0] + # FIXME limited + if self.nightly_release in ['21.0', '21.0-mc16d'] and package in ['Tier0ChainTests']: + log.info("Copy %s to eos area", package) + self.copy(package) + del all_results[jedi_id] + return 0 + def task_status(self, jedi_id): + """ + Wait for job to finish. + + Return final status of a task, or None if not finished + """ + log = logging.getLogger(MODULE) + + # fake return for simulation + if jedi_id == 0: + return "done" + + try: + r = urllib2.urlopen('https://bigpanda.cern.ch/task/' + str(jedi_id) + '?json=true') + s = json.load(r) + status = s['task']['superstatus'] + if status in ["done", "finished", "failed", "aborted", "broken"]: + log.info("Task: %s %s", str(jedi_id), str(status)) + return status + except urllib2.HTTPError, e: + log.error('%s for %s status', str(e.code), str(jedi_id)) + return None + def task(self, script_directory, package, job_type, sequence_tag, no_action=False): """ Submit a task, consisting of multiple jobs. @@ -541,7 +586,7 @@ class ArtGrid(ArtBase): env['PATH'] = '.:' + env['PATH'] env['ART_GRID_OPTIONS'] = grid_options - test_directories = self.get_test_directories(script_directory) + test_directories = self.get_test_directories(self.get_script_directory()) test_directory = test_directories[package] number_of_batch_tests = len(self.get_files(test_directory, job_type, "batch", self.nightly_release, self.project, self.platform)) @@ -578,9 +623,9 @@ class ArtGrid(ArtBase): for test_name in self.get_files(test_directory, job_type, "single", self.nightly_release, self.project, self.platform): job = os.path.join(test_directory, test_name) header = ArtHeader(job) - inds = header.get('art-input') - nFiles = header.get('art-input-nfiles') - split = header.get('art-input-split') + inds = header.get(ArtHeader.ART_INPUT) + nFiles = header.get(ArtHeader.ART_INPUT_NFILES) + split = header.get(ArtHeader.ART_INPUT_SPLIT) outfile_test = self.get_outfile(user, package, sequence_tag, str(index)) if len(outfile_test) > MAX_OUTFILE_LEN: @@ -652,16 +697,7 @@ class ArtGrid(ArtBase): result['name'] = test_name result['exit_code'] = exit_code result['test_directory'] = test_directory - - # find all 'art-result: x' or 'art-result: [x]' and append them to result list - result['result'] = [] - matches = re.findall(r"art-result: (.*)", output) - for match in matches: - item = json.loads(match) - if isinstance(item, list): - result['result'].extend(item) - else: - result['result'].append(item) + result['result'] = self.get_art_results(output) # write out results with open(os.path.join(ArtGrid.ART_JOB), 'w') as jobfile: @@ -727,7 +763,7 @@ class ArtGrid(ArtBase): result[test_name] = int(grid_index) return result - def list(self, package, job_type, index_type, json_format, user): + def list(self, package, job_type, index_type, json_format, user, nogrid): """TBD.""" log = logging.getLogger(MODULE) user = ArtGrid.ARTPROD if user is None else user @@ -735,8 +771,9 @@ class ArtGrid(ArtBase): # make sure script directory exist self.exit_if_no_script_directory() - log.info("Getting grid map...") - grid_map = self.get_grid_map(user, package) + if not nogrid: + log.info("Getting grid map...") + grid_map = self.get_grid_map(user, package) log.info("Getting test names...") test_names = self.get_list(self.get_script_directory(), package, job_type, index_type) @@ -745,7 +782,7 @@ class ArtGrid(ArtBase): name = os.path.splitext(test_name)[0] json_array.append({ 'name': name, - 'grid_index': str(grid_map[name]) if name in grid_map else '-1' + 'grid_index': str(grid_map[name]) if not nogrid and name in grid_map else '-1' }) if json_format: @@ -758,9 +795,10 @@ class ArtGrid(ArtBase): i += 1 # print warnings - for entry in json_array: - if entry['grid_index'] < 0: - log.warning('test %s could not be found in json or log', entry['name']) + if not nogrid: + for entry in json_array: + if entry['grid_index'] < 0: + log.warning('test %s could not be found in json or log', entry['name']) return 0 diff --git a/Tools/ART/python/ART/art_header.py b/Tools/ART/python/ART/art_header.py index 372cff4983283e1a84f02bd170b155ffc3cb8c47..23335e4741c1b105ed20e84c283fdcdbebfbbea4 100644 --- a/Tools/ART/python/ART/art_header.py +++ b/Tools/ART/python/ART/art_header.py @@ -17,6 +17,15 @@ MODULE = "art.header" class ArtHeader(object): """TBD.""" + ART_CI = 'art-ci' + ART_DESCRIPTION = 'art-description' + ART_INCLUDE = 'art-include' + ART_INPUT = 'art-input' + ART_INPUT_NFILES = 'art-input-nfiles' + ART_INPUT_SPLIT = 'art-input-split' + ART_OUTPUT = 'art-output' + ART_TYPE = 'art-type' + def __init__(self, filename): """TBD.""" self.header_format = re.compile(r'#\s(art-[\w-]+):\s+(.+)$') @@ -29,18 +38,18 @@ class ArtHeader(object): self.header = {} # general - self.add('art-description', StringType, '') - self.add('art-type', StringType, None, ['build', 'grid']) + self.add(ArtHeader.ART_DESCRIPTION, StringType, '') + self.add(ArtHeader.ART_TYPE, StringType, None, ['build', 'grid']) + self.add(ArtHeader.ART_INCLUDE, ListType, ['*']) # "build" type only - self.add('art-ci', ListType, []) + self.add(ArtHeader.ART_CI, ListType, []) # "grid" type only - self.add('art-include', ListType, ['*']) - self.add('art-output', ListType, []) - self.add('art-input', StringType, None) - self.add('art-input-nfiles', IntType, 1) - self.add('art-input-split', IntType, 0) + self.add(ArtHeader.ART_OUTPUT, ListType, []) + self.add(ArtHeader.ART_INPUT, StringType, None) + self.add(ArtHeader.ART_INPUT_NFILES, IntType, 1) + self.add(ArtHeader.ART_INPUT_SPLIT, IntType, 0) self.read(filename) diff --git a/Tools/ART/scripts/art-diff.py b/Tools/ART/scripts/art-diff.py new file mode 100755 index 0000000000000000000000000000000000000000..8e56071c97483aafef3ea2fd3356d90169aad3fc --- /dev/null +++ b/Tools/ART/scripts/art-diff.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python +# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +""" +ART - ATLAS Release Tester - Diff. + +Usage: + art-diff.py [--diff-type=<diff_type> --exclude=<pattern>... --platform=<platform> --platform-ref=<platform>] <nightly_release> <project> <nightly_tag> <nightly_release_ref> <platform_ref> <nightly_tag_ref> <package> + art-diff.py [--diff-type=<diff_type> --exclude=<pattern>...] <dir> <ref_dir> + +Options: + --diff-type=<diff_type> Type of diff (e.g. diff-pool or diff-root) [default: diff-pool] + --exclude=<pattern>... Exclude test files according to pattern + -h --help Show this screen + --platform=<platform> Platform [default: x86_64-slc6-gcc62-opt] + --platform-ref=<platform> Reference Platform [default: x86_64-slc6-gcc62-opt] + --version Show version + +Arguments: + dir Directory to compare + nightly_release Name of the nightly release (e.g. 21.0) + nightly_release_ref Reference Name of the nightly release (e.g. 21.0) + nightly_tag Nightly tag (e.g. 2017-02-26T2119) + nightly_tag_ref Reference Nightly tag (e.g. 2017-02-26T2119) + package Package of the test (e.g. Tier0ChainTests) + project Name of the project (e.g. Athena) + project_ref Reference Name of the project (e.g. Athena) + ref_dir Directory to compare to +""" + +__author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>" + +import fnmatch +import glob +import os +import re +import shlex +import subprocess +import sys + +from ART.docopt import docopt + +VERSION = "0.6.7" +ATHENA_STDOUT = "athena_stdout.txt" + + +class ArtDiff(object): + """Class for comparing output files.""" + + EOS_OUTPUT_DIR = '/eos/atlas/atlascerngroupdisk/data-art/grid-output' + + def __init__(self, arguments): + """Constructor of ArtDiff.""" + diff_type = arguments['--diff-type'] + excludes = arguments['--exclude'] + if arguments['<dir>'] is None: + nightly_release = arguments['<nightly_release>'] + project = arguments['<project>'] + platform = arguments['--platform'] + nightly_tag = arguments['<nightly_tag>'] + + nightly_release_ref = arguments['<nightly_release_ref>'] + project_ref = arguments['<project_ref>'] + platform_ref = arguments['--platform_ref'] + nightly_tag_ref = arguments['<nightly_tag_ref>'] + + package = arguments['<package>'] + + exit(self.diff(nightly_release, project, platform, nightly_tag, nightly_release_ref, project_ref, platform_ref, nightly_tag_ref, package, diff_type, excludes)) + + # directory compare + directory = arguments['<dir>'] + ref_dir = arguments['<ref_dir>'] + exit(self.diff_dirs(directory, ref_dir, diff_type, excludes)) + + def diff(self, nightly_release, project, platform, nightly_tag, nightly_release_ref, project_ref, platform_ref, nightly_tag_ref, package, diff_type, excludes=[]): + """Run difference between two results.""" + val_dir = os.path.join(ArtDiff.EOS_OUTPUT_DIR, nightly_release, nightly_tag, project, platform, package) + ref_dir = os.path.join(ArtDiff.EOS_OUTPUT_DIR, nightly_release_ref, nightly_tag_ref, project_ref, platform_ref, package) + return self.diff_dirs(val_dir, ref_dir, diff_type, excludes) + + def diff_dirs(self, val_dir, ref_dir, diff_type, excludes=[]): + """Run difference between two directories.""" + print "val_dir: %s" % val_dir + print "ref_dir: %s" % ref_dir + + stat_per_chain = {} + for test_name in os.listdir(val_dir): + # skip tests in pattern + exclude_test = False + for exclude in excludes: + if fnmatch.fnmatch(test_name, exclude): + print "Excluding test %s according to pattern %s" % (test_name, exclude) + exclude_test = True + break + if exclude_test: + continue + + print "******************************************" + print "Test: %s" % test_name + print "******************************************" + + val_result = self.get_result(os.path.join(val_dir, test_name)) + ref_result = self.get_result(os.path.join(val_dir, test_name)) + for key, value in val_result.iteritems(): + if key in ref_result: + print "%-10s: ref: %d events, val: %d events" % (key, int(ref_result[key][1]), int(val_result[key][1])) + + test_dir = os.path.join(val_dir, test_name) + test_patterns = ['*AOD*.pool.root', '*ESD*.pool.root', '*HITS*.pool.root', '*RDO*.pool.root', '*TAG*.root'] + test_files = [] + for test_pattern in test_patterns: + test_files.extend(glob.glob(os.path.join(test_dir, test_pattern))) + for test_file in test_files: + extension = '.root' + name = os.path.splitext(os.path.basename(test_file))[0] # remove .root + if name.endswith('.pool'): + extension = '.pool.root' + name = os.path.splitext(os.path.basename(name))[0] # remove .pool + val_file = os.path.join(val_dir, test_name, name + extension) + ref_file = os.path.join(ref_dir, test_name, name + extension) + print "val_file: %s" % val_file + print "ref_file: %s" % ref_file + + if not os.path.exists(ref_file): + print "no test found in ref_dir to compare: %s" % ref_file + continue + + # add the test to the summary if it was not already there + if test_name not in stat_per_chain: + stat_per_chain[test_name] = 0 + + if extension == '.pool.root': + if diff_type == 'diff-pool': + stat_per_chain[test_name] |= self.diff_pool(val_file, ref_file) + else: + stat_per_chain[test_name] |= self.diff_root(val_file, ref_file) + else: + stat_per_chain[test_name] |= self.diff_tag(val_file, ref_file) + + result = 0 + for filename, status in stat_per_chain.iteritems(): + if status: + print "%-70s CHANGED" % filename + result = 1 + else: + print "%-70s IDENTICAL" % filename + + return result + + def get_result(self, directory): + """ + Return map [ESD|AOD,...] -> (success, succeeded event count). + + find event counts in logfile + 'Event count check for AOD to TAG passed: all processed events found (500 output events)' + 'Event count check for BS to ESD failed: found 480 events, expected 500' + """ + result = {} + for entry in os.listdir(directory): + if re.match(r"tarball_PandaJob_(\d+)_(\w+)", entry): + logfile = os.path.join(directory, entry, ATHENA_STDOUT) + with open(logfile, "r") as f: + for line in f: + match = re.search(r"Event count check for \w+ to (\w+) (passed|failed):[^\d]+(\d+)", line) + if match: + result[match.group(1)] = (match.group(2), match.group(3)) + return result + + def diff_tag(self, file_name, ref_file): + """TBD.""" + (code, out, err) = self.run_command("diffTAGTree.py " + file_name + " " + ref_file) + if code != 0: + print "Error: %d" % code + print err + + print out + return code + + def diff_pool(self, file_name, ref_file): + """TBD.""" + import PyUtils.PoolFile as PF + + # diff-pool + df = PF.DiffFiles(refFileName=ref_file, chkFileName=file_name, ignoreList=['RecoTimingObj_p1_RAWtoESD_timings', 'RecoTimingObj_p1_ESDtoAOD_timings']) + df.printSummary() + stat = df.status() + print stat + del df + + return stat + + def diff_root(self, file_name, ref_file, entries=-1): + """TBD.""" + # diff-root + (code, out, err) = self.run_command("acmd.py diff-root " + file_name + " " + ref_file + " --error-mode resilient --ignore-leaves RecoTimingObj_p1_HITStoRDO_timings RecoTimingObj_p1_RAWtoESD_mems RecoTimingObj_p1_RAWtoESD_timings RAWtoESD_mems RAWtoESD_timings ESDtoAOD_mems ESDtoAOD_timings HITStoRDO_timings RAWtoALL_mems RAWtoALL_timings RecoTimingObj_p1_RAWtoALL_mems RecoTimingObj_p1_RAWtoALL_timings RecoTimingObj_p1_EVNTtoHITS_timings --entries " + str(entries)) + if code != 0: + print "Error: %d" % code + print err + + print out + return code + + def run_command(self, cmd, dir=None, shell=False, env=None): + """ + Run the given command locally. + + The command runs as separate subprocesses for every piped command. + Returns tuple of exit_code, output and err. + """ + print "Execute: %s" % cmd + if "|" in cmd: + cmd_parts = cmd.split('|') + else: + cmd_parts = [] + cmd_parts.append(cmd) + i = 0 + p = {} + for cmd_part in cmd_parts: + cmd_part = cmd_part.strip() + if i == 0: + p[i] = subprocess.Popen(shlex.split(cmd_part), stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dir, shell=shell, env=env) + else: + p[i] = subprocess.Popen(shlex.split(cmd_part), stdin=p[i - 1].stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=dir, shell=shell, env=env) + i = i + 1 + (output, err) = p[i - 1].communicate() + exit_code = p[0].wait() + + return exit_code, str(output), str(err) + + +if __name__ == '__main__': + if sys.version_info < (2, 7, 0): + sys.stderr.write("You need python 2.7 or later to run this script\n") + exit(1) + + arguments = docopt(__doc__, version=os.path.splitext(os.path.basename(__file__))[0] + ' ' + VERSION) + ArtDiff(arguments) diff --git a/Tools/ART/scripts/art-share.py b/Tools/ART/scripts/art-share.py index 77a97f00d1dc9b65cfb7a0974d9854f6a90b3e17..9d776fc777d0fb5eb0f2b99d0f80552fb527b070 100755 --- a/Tools/ART/scripts/art-share.py +++ b/Tools/ART/scripts/art-share.py @@ -9,7 +9,7 @@ Usage: Options: -h --help Show this screen --version Show version - --q --quiet Show less information, only warnings and errors + -q --quiet Show less information, only warnings and errors --write Write to directory -v --verbose Show more information, debug level diff --git a/Tools/ART/scripts/art.py b/Tools/ART/scripts/art.py index 54d8d37a47c6fa3a5689c820863bbb41a6038431..1bcf7ab382deab29a0ad4e86c0bd35d445d4de89 100755 --- a/Tools/ART/scripts/art.py +++ b/Tools/ART/scripts/art.py @@ -7,13 +7,13 @@ Usage: art.py run [-v -q --type=<T> --max-jobs=<N> --ci] <script_directory> <sequence_tag> art.py grid [-v -q --type=<T> -n] <script_directory> <sequence_tag> art.py submit [-v -q --type=<T> -n] <sequence_tag> <nightly_release> <project> <platform> <nightly_tag> [<package>] - art.py copy [-v -q --user=<user> --dst=<dir>] <sequence_tag> <nightly_release> <project> <platform> <nightly_tag> [<package>] + art.py copy [-v -q --user=<user> --dst=<dir>] <nightly_release> <project> <platform> <nightly_tag> <package> art.py validate [-v -q] <script_directory> art.py included [-v -q --type=<T> --test-type=<TT>] <script_directory> [<nightly_release> <project> <platform>] art.py compare grid [-v -q --days=<D> --user=<user>] <nightly_release> <project> <platform> <nightly_tag> <package> <test_name> <file_name>... art.py compare ref [-v -q] <file_name> <ref_file> art.py download [-v -q] <input_file> - art.py list grid [-v -q --user=<user> --json --type=<T> --test-type=<TT>] <package> <nightly_release> <project> <platform> <nightly_tag> + art.py list grid [-v -q --user=<user> --json --type=<T> --test-type=<TT> --nogrid] <package> <nightly_release> <project> <platform> <nightly_tag> art.py log grid [-v -q --user=<user>] <package> <test_name> <nightly_release> <project> <platform> <nightly_tag> art.py output grid [-v -q --user=<user>] <package> <test_name> <nightly_release> <project> <platform> <nightly_tag> @@ -25,6 +25,7 @@ Options: --json Output in json format --max-jobs=<N> Maximum number of concurrent jobs to run [default: 0] -n --no-action No real submit will be done + --nogrid Do not retrieve grid indices -q --quiet Show less information, only warnings and errors --test-type=<TT> Type of test (e.g. all, batch or single) [default: all] --type=<T> Type of job (e.g. grid, build) @@ -61,7 +62,7 @@ Arguments: """ __author__ = "Tulay Cuhadar Donszelmann <tcuhadar@cern.ch>" -__version__ = '0.6.5' +__version__ = '0.6.10' import logging import os @@ -73,6 +74,8 @@ from ART import ArtBase, ArtGrid, ArtBuild from ART.art_misc import set_log +MODULE = "art" + # # First list the double commands # @@ -106,7 +109,8 @@ def list(package, nightly_release, project, platform, nightly_tag, **kwargs): index_type = kwargs['test_type'] json_format = kwargs['json'] user = kwargs['user'] - exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).list(package, job_type, index_type, json_format, user)) + nogrid = kwargs['nogrid'] + exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).list(package, job_type, index_type, json_format, user, nogrid)) @dispatch.on('log', 'grid') @@ -132,16 +136,18 @@ def submit(sequence_tag, nightly_release, project, platform, nightly_tag, **kwar """Submit nightly jobs to the grid, NOT for users.""" set_log(kwargs) art_directory = os.path.dirname(os.path.realpath(sys.argv[0])) - job_type = kwargs['type'] + job_type = 'grid' if kwargs['type'] is None else kwargs['type'] package = kwargs['package'] no_action = kwargs['no_action'] - exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).task_list(job_type, sequence_tag, package, no_action)) + wait_and_copy = True + exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).task_list(job_type, sequence_tag, package, no_action, wait_and_copy)) @dispatch.on('grid') def grid(script_directory, sequence_tag, **kwargs): """Run jobs from a package on the grid, needs release and grid setup.""" set_log(kwargs) + log = logging.getLogger(MODULE) art_directory = os.path.dirname(os.path.realpath(sys.argv[0])) try: nightly_release = os.environ['AtlasBuildBranch'] @@ -151,16 +157,18 @@ def grid(script_directory, sequence_tag, **kwargs): except KeyError, e: log.critical("Environment variable not set %s", e) sys.exit(1) - art_type = 'grid' if kwargs['type'] is None else kwargs['type'] + job_type = 'grid' if kwargs['type'] is None else kwargs['type'] package = None no_action = kwargs['no_action'] - exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, True).task_list(art_type, sequence_tag, package, no_action)) + wait_and_copy = False + exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory, True).task_list(job_type, sequence_tag, package, no_action, wait_and_copy)) @dispatch.on('run') def run(script_directory, sequence_tag, **kwargs): """Run jobs from a package in a local build, needs release and grid setup.""" set_log(kwargs) + log = logging.getLogger(MODULE) art_directory = os.path.dirname(os.path.realpath(sys.argv[0])) try: nightly_release = os.environ['AtlasBuildBranch'] @@ -170,12 +178,12 @@ def run(script_directory, sequence_tag, **kwargs): except KeyError, e: log.critical("Environment variable not set %s", e) sys.exit(1) - art_type = 'build' if kwargs['type'] is None else kwargs['type'] - exit(ArtBuild(art_directory, nightly_release, project, platform, nightly_tag, script_directory, max_jobs=int(kwargs['max_jobs']), ci=kwargs['ci']).task_list(art_type, sequence_tag)) + job_type = 'build' if kwargs['type'] is None else kwargs['type'] + exit(ArtBuild(art_directory, nightly_release, project, platform, nightly_tag, script_directory, max_jobs=int(kwargs['max_jobs']), ci=kwargs['ci']).task_list(job_type, sequence_tag)) @dispatch.on('copy') -def copy(sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs): +def copy(nightly_release, project, platform, nightly_tag, **kwargs): """Copy outputs to eos area.""" set_log(kwargs) art_directory = os.path.dirname(os.path.realpath(sys.argv[0])) @@ -183,7 +191,7 @@ def copy(sequence_tag, nightly_release, project, platform, nightly_tag, **kwargs # NOTE: default depends on USER, not set it here but in ArtGrid.copy dst = kwargs['dst'] user = kwargs['user'] - exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).copy(sequence_tag, package, dst, user)) + exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).copy(package, dst, user)) @dispatch.on('validate')