From 1b97dca71249b4a8aa959dc51c20c6e9e1c54f4f Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Wed, 22 Jan 2020 09:46:50 +0100 Subject: [PATCH 01/34] saving and getting checkout artifacts from repo --- python/LbNightlyTools/Configuration.py | 61 ++++++++++++++++++- python/LbNightlyTools/Scripts/Build.py | 7 ++- python/LbNightlyTools/Scripts/Checkout.py | 11 +++- python/LbNightlyTools/Scripts/Common.py | 26 +++++--- python/LbNightlyTools/Scripts/EnabledSlots.py | 18 ++++++ 5 files changed, 109 insertions(+), 14 deletions(-) diff --git a/python/LbNightlyTools/Configuration.py b/python/LbNightlyTools/Configuration.py index 1cd94610..22a94a40 100644 --- a/python/LbNightlyTools/Configuration.py +++ b/python/LbNightlyTools/Configuration.py @@ -22,6 +22,7 @@ import copy import urllib2 import urllib import json +from subprocess import call from datetime import datetime from collections import OrderedDict from StringIO import StringIO @@ -482,6 +483,8 @@ class Project(object): self.ignore_slot_build_tool = kwargs['ignore_slot_build_tool'] self.build_results = None + self.hash = self.calc_hash() + self.checkedout = False def toDict(self): ''' @@ -943,6 +946,35 @@ class Project(object): self._fixCMT(patchfile, dryrun=dryrun) self._fixProjectConfigJSON(patchfile, dryrun=dryrun) + + def calc_hash(self): + ''' + Returns the hash of the project which is the concatenation + of the commit id, and optionally merges. + ''' + # TODO: what about tagged versions and data packages? + hash = '' + try: + hash += self.checkout_opts['commit'][:7] + for merge in self.checkout_opts['merges']: + hash += merge[1][:7] + except KeyError: + __log__.debug('no commit id and merges for: ' + str(self.toDict())) + pass + return hash + + + def checkout_artifact_exists(self, artifacts_repo): + ''' + Checks if the project with this hash has already been checked out + ''' + file = os.path.join(artifacts_repo, + self.name + self.hash + '.zip') + if os.path.exists(file): + return True + else: + return False + class Package(object): ''' @@ -965,6 +997,7 @@ class Package(object): self.container = None self.checkout = kwargs.get('checkout') self.checkout_opts = kwargs.get('checkout_opts', {}) + self.hash = self.calc_hash() @property def slot(self): @@ -1072,6 +1105,11 @@ class Package(object): return "{0}/{1}".format(self.name, self.version) + def calc_hash(self): + # TODO: to be improved ? + return self.version + + class _ContainedList(object): ''' Helper class to handle a list of projects bound to a slot. @@ -1720,14 +1758,33 @@ class Slot(object): pass context = kwargs.pop('context', NullContext) - + artifacts_repo = kwargs.pop('artifacts_repo') results = OrderedDict() for project in self.activeProjects: if projects is None or project.name in projects: results[project.name] = {} try: with context(project) as ctx: - results[project.name] = project.checkout(**kwargs) + checkout_metadata_file = os.path.join( + artifacts_repo, + project.name + + project.hash + + '.txt') + if project.checkout_artifact_exists(artifacts_repo): + __log__.debug('project {} with hash {} ' + 'has already been checked out' + .format(project.name, project.hash)) + call(['unzip','-q', checkout_metadata_file.replace('.txt', '.zip')]) + with open(checkout_metadata_file, 'r') as infile: + checkout_metadata = json.load(infile) + results[project.name] = checkout_metadata + project.checkedout = True + continue + else: + results[project.name] = project.checkout(**kwargs) + __log__.debug('saving checkout metadata to: ' + checkout_metadata_file) + with open(checkout_metadata_file, 'w') as outfile: + json.dump(results, outfile) ctx.result = results[project.name] except (RuntimeError, AssertionError) as x: msg = 'failed to checkout {}: {}: {}'.format( diff --git a/python/LbNightlyTools/Scripts/Build.py b/python/LbNightlyTools/Scripts/Build.py index faa844a3..847b774a 100644 --- a/python/LbNightlyTools/Scripts/Build.py +++ b/python/LbNightlyTools/Scripts/Build.py @@ -100,6 +100,11 @@ def unpackArtifacts(src, dest): # --clean option) call(['unzip', '-n', '-q', f], cwd=dest) + # TODO: copy and unpack only projects in slot, remove copying from eos in build.sh + # for project in slot: + # get_checkout_artifact + # unpack_artifact + try: from LbEnv import which @@ -487,7 +492,7 @@ string(REPLACE "$${NIGHTLY_BUILD_ROOT}" "$${CMAKE_CURRENT_LIST_DIR}" }) if os.path.exists(os.path.join(self.artifacts_dir, 'slot.patch')): - self.log.warning('Applaying patch file') + self.log.warning('Applying patch file') log_call([ 'patch', '-p1', '-i', os.path.join(self.artifacts_dir, 'slot.patch') diff --git a/python/LbNightlyTools/Scripts/Checkout.py b/python/LbNightlyTools/Scripts/Checkout.py index 99b0a2de..78fcd0a4 100644 --- a/python/LbNightlyTools/Scripts/Checkout.py +++ b/python/LbNightlyTools/Scripts/Checkout.py @@ -135,7 +135,8 @@ class Script(BaseScript): element, 'src', build_id=self.options.build_id, - artifacts_dir=self.artifacts_dir) + artifacts_dir=self.options.artifacts_repo, + hash=element.hash) def main(self): """ Main logic of the script """ @@ -200,7 +201,8 @@ class Script(BaseScript): slot.checkout( projects=opts.projects, ignore_errors=opts.ignore_checkout_errors, - context=CheckoutContext) + context=CheckoutContext, + artifacts_repo=opts.artifacts_repo) with open(join(self.artifacts_dir, 'slot.patch'), 'w') as patchfile: @@ -293,10 +295,13 @@ class Script(BaseScript): # ignore missing directories # (the project may not have been checked out) if not os.path.exists(join(self.build_dir, element.baseDir)): + # TODO fix this (correct path ?) self.log.warning('no sources for %s, skip packing', element) continue if isinstance(element, DataProject): continue # ignore DataProjects, because we pack packages + if project.checkedout: + continue self.log.info('packing %s %s...', element.name, element.version) @@ -312,7 +317,7 @@ class Script(BaseScript): contname.append(self.options.build_id) contname.append('src.zip') pack([container], - join(self.artifacts_dir, 'packs', 'src', '.'.join(contname)), + join(self.options.artifacts_repo, 'packs', 'src', '.'.join(contname)), cwd=self.build_dir, checksum='md5', dereference=False, diff --git a/python/LbNightlyTools/Scripts/Common.py b/python/LbNightlyTools/Scripts/Common.py index 1f5ce6ff..f9c3d781 100755 --- a/python/LbNightlyTools/Scripts/Common.py +++ b/python/LbNightlyTools/Scripts/Common.py @@ -81,6 +81,12 @@ def addBasicOptions(parser): metavar='DIR', help='directory where to store the artifacts') + parser.add_option( + '--artifacts-repo', + action='store', + metavar='DIR', + help='repository with the checkout artifacts') + parser.add_option( '--projects', action='store', @@ -786,7 +792,7 @@ class BaseScript(PlainScript): cwd=self._buildDir(proj)).communicate() -def genPackageName(proj, platform, build_id=None, artifacts_dir=None): +def genPackageName(proj, platform, build_id=None, artifacts_dir=None, hash=None): ''' Generate the source/binary tarball name for a project/package. @@ -801,13 +807,17 @@ def genPackageName(proj, platform, build_id=None, artifacts_dir=None): ... build_id='dummy', artifacts_dir='artifacts') 'artifacts/packs/x86_64-slc6-gcc48-dbg/Gaudi.v25r0.dummy.x86_64-slc6-gcc48-dbg.zip' ''' - packname = [proj.name.replace('/', '_'), proj.version] - if build_id: - packname.append(build_id) - packname.append(platform) - packname.append('zip') - packname = '.'.join(packname) - packname = os.path.join('packs', platform, packname) + if hash: + packname = proj.name.replace('/', '_') + hash + '.zip' + else: + packname = [proj.name.replace('/', '_'), proj.version] + if build_id: + packname.append(build_id) + packname.append(platform) + packname.append('zip') + packname = '.'.join(packname) + packname = os.path.join('packs', platform, packname) if artifacts_dir: packname = os.path.join(artifacts_dir, packname) + print('packname is: ' + packname) return packname diff --git a/python/LbNightlyTools/Scripts/EnabledSlots.py b/python/LbNightlyTools/Scripts/EnabledSlots.py index eba09f96..cedda2bb 100644 --- a/python/LbNightlyTools/Scripts/EnabledSlots.py +++ b/python/LbNightlyTools/Scripts/EnabledSlots.py @@ -97,6 +97,24 @@ class Script(PlainScript): 'config': slot.toDict(), 'date': self.options.date, } + for project in slot.toDict()['projects']: + filename = 'project-params-' + project['name'] + try: + filename += project['checkout_opts']['commit'][:7] + for merge in project['checkout_opts']['merges']: + filename += merge[1][:7] + except KeyError: + pass + filename += '.txt' + if os.path.isfile(filename): + self.log.info('project: {} already marked to be checked out' + .format(project['name'])) + else: + with open(filename, 'w') as f: + self.log.info('new project to checkout: {}' + .format(project['name'])) + f.write(str(project)) + if not self.options.submit: self.log.debug(' slot info: {}\n{}'.format( key, json.dumps(value, indent=2))) -- GitLab From 1dba6b1108ebf216dc21872857011394e3b6a775 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Wed, 19 Feb 2020 17:45:35 +0100 Subject: [PATCH 02/34] add class representing ArtifactsRepository --- python/LbNightlyTools/Utils.py | 180 +++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index 15e46713..f6002f66 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -23,6 +23,7 @@ import subprocess import time import urllib2 import re +import requests from datetime import datetime import gitlab @@ -374,6 +375,185 @@ def recursive_update(dest, data): return dest +def get_repo_type(uri): + ''' + Returns type of repository based on uri provided in the argument. + It may return 'eos', 'file', 'http' or None + ''' + try: + from urlparse import urlparse + except ImportError(): + from urllib.parse import urlparse + + result = urlparse(uri) + if result.scheme == '': + if result.path.startswith('/eos/'): + return 'eos' + else: + return 'file' + elif result.scheme == 'root': + return 'eos' + elif result.scheme == 'http': + return 'http' + return None + + +class ArtifactsRepository(object): + ''' + Class representing artifacts repository. + ''' + + def __init__(self, uri): + ''' + Initialises the repository based on its URI + ''' + self.uri = uri + + def pull_artifacts(self, artifacts_name, dest): + ''' + Gets artifacts from the repository and unzips them + + Arguments: + artifacts_name: name of the artifacts file + + Keyword arguments: + dest: to specify destination directory for unzipping + + Returns: True if the artifacts have been extracted, False otherwise + ''' + raise NotImplementedError('Should be implemented in the inheriting class') + + def push_artifacts(self, artifacts_name): + ''' + Pushes artifacts to the repository + + Arguments: + artifacts_name: name of the artifcats file + + Returns: True if the artifacts have been pushed, False otherwise + ''' + raise NotImplementedError('Should be implemented in the inheriting class') + + def artifacts_exist(self, artifacts_name): + ''' + Checks if artifacts exist + + Arguments: + artifacts_name: name of the artifcats file + + Returns: True if the artifacts exist, False otherwise + ''' + raise NotImplementedError('Should be implemented in the inheriting class') + + +def repo_factory(uri): + ''' + Factory function creating artifacts repository + object based on the URI provided in the argument. + ''' + + class FileRepository(ArtifactsRepository): + ''' + Class defining repository in the local file system. + ''' + + def pull_artifacts(self, artifacts_name, dest): + try: + subprocess.check_call( + ['unzip', '-q', os.path.join(self.uri, artifacts_name)], cwd=dest) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Unzipping failed: {}'.format(str(ex))) + return False + return True + + def push_artifacts(self, artifacts_name): + try: + logging.debug('pushing {} to {}'.format(artifacts_name, self.uri)) + subprocess.call(['cp', artifacts_name, self.uri]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pushing artifacts to the repository failed: {}' + .format(str(ex))) + return False + return True + + def artifacts_exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) + + class EosRepository(ArtifactsRepository): + ''' + Class defining repository on EOS. + ''' + + def pull_artifacts(self, artifacts_name, dest): + try: + subprocess.check_call( + ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name], + cwd=dest) + subprocess.check_call(['unzip', '-q', artifacts_name], + cwd=dest) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pulling failed: {}'.format(str(ex))) + return False + return True + + def push_artifacts(self, artifacts_name): + try: + subprocess.check_call([ + 'xrdcp', artifacts_name, os.path.join( + self.uri, + os.path.basename(artifacts_name)) + ]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pushing artifacts to the repository failed: {}' + .format(str(ex))) + return False + return True + + def artifacts_exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) + + class HttpRepository(ArtifactsRepository): + ''' + Class defining repository through HTTP request methods. + ''' + + def pull_artifacts(self, artifacts_name, dest): + r = requests.get(os.path.join(self.uri, artifacts_name)) + if r.status_code != 200: + logging.debug('Resource does not exist') + return False + with open(os.path.join(dest, artifacts_name), 'wb') as f: + f.write(r.content) + subprocess.check_call(['unzip','-q', os.path.join(dest, artifacts_name)], + cwd=dest) + return True + + def push_artifacts(self, artifacts_name, auth=('', '')): + r = requests.put(os.path.join(self.uri, artifacts_name), + data=open(artifacts_name, 'rb').read(), + headers={'content-type': 'application/zip'}, + auth=auth) + if r.status_code != 201: + logging.debug('Pushing artifacts to the repository failed') + return False + return True + + def artifacts_exist(self, artifacts_name): + r = requests.get(os.path.join(self.uri, artifacts_name)) + if r.status_code == 200: + return True + return False + + if get_repo_type(uri) == 'file': + return FileRepository(uri) + elif get_repo_type(uri) == 'eos': + return EosRepository(uri) + elif get_repo_type(uri) == 'http': + return HttpRepository(uri) + else: + raise ValueError("Unrecognised repository type for: {}".format(uri)) + + class Dashboard(object): ''' Wrapper for the CouchDB-based dashboard. -- GitLab From d7d5b5565aba234f84f6e569ead40f8cd9b8ebac Mon Sep 17 00:00:00 2001 From: Maciej Pawel Szymanski <maciej.szymanski@cern.ch> Date: Thu, 20 Feb 2020 08:35:32 +0000 Subject: [PATCH 03/34] Apply suggestion to python/LbNightlyTools/Utils.py --- python/LbNightlyTools/Utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index f6002f66..505cf97d 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -393,7 +393,7 @@ def get_repo_type(uri): return 'file' elif result.scheme == 'root': return 'eos' - elif result.scheme == 'http': + elif result.scheme in ('http', 'https'): return 'http' return None -- GitLab From 18bc92149cc0f13924cd056593fba1be811397bc Mon Sep 17 00:00:00 2001 From: Maciej Pawel Szymanski <maciej.szymanski@cern.ch> Date: Thu, 20 Feb 2020 08:37:14 +0000 Subject: [PATCH 04/34] Apply suggestion to python/LbNightlyTools/Utils.py --- python/LbNightlyTools/Utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index 505cf97d..61f70264 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -386,7 +386,7 @@ def get_repo_type(uri): from urllib.parse import urlparse result = urlparse(uri) - if result.scheme == '': + if not result.scheme or result.scheme == 'file': if result.path.startswith('/eos/'): return 'eos' else: -- GitLab From 8c25eeb07b4431f29d3c22ed4b69da517d6a5dd7 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 10:45:38 +0100 Subject: [PATCH 05/34] use urllib instead of requests --- python/LbNightlyTools/Utils.py | 47 +++++++++++++++++----------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index f6002f66..4625155d 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -23,7 +23,7 @@ import subprocess import time import urllib2 import re -import requests +from urllib.request import urlopen, Request from datetime import datetime import gitlab @@ -517,31 +517,30 @@ def repo_factory(uri): Class defining repository through HTTP request methods. ''' - def pull_artifacts(self, artifacts_name, dest): - r = requests.get(os.path.join(self.uri, artifacts_name)) - if r.status_code != 200: - logging.debug('Resource does not exist') - return False - with open(os.path.join(dest, artifacts_name), 'wb') as f: - f.write(r.content) - subprocess.check_call(['unzip','-q', os.path.join(dest, artifacts_name)], - cwd=dest) - return True + def pull(self, artifacts_name): + with urlopen( + os.path.join(self.uri, artifacts_name)) as response, open(artifacts_name, 'wb') as out_file: + shutil.copyfileobj(response, out_file) + return out_file + + def push(self, artifacts_name, auth=('', '')): + req = Request( + url=os.path.join(self.uri, artifacts_name), + data=open(artifacts_name, 'rb').read(), + method='PUT') + with urlopen(req) as f: + pass + if f.status != 201: + logging.debug('Pushing artifacts to the repository failed') + return False + return True - def push_artifacts(self, artifacts_name, auth=('', '')): - r = requests.put(os.path.join(self.uri, artifacts_name), - data=open(artifacts_name, 'rb').read(), - headers={'content-type': 'application/zip'}, - auth=auth) - if r.status_code != 201: - logging.debug('Pushing artifacts to the repository failed') - return False + def exist(self, artifacts_name): + from urllib.error import HTTPError + try: + urlopen(os.path.join(self.uri, artifacts_name)).close() return True - - def artifacts_exist(self, artifacts_name): - r = requests.get(os.path.join(self.uri, artifacts_name)) - if r.status_code == 200: - return True + except HTTPError: return False if get_repo_type(uri) == 'file': -- GitLab From aa1689221dc9d087db3462cf01e0b840cb302ec4 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 10:46:15 +0100 Subject: [PATCH 06/34] refactor factory method --- python/LbNightlyTools/Utils.py | 69 +++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index 4625155d..d93d463b 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -375,29 +375,6 @@ def recursive_update(dest, data): return dest -def get_repo_type(uri): - ''' - Returns type of repository based on uri provided in the argument. - It may return 'eos', 'file', 'http' or None - ''' - try: - from urlparse import urlparse - except ImportError(): - from urllib.parse import urlparse - - result = urlparse(uri) - if result.scheme == '': - if result.path.startswith('/eos/'): - return 'eos' - else: - return 'file' - elif result.scheme == 'root': - return 'eos' - elif result.scheme == 'http': - return 'http' - return None - - class ArtifactsRepository(object): ''' Class representing artifacts repository. @@ -543,14 +520,44 @@ def repo_factory(uri): except HTTPError: return False - if get_repo_type(uri) == 'file': - return FileRepository(uri) - elif get_repo_type(uri) == 'eos': - return EosRepository(uri) - elif get_repo_type(uri) == 'http': - return HttpRepository(uri) - else: - raise ValueError("Unrecognised repository type for: {}".format(uri)) + +def repo_type(uri): + ''' + Returns type of repository based on uri provided in the argument. + It may return 'eos', 'file', 'http' or None + ''' + try: + from urlparse import urlparse + except ImportError(): + from urllib.parse import urlparse + + result = urlparse(uri) + if result.scheme == '': + if result.path.startswith('/eos/'): + return 'eos' + else: + return 'file' + elif result.scheme == 'root': + return 'eos' + elif result.scheme == 'http': + return 'http' + return None + + +def repo_factory(url): + ''' + Factory function creating artifacts repository + object based on the URI provided in the argument. + ''' + _repos = { + 'file': FileRepository, + 'eos': EosRepository, + 'http': HttpRepository, + } + repo = _repos.get(repo_type(url)) + if not repo: + raise ValueError('Unrecognised repository type: {}'.format(url)) + return repo(url) class Dashboard(object): -- GitLab From beafb62df3083378a5f7bc6d09c5851370758776 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 10:47:12 +0100 Subject: [PATCH 07/34] remove from methods name redundant artifacts word --- python/LbNightlyTools/Utils.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index d93d463b..b56da743 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -386,21 +386,18 @@ class ArtifactsRepository(object): ''' self.uri = uri - def pull_artifacts(self, artifacts_name, dest): + def pull(self, artifacts_name): ''' - Gets artifacts from the repository and unzips them + Returns a file object in the read mode Arguments: artifacts_name: name of the artifacts file - Keyword arguments: - dest: to specify destination directory for unzipping - - Returns: True if the artifacts have been extracted, False otherwise + Returns: File object ''' raise NotImplementedError('Should be implemented in the inheriting class') - def push_artifacts(self, artifacts_name): + def push(self, artifacts_name): ''' Pushes artifacts to the repository @@ -411,7 +408,7 @@ class ArtifactsRepository(object): ''' raise NotImplementedError('Should be implemented in the inheriting class') - def artifacts_exist(self, artifacts_name): + def exist(self, artifacts_name): ''' Checks if artifacts exist -- GitLab From 50a923edd26ae13a043e8df2ae57a29788f62f56 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 10:47:53 +0100 Subject: [PATCH 08/34] dont unzip in pull --- python/LbNightlyTools/Utils.py | 107 +++++++++++++++------------------ 1 file changed, 48 insertions(+), 59 deletions(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index b56da743..609f14af 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -420,76 +420,65 @@ class ArtifactsRepository(object): raise NotImplementedError('Should be implemented in the inheriting class') -def repo_factory(uri): +class FileRepository(ArtifactsRepository): ''' - Factory function creating artifacts repository - object based on the URI provided in the argument. + Class defining repository in the local file system. ''' - class FileRepository(ArtifactsRepository): - ''' - Class defining repository in the local file system. - ''' + def pull(self, artifacts_name): + artifact = open(os.path.join(self.uri, artifacts_name), 'r') + return artifact - def pull_artifacts(self, artifacts_name, dest): - try: - subprocess.check_call( - ['unzip', '-q', os.path.join(self.uri, artifacts_name)], cwd=dest) - except (subprocess.CalledProcessError, OSError) as ex: - logging.debug('Unzipping failed: {}'.format(str(ex))) - return False - return True + def push(self, artifacts_name): + try: + subprocess.call(['cp', artifacts_name, self.uri]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pushing artifacts to the repository failed: {}' + .format(str(ex))) + return False + return True - def push_artifacts(self, artifacts_name): - try: - logging.debug('pushing {} to {}'.format(artifacts_name, self.uri)) - subprocess.call(['cp', artifacts_name, self.uri]) - except (subprocess.CalledProcessError, OSError) as ex: - logging.debug('Pushing artifacts to the repository failed: {}' - .format(str(ex))) - return False - return True + def exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) - def artifacts_exist(self, artifacts_name): - return os.path.exists(os.path.join(self.uri, artifacts_name)) - class EosRepository(ArtifactsRepository): - ''' - Class defining repository on EOS. - ''' +class EosRepository(ArtifactsRepository): + ''' + Class defining repository on EOS. + ''' - def pull_artifacts(self, artifacts_name, dest): - try: - subprocess.check_call( - ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name], - cwd=dest) - subprocess.check_call(['unzip', '-q', artifacts_name], - cwd=dest) - except (subprocess.CalledProcessError, OSError) as ex: - logging.debug('Pulling failed: {}'.format(str(ex))) - return False - return True + def pull(self, artifacts_name): + try: + subprocess.check_call( + ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name], + ) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pulling failed: {}'.format(str(ex))) + return None + artifact = open(artifacts_name, 'r') + return artifact - def push_artifacts(self, artifacts_name): - try: - subprocess.check_call([ - 'xrdcp', artifacts_name, os.path.join( - self.uri, - os.path.basename(artifacts_name)) - ]) - except (subprocess.CalledProcessError, OSError) as ex: - logging.debug('Pushing artifacts to the repository failed: {}' - .format(str(ex))) - return False - return True + def push(self, artifacts_name): + try: + subprocess.check_call([ + 'xrdcp', artifacts_name, os.path.join( + self.uri, + os.path.basename(artifacts_name)) + ]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pushing artifacts to the repository failed: {}' + .format(str(ex))) + return False + return True - def artifacts_exist(self, artifacts_name): - return os.path.exists(os.path.join(self.uri, artifacts_name)) + def exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) - class HttpRepository(ArtifactsRepository): - ''' - Class defining repository through HTTP request methods. - ''' + +class HttpRepository(ArtifactsRepository): + ''' + Class defining repository through HTTP request methods. + ''' def pull(self, artifacts_name): with urlopen( -- GitLab From 8ec53449b34c55ea5ff91cf2485afd6922c2ef07 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 10:50:47 +0100 Subject: [PATCH 09/34] fix function name --- python/LbNightlyTools/Utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index 7c8afaf3..a4264e92 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -375,7 +375,7 @@ def recursive_update(dest, data): return dest -def get_repo_type(uri): +def repo_type(uri): ''' Returns type of repository based on uri provided in the argument. It may return 'eos', 'file', 'http' or None -- GitLab From ffe0383232f36cc1f77002b80b1d76d0ac9d8225 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 11:13:28 +0100 Subject: [PATCH 10/34] remove redundant definition --- python/LbNightlyTools/Utils.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index a4264e92..609f14af 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -375,29 +375,6 @@ def recursive_update(dest, data): return dest -def repo_type(uri): - ''' - Returns type of repository based on uri provided in the argument. - It may return 'eos', 'file', 'http' or None - ''' - try: - from urlparse import urlparse - except ImportError(): - from urllib.parse import urlparse - - result = urlparse(uri) - if not result.scheme or result.scheme == 'file': - if result.path.startswith('/eos/'): - return 'eos' - else: - return 'file' - elif result.scheme == 'root': - return 'eos' - elif result.scheme in ('http', 'https'): - return 'http' - return None - - class ArtifactsRepository(object): ''' Class representing artifacts repository. -- GitLab From e2434e053a46d3b06135a36de905e5da2857bdaf Mon Sep 17 00:00:00 2001 From: Gitlab CI <noreply@cern.ch> Date: Fri, 21 Feb 2020 10:18:45 +0000 Subject: [PATCH 11/34] Fixed formatting patch generated by https://gitlab.cern.ch/lhcb-core/LbNightlyTools/-/jobs/7285194 --- python/LbNightlyTools/Utils.py | 41 +++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index 609f14af..20949e12 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -395,7 +395,8 @@ class ArtifactsRepository(object): Returns: File object ''' - raise NotImplementedError('Should be implemented in the inheriting class') + raise NotImplementedError( + 'Should be implemented in the inheriting class') def push(self, artifacts_name): ''' @@ -406,7 +407,8 @@ class ArtifactsRepository(object): Returns: True if the artifacts have been pushed, False otherwise ''' - raise NotImplementedError('Should be implemented in the inheriting class') + raise NotImplementedError( + 'Should be implemented in the inheriting class') def exist(self, artifacts_name): ''' @@ -417,7 +419,8 @@ class ArtifactsRepository(object): Returns: True if the artifacts exist, False otherwise ''' - raise NotImplementedError('Should be implemented in the inheriting class') + raise NotImplementedError( + 'Should be implemented in the inheriting class') class FileRepository(ArtifactsRepository): @@ -433,8 +436,9 @@ class FileRepository(ArtifactsRepository): try: subprocess.call(['cp', artifacts_name, self.uri]) except (subprocess.CalledProcessError, OSError) as ex: - logging.debug('Pushing artifacts to the repository failed: {}' - .format(str(ex))) + logging.debug( + 'Pushing artifacts to the repository failed: {}'.format( + str(ex))) return False return True @@ -449,9 +453,10 @@ class EosRepository(ArtifactsRepository): def pull(self, artifacts_name): try: - subprocess.check_call( - ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name], - ) + subprocess.check_call([ + 'xrdcp', + os.path.join(self.uri, artifacts_name), artifacts_name + ], ) except (subprocess.CalledProcessError, OSError) as ex: logging.debug('Pulling failed: {}'.format(str(ex))) return None @@ -461,13 +466,13 @@ class EosRepository(ArtifactsRepository): def push(self, artifacts_name): try: subprocess.check_call([ - 'xrdcp', artifacts_name, os.path.join( - self.uri, - os.path.basename(artifacts_name)) + 'xrdcp', artifacts_name, + os.path.join(self.uri, os.path.basename(artifacts_name)) ]) except (subprocess.CalledProcessError, OSError) as ex: - logging.debug('Pushing artifacts to the repository failed: {}' - .format(str(ex))) + logging.debug( + 'Pushing artifacts to the repository failed: {}'.format( + str(ex))) return False return True @@ -481,16 +486,16 @@ class HttpRepository(ArtifactsRepository): ''' def pull(self, artifacts_name): - with urlopen( - os.path.join(self.uri, artifacts_name)) as response, open(artifacts_name, 'wb') as out_file: + with urlopen(os.path.join(self.uri, artifacts_name)) as response, open( + artifacts_name, 'wb') as out_file: shutil.copyfileobj(response, out_file) return out_file def push(self, artifacts_name, auth=('', '')): req = Request( - url=os.path.join(self.uri, artifacts_name), - data=open(artifacts_name, 'rb').read(), - method='PUT') + url=os.path.join(self.uri, artifacts_name), + data=open(artifacts_name, 'rb').read(), + method='PUT') with urlopen(req) as f: pass if f.status != 201: -- GitLab From 4eeefca55beca91af4feb83951621335d2097fb2 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 12:36:50 +0100 Subject: [PATCH 12/34] re-apply suggestions --- python/LbNightlyTools/Utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index 20949e12..d7ca6660 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -523,14 +523,14 @@ def repo_type(uri): from urllib.parse import urlparse result = urlparse(uri) - if result.scheme == '': + if not result.scheme or result.scheme == 'file': if result.path.startswith('/eos/'): return 'eos' else: return 'file' elif result.scheme == 'root': return 'eos' - elif result.scheme == 'http': + elif result.scheme in ('http', 'https'): return 'http' return None -- GitLab From 72202d4334c975a2277d9d51d0035855e4f95bd6 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 14:24:54 +0100 Subject: [PATCH 13/34] add module handling interactions with artifacts repository --- python/LbNightlyTools/Repository.py | 182 ++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 python/LbNightlyTools/Repository.py diff --git a/python/LbNightlyTools/Repository.py b/python/LbNightlyTools/Repository.py new file mode 100644 index 00000000..77a0e8fc --- /dev/null +++ b/python/LbNightlyTools/Repository.py @@ -0,0 +1,182 @@ +############################################################################### +# (c) Copyright 2013 CERN # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +''' +Code handling interactions with artifacts repository +''' + +_repo_handlers = {} + +def register_for(*schemes): + def _reg(cls): + global _repo_handlers + _repo_handlers.update((s, cls) for s in schemes) + return cls + return _reg + +def get_repo_type(uri): + ''' + Returns type of repository based on uri provided in the argument. + It may return 'eos', 'file', 'http' or None + ''' + from urllib.parse import urlparse + + result = urlparse(uri) + if not result.scheme or result.scheme == 'file': + if result.path.startswith('/eos/'): + return 'eos' + else: + return 'file' + elif result.scheme == 'root': + return 'eos' + elif result.scheme in ('http', 'https'): + return 'http' + return None + +class ArtifactsRepository(object): + ''' + Class representing artifacts repository. + ''' + + def __init__(self, uri): + ''' + Initialises the repository based on its URI + ''' + self.uri = uri + + def pull(self, artifacts_name): + ''' + Returns a file object in the read mode + + Arguments: + artifacts_name: name of the artifacts file + + Returns: File object + ''' + raise NotImplementedError( + 'Should be implemented in the inheriting class') + + def push(self, artifacts_name): + ''' + Pushes artifacts to the repository + + Arguments: + artifacts_name: name of the artifcats file + + Returns: True if the artifacts have been pushed, False otherwise + ''' + raise NotImplementedError( + 'Should be implemented in the inheriting class') + + def exist(self, artifacts_name): + ''' + Checks if artifacts exist + + Arguments: + artifacts_name: name of the artifcats file + + Returns: True if the artifacts exist, False otherwise + ''' + raise NotImplementedError( + 'Should be implemented in the inheriting class') + +@register_for('file') +class FileRepository(ArtifactsRepository): + ''' + Class defining repository in the local file system. + ''' + + def pull(self, artifacts_name): + artifact = open(os.path.join(self.uri, artifacts_name), 'r') + return artifact + + def push(self, artifacts_name): + try: + subprocess.call(['cp', artifacts_name, self.uri]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug( + 'Pushing artifacts to the repository failed: {}'.format( + str(ex))) + return False + return True + + def exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) + +@register_for('eos') +class EosRepository(ArtifactsRepository): + ''' + Class defining repository on EOS. + ''' + + def pull(self, artifacts_name): + try: + subprocess.check_call([ + 'xrdcp', + os.path.join(self.uri, artifacts_name), artifacts_name + ], ) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pulling failed: {}'.format(str(ex))) + return None + artifact = open(artifacts_name, 'r') + return artifact + + def push(self, artifacts_name): + try: + subprocess.check_call([ + 'xrdcp', artifacts_name, + os.path.join(self.uri, os.path.basename(artifacts_name)) + ]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug( + 'Pushing artifacts to the repository failed: {}'.format( + str(ex))) + return False + return True + + def exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) + +@register_for('http') +class HttpRepository(ArtifactsRepository): + ''' + Class defining repository through HTTP request methods. + ''' + + def pull(self, artifacts_name): + with urlopen(os.path.join(self.uri, artifacts_name)) as response, open( + artifacts_name, 'wb') as out_file: + shutil.copyfileobj(response, out_file) + return out_file + + def push(self, artifacts_name, auth=('', '')): + req = Request( + url=os.path.join(self.uri, artifacts_name), + data=open(artifacts_name, 'rb').read(), + method='PUT') + with urlopen(req) as f: + pass + if f.status != 201: + logging.debug('Pushing artifacts to the repository failed') + return False + return True + + def exist(self, artifacts_name): + from urllib.error import HTTPError + try: + urlopen(os.path.join(self.uri, artifacts_name)).close() + return True + except HTTPError: + return False + +def connect(uri, *args, **kwargs): + global _repo_handlers + return _repo_handlers[get_repo_type(uri)](uri, *args, **kwargs) + -- GitLab From d40fd1dd21acd38de0b915801292d4988a66f38d Mon Sep 17 00:00:00 2001 From: Maciej Pawel Szymanski <maciej.szymanski@cern.ch> Date: Fri, 21 Feb 2020 12:27:41 +0000 Subject: [PATCH 14/34] Revert "Merge branch 'maszyman-artifacts-repository' of..." This reverts commit 268274adb987f856f2e924750fe07fa554ec1a9e --- python/LbNightlyTools/BuildMethods.py | 3 +- python/LbNightlyTools/MergeRequestBuilds.py | 9 +- python/LbNightlyTools/Scripts/Common.py | 4 +- python/LbNightlyTools/Scripts/Test.py | 17 +- python/LbNightlyTools/Utils.py | 177 -------------------- setup.py | 2 +- 6 files changed, 15 insertions(+), 197 deletions(-) diff --git a/python/LbNightlyTools/BuildMethods.py b/python/LbNightlyTools/BuildMethods.py index 6986c1b8..b4756cab 100644 --- a/python/LbNightlyTools/BuildMethods.py +++ b/python/LbNightlyTools/BuildMethods.py @@ -210,7 +210,8 @@ class cmt(make): or os.environ.get('BINARY_TAG')) if 'GAUDI_QMTEST_HTML_OUTPUT' not in env: bin_dir = os.path.join( - os.path.abspath(proj.baseDir), 'build', 'html') + os.path.abspath(proj.baseDir), + 'build.{CMTCONFIG}'.format(**env), 'html') env['GAUDI_QMTEST_HTML_OUTPUT'] = bin_dir kwargs['env'] = env return self._make('test', proj, **kwargs) diff --git a/python/LbNightlyTools/MergeRequestBuilds.py b/python/LbNightlyTools/MergeRequestBuilds.py index a2279270..d5a0ce78 100644 --- a/python/LbNightlyTools/MergeRequestBuilds.py +++ b/python/LbNightlyTools/MergeRequestBuilds.py @@ -65,13 +65,10 @@ def mr_getrefdate(mrs): for mr, project in mrs: # Parse the date of the time when the MR was committed ref_sha1 = mr.attributes['diff_refs']['start_sha'] - committed_date = project.commits.get( - ref_sha1).attributes['committed_date'] - committed_date = committed_date.rstrip('Z') - if '+' in committed_date: - committed_date = committed_date.split('+', 1)[0] mr_ref_dates.append( - datetime.datetime.strptime(committed_date, "%Y-%m-%dT%H:%M:%S.%f")) + datetime.datetime.strptime( + project.commits.get(ref_sha1).attributes['committed_date'], + "%Y-%m-%dT%H:%M:%S.%fZ")) # Return the latest time corresponsing to the last MR branched from it's project return sorted(mr_ref_dates)[-1].isoformat() diff --git a/python/LbNightlyTools/Scripts/Common.py b/python/LbNightlyTools/Scripts/Common.py index b1b72d35..f9c3d781 100755 --- a/python/LbNightlyTools/Scripts/Common.py +++ b/python/LbNightlyTools/Scripts/Common.py @@ -101,7 +101,7 @@ def addBasicOptions(parser): parser.set_defaults( build_id='{slot}', - slot_build_id=None, + slot_build_id=0, artifacts_dir='artifacts', summary_prefix='') return parser @@ -720,7 +720,7 @@ class BaseScript(PlainScript): if self.summary_base: ensureDirs([self._summaryDir()]) - if opts.slot_build_id is not None: + if opts.slot_build_id: self.slot.build_id = self.options.slot_build_id elif not self.slot.build_id: self.slot.build_id = int(os.environ.get('slot_build_id', 0)) diff --git a/python/LbNightlyTools/Scripts/Test.py b/python/LbNightlyTools/Scripts/Test.py index 7e38db3a..8498597b 100644 --- a/python/LbNightlyTools/Scripts/Test.py +++ b/python/LbNightlyTools/Scripts/Test.py @@ -199,17 +199,14 @@ class Script(BaseScript): for proj, _ in self.slot.testGen( projects=opts.projects, before=before, jobs=opts.jobs): - html_src = self._buildDir(proj, 'build', 'html') # Always use the most recent CTEST_CONVERTER, if available - # and we are in a CMake project - if os.path.exists(self._buildDir(proj, 'build', 'Testing')): - if CTEST_CONVERTER: - if os.path.exists(html_src): - shutil.rmtree(html_src) - call([CTEST_CONVERTER], - cwd=self._buildDir(proj, 'build')) - summary_json = os.path.join(html_src, 'summary.json') - fixFailureCauses(summary_json) + html_src = self._buildDir(proj, 'build', 'html') + if CTEST_CONVERTER: + if os.path.exists(html_src): + shutil.rmtree(html_src) + call([CTEST_CONVERTER], cwd=self._buildDir(proj, 'build')) + summary_json = os.path.join(html_src, 'summary.json') + fixFailureCauses(summary_json) # update annotations with cpuinfo summary try: diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index d7ca6660..15e46713 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -23,7 +23,6 @@ import subprocess import time import urllib2 import re -from urllib.request import urlopen, Request from datetime import datetime import gitlab @@ -375,182 +374,6 @@ def recursive_update(dest, data): return dest -class ArtifactsRepository(object): - ''' - Class representing artifacts repository. - ''' - - def __init__(self, uri): - ''' - Initialises the repository based on its URI - ''' - self.uri = uri - - def pull(self, artifacts_name): - ''' - Returns a file object in the read mode - - Arguments: - artifacts_name: name of the artifacts file - - Returns: File object - ''' - raise NotImplementedError( - 'Should be implemented in the inheriting class') - - def push(self, artifacts_name): - ''' - Pushes artifacts to the repository - - Arguments: - artifacts_name: name of the artifcats file - - Returns: True if the artifacts have been pushed, False otherwise - ''' - raise NotImplementedError( - 'Should be implemented in the inheriting class') - - def exist(self, artifacts_name): - ''' - Checks if artifacts exist - - Arguments: - artifacts_name: name of the artifcats file - - Returns: True if the artifacts exist, False otherwise - ''' - raise NotImplementedError( - 'Should be implemented in the inheriting class') - - -class FileRepository(ArtifactsRepository): - ''' - Class defining repository in the local file system. - ''' - - def pull(self, artifacts_name): - artifact = open(os.path.join(self.uri, artifacts_name), 'r') - return artifact - - def push(self, artifacts_name): - try: - subprocess.call(['cp', artifacts_name, self.uri]) - except (subprocess.CalledProcessError, OSError) as ex: - logging.debug( - 'Pushing artifacts to the repository failed: {}'.format( - str(ex))) - return False - return True - - def exist(self, artifacts_name): - return os.path.exists(os.path.join(self.uri, artifacts_name)) - - -class EosRepository(ArtifactsRepository): - ''' - Class defining repository on EOS. - ''' - - def pull(self, artifacts_name): - try: - subprocess.check_call([ - 'xrdcp', - os.path.join(self.uri, artifacts_name), artifacts_name - ], ) - except (subprocess.CalledProcessError, OSError) as ex: - logging.debug('Pulling failed: {}'.format(str(ex))) - return None - artifact = open(artifacts_name, 'r') - return artifact - - def push(self, artifacts_name): - try: - subprocess.check_call([ - 'xrdcp', artifacts_name, - os.path.join(self.uri, os.path.basename(artifacts_name)) - ]) - except (subprocess.CalledProcessError, OSError) as ex: - logging.debug( - 'Pushing artifacts to the repository failed: {}'.format( - str(ex))) - return False - return True - - def exist(self, artifacts_name): - return os.path.exists(os.path.join(self.uri, artifacts_name)) - - -class HttpRepository(ArtifactsRepository): - ''' - Class defining repository through HTTP request methods. - ''' - - def pull(self, artifacts_name): - with urlopen(os.path.join(self.uri, artifacts_name)) as response, open( - artifacts_name, 'wb') as out_file: - shutil.copyfileobj(response, out_file) - return out_file - - def push(self, artifacts_name, auth=('', '')): - req = Request( - url=os.path.join(self.uri, artifacts_name), - data=open(artifacts_name, 'rb').read(), - method='PUT') - with urlopen(req) as f: - pass - if f.status != 201: - logging.debug('Pushing artifacts to the repository failed') - return False - return True - - def exist(self, artifacts_name): - from urllib.error import HTTPError - try: - urlopen(os.path.join(self.uri, artifacts_name)).close() - return True - except HTTPError: - return False - - -def repo_type(uri): - ''' - Returns type of repository based on uri provided in the argument. - It may return 'eos', 'file', 'http' or None - ''' - try: - from urlparse import urlparse - except ImportError(): - from urllib.parse import urlparse - - result = urlparse(uri) - if not result.scheme or result.scheme == 'file': - if result.path.startswith('/eos/'): - return 'eos' - else: - return 'file' - elif result.scheme == 'root': - return 'eos' - elif result.scheme in ('http', 'https'): - return 'http' - return None - - -def repo_factory(url): - ''' - Factory function creating artifacts repository - object based on the URI provided in the argument. - ''' - _repos = { - 'file': FileRepository, - 'eos': EosRepository, - 'http': HttpRepository, - } - repo = _repos.get(repo_type(url)) - if not repo: - raise ValueError('Unrecognised repository type: {}'.format(url)) - return repo(url) - - class Dashboard(object): ''' Wrapper for the CouchDB-based dashboard. diff --git a/setup.py b/setup.py index 655e5e12..219cc126 100644 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ setup( "LbCommon>=0.0.7", "LbSoftConfDb2Clients", "LbDevTools", - "python-gitlab" + ('<2' if version_info < (3, 0) else ''), + "python-gitlab", "pika>=1.0.0", "CouchDB", "tabulate", -- GitLab From f22a9024bed64ec362118846ade7bd12d147e1b1 Mon Sep 17 00:00:00 2001 From: Marco Clemencic <marco.clemencic@cern.ch> Date: Tue, 28 Jan 2020 17:53:18 +0100 Subject: [PATCH 15/34] Allow override of --slot-build-id with 0 --- python/LbNightlyTools/Scripts/Common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/LbNightlyTools/Scripts/Common.py b/python/LbNightlyTools/Scripts/Common.py index f9c3d781..b1b72d35 100755 --- a/python/LbNightlyTools/Scripts/Common.py +++ b/python/LbNightlyTools/Scripts/Common.py @@ -101,7 +101,7 @@ def addBasicOptions(parser): parser.set_defaults( build_id='{slot}', - slot_build_id=0, + slot_build_id=None, artifacts_dir='artifacts', summary_prefix='') return parser @@ -720,7 +720,7 @@ class BaseScript(PlainScript): if self.summary_base: ensureDirs([self._summaryDir()]) - if opts.slot_build_id: + if opts.slot_build_id is not None: self.slot.build_id = self.options.slot_build_id elif not self.slot.build_id: self.slot.build_id = int(os.environ.get('slot_build_id', 0)) -- GitLab From 1fac548d80dfc1fe2828da17079055fa91fdd258 Mon Sep 17 00:00:00 2001 From: Marco Clemencic <marco.clemencic@cern.ch> Date: Tue, 28 Jan 2020 18:01:47 +0100 Subject: [PATCH 16/34] Constrain python-gitlab version on Python2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 219cc126..655e5e12 100644 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ setup( "LbCommon>=0.0.7", "LbSoftConfDb2Clients", "LbDevTools", - "python-gitlab", + "python-gitlab" + ('<2' if version_info < (3, 0) else ''), "pika>=1.0.0", "CouchDB", "tabulate", -- GitLab From 582cdb5437a9ed3fbf2656b7112832e04c55c273 Mon Sep 17 00:00:00 2001 From: Marco Clemencic <marco.clemencic@cern.ch> Date: Wed, 29 Jan 2020 22:27:14 +0100 Subject: [PATCH 17/34] Strip timezone part from committed_date before parsing --- python/LbNightlyTools/MergeRequestBuilds.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/LbNightlyTools/MergeRequestBuilds.py b/python/LbNightlyTools/MergeRequestBuilds.py index d5a0ce78..a2279270 100644 --- a/python/LbNightlyTools/MergeRequestBuilds.py +++ b/python/LbNightlyTools/MergeRequestBuilds.py @@ -65,10 +65,13 @@ def mr_getrefdate(mrs): for mr, project in mrs: # Parse the date of the time when the MR was committed ref_sha1 = mr.attributes['diff_refs']['start_sha'] + committed_date = project.commits.get( + ref_sha1).attributes['committed_date'] + committed_date = committed_date.rstrip('Z') + if '+' in committed_date: + committed_date = committed_date.split('+', 1)[0] mr_ref_dates.append( - datetime.datetime.strptime( - project.commits.get(ref_sha1).attributes['committed_date'], - "%Y-%m-%dT%H:%M:%S.%fZ")) + datetime.datetime.strptime(committed_date, "%Y-%m-%dT%H:%M:%S.%f")) # Return the latest time corresponsing to the last MR branched from it's project return sorted(mr_ref_dates)[-1].isoformat() -- GitLab From 327817b9d7c2847539024d8688cd85b36da6ed19 Mon Sep 17 00:00:00 2001 From: Marco Clemencic <marco.clemencic@cern.ch> Date: Fri, 31 Jan 2020 10:10:09 +0100 Subject: [PATCH 18/34] Do not try to run CTEST_CONVERTER for non-CMake builds --- python/LbNightlyTools/Scripts/Test.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/python/LbNightlyTools/Scripts/Test.py b/python/LbNightlyTools/Scripts/Test.py index 8498597b..7e38db3a 100644 --- a/python/LbNightlyTools/Scripts/Test.py +++ b/python/LbNightlyTools/Scripts/Test.py @@ -199,14 +199,17 @@ class Script(BaseScript): for proj, _ in self.slot.testGen( projects=opts.projects, before=before, jobs=opts.jobs): - # Always use the most recent CTEST_CONVERTER, if available html_src = self._buildDir(proj, 'build', 'html') - if CTEST_CONVERTER: - if os.path.exists(html_src): - shutil.rmtree(html_src) - call([CTEST_CONVERTER], cwd=self._buildDir(proj, 'build')) - summary_json = os.path.join(html_src, 'summary.json') - fixFailureCauses(summary_json) + # Always use the most recent CTEST_CONVERTER, if available + # and we are in a CMake project + if os.path.exists(self._buildDir(proj, 'build', 'Testing')): + if CTEST_CONVERTER: + if os.path.exists(html_src): + shutil.rmtree(html_src) + call([CTEST_CONVERTER], + cwd=self._buildDir(proj, 'build')) + summary_json = os.path.join(html_src, 'summary.json') + fixFailureCauses(summary_json) # update annotations with cpuinfo summary try: -- GitLab From cc6728e959f0d69fe67c0cecc6925a2649b64847 Mon Sep 17 00:00:00 2001 From: Marco Clemencic <marco.clemencic@cern.ch> Date: Fri, 31 Jan 2020 10:25:09 +0100 Subject: [PATCH 19/34] Make CMT test output location consistent with CMake --- python/LbNightlyTools/BuildMethods.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/LbNightlyTools/BuildMethods.py b/python/LbNightlyTools/BuildMethods.py index b4756cab..6986c1b8 100644 --- a/python/LbNightlyTools/BuildMethods.py +++ b/python/LbNightlyTools/BuildMethods.py @@ -210,8 +210,7 @@ class cmt(make): or os.environ.get('BINARY_TAG')) if 'GAUDI_QMTEST_HTML_OUTPUT' not in env: bin_dir = os.path.join( - os.path.abspath(proj.baseDir), - 'build.{CMTCONFIG}'.format(**env), 'html') + os.path.abspath(proj.baseDir), 'build', 'html') env['GAUDI_QMTEST_HTML_OUTPUT'] = bin_dir kwargs['env'] = env return self._make('test', proj, **kwargs) -- GitLab From 72863e77b1c2184f9ce9827f300a701c5d2641a3 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Wed, 19 Feb 2020 17:45:35 +0100 Subject: [PATCH 20/34] add class representing ArtifactsRepository --- python/LbNightlyTools/Utils.py | 180 +++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index 15e46713..f6002f66 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -23,6 +23,7 @@ import subprocess import time import urllib2 import re +import requests from datetime import datetime import gitlab @@ -374,6 +375,185 @@ def recursive_update(dest, data): return dest +def get_repo_type(uri): + ''' + Returns type of repository based on uri provided in the argument. + It may return 'eos', 'file', 'http' or None + ''' + try: + from urlparse import urlparse + except ImportError(): + from urllib.parse import urlparse + + result = urlparse(uri) + if result.scheme == '': + if result.path.startswith('/eos/'): + return 'eos' + else: + return 'file' + elif result.scheme == 'root': + return 'eos' + elif result.scheme == 'http': + return 'http' + return None + + +class ArtifactsRepository(object): + ''' + Class representing artifacts repository. + ''' + + def __init__(self, uri): + ''' + Initialises the repository based on its URI + ''' + self.uri = uri + + def pull_artifacts(self, artifacts_name, dest): + ''' + Gets artifacts from the repository and unzips them + + Arguments: + artifacts_name: name of the artifacts file + + Keyword arguments: + dest: to specify destination directory for unzipping + + Returns: True if the artifacts have been extracted, False otherwise + ''' + raise NotImplementedError('Should be implemented in the inheriting class') + + def push_artifacts(self, artifacts_name): + ''' + Pushes artifacts to the repository + + Arguments: + artifacts_name: name of the artifcats file + + Returns: True if the artifacts have been pushed, False otherwise + ''' + raise NotImplementedError('Should be implemented in the inheriting class') + + def artifacts_exist(self, artifacts_name): + ''' + Checks if artifacts exist + + Arguments: + artifacts_name: name of the artifcats file + + Returns: True if the artifacts exist, False otherwise + ''' + raise NotImplementedError('Should be implemented in the inheriting class') + + +def repo_factory(uri): + ''' + Factory function creating artifacts repository + object based on the URI provided in the argument. + ''' + + class FileRepository(ArtifactsRepository): + ''' + Class defining repository in the local file system. + ''' + + def pull_artifacts(self, artifacts_name, dest): + try: + subprocess.check_call( + ['unzip', '-q', os.path.join(self.uri, artifacts_name)], cwd=dest) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Unzipping failed: {}'.format(str(ex))) + return False + return True + + def push_artifacts(self, artifacts_name): + try: + logging.debug('pushing {} to {}'.format(artifacts_name, self.uri)) + subprocess.call(['cp', artifacts_name, self.uri]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pushing artifacts to the repository failed: {}' + .format(str(ex))) + return False + return True + + def artifacts_exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) + + class EosRepository(ArtifactsRepository): + ''' + Class defining repository on EOS. + ''' + + def pull_artifacts(self, artifacts_name, dest): + try: + subprocess.check_call( + ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name], + cwd=dest) + subprocess.check_call(['unzip', '-q', artifacts_name], + cwd=dest) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pulling failed: {}'.format(str(ex))) + return False + return True + + def push_artifacts(self, artifacts_name): + try: + subprocess.check_call([ + 'xrdcp', artifacts_name, os.path.join( + self.uri, + os.path.basename(artifacts_name)) + ]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pushing artifacts to the repository failed: {}' + .format(str(ex))) + return False + return True + + def artifacts_exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) + + class HttpRepository(ArtifactsRepository): + ''' + Class defining repository through HTTP request methods. + ''' + + def pull_artifacts(self, artifacts_name, dest): + r = requests.get(os.path.join(self.uri, artifacts_name)) + if r.status_code != 200: + logging.debug('Resource does not exist') + return False + with open(os.path.join(dest, artifacts_name), 'wb') as f: + f.write(r.content) + subprocess.check_call(['unzip','-q', os.path.join(dest, artifacts_name)], + cwd=dest) + return True + + def push_artifacts(self, artifacts_name, auth=('', '')): + r = requests.put(os.path.join(self.uri, artifacts_name), + data=open(artifacts_name, 'rb').read(), + headers={'content-type': 'application/zip'}, + auth=auth) + if r.status_code != 201: + logging.debug('Pushing artifacts to the repository failed') + return False + return True + + def artifacts_exist(self, artifacts_name): + r = requests.get(os.path.join(self.uri, artifacts_name)) + if r.status_code == 200: + return True + return False + + if get_repo_type(uri) == 'file': + return FileRepository(uri) + elif get_repo_type(uri) == 'eos': + return EosRepository(uri) + elif get_repo_type(uri) == 'http': + return HttpRepository(uri) + else: + raise ValueError("Unrecognised repository type for: {}".format(uri)) + + class Dashboard(object): ''' Wrapper for the CouchDB-based dashboard. -- GitLab From 24a777e6d90f47361184c56cdacb60192c072914 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 10:45:38 +0100 Subject: [PATCH 21/34] use urllib instead of requests --- python/LbNightlyTools/Utils.py | 47 +++++++++++++++++----------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index f6002f66..4625155d 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -23,7 +23,7 @@ import subprocess import time import urllib2 import re -import requests +from urllib.request import urlopen, Request from datetime import datetime import gitlab @@ -517,31 +517,30 @@ def repo_factory(uri): Class defining repository through HTTP request methods. ''' - def pull_artifacts(self, artifacts_name, dest): - r = requests.get(os.path.join(self.uri, artifacts_name)) - if r.status_code != 200: - logging.debug('Resource does not exist') - return False - with open(os.path.join(dest, artifacts_name), 'wb') as f: - f.write(r.content) - subprocess.check_call(['unzip','-q', os.path.join(dest, artifacts_name)], - cwd=dest) - return True + def pull(self, artifacts_name): + with urlopen( + os.path.join(self.uri, artifacts_name)) as response, open(artifacts_name, 'wb') as out_file: + shutil.copyfileobj(response, out_file) + return out_file + + def push(self, artifacts_name, auth=('', '')): + req = Request( + url=os.path.join(self.uri, artifacts_name), + data=open(artifacts_name, 'rb').read(), + method='PUT') + with urlopen(req) as f: + pass + if f.status != 201: + logging.debug('Pushing artifacts to the repository failed') + return False + return True - def push_artifacts(self, artifacts_name, auth=('', '')): - r = requests.put(os.path.join(self.uri, artifacts_name), - data=open(artifacts_name, 'rb').read(), - headers={'content-type': 'application/zip'}, - auth=auth) - if r.status_code != 201: - logging.debug('Pushing artifacts to the repository failed') - return False + def exist(self, artifacts_name): + from urllib.error import HTTPError + try: + urlopen(os.path.join(self.uri, artifacts_name)).close() return True - - def artifacts_exist(self, artifacts_name): - r = requests.get(os.path.join(self.uri, artifacts_name)) - if r.status_code == 200: - return True + except HTTPError: return False if get_repo_type(uri) == 'file': -- GitLab From 1236057df4efafceb58a1c23bd878ac83551a695 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 10:46:15 +0100 Subject: [PATCH 22/34] refactor factory method --- python/LbNightlyTools/Utils.py | 69 +++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index 4625155d..d93d463b 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -375,29 +375,6 @@ def recursive_update(dest, data): return dest -def get_repo_type(uri): - ''' - Returns type of repository based on uri provided in the argument. - It may return 'eos', 'file', 'http' or None - ''' - try: - from urlparse import urlparse - except ImportError(): - from urllib.parse import urlparse - - result = urlparse(uri) - if result.scheme == '': - if result.path.startswith('/eos/'): - return 'eos' - else: - return 'file' - elif result.scheme == 'root': - return 'eos' - elif result.scheme == 'http': - return 'http' - return None - - class ArtifactsRepository(object): ''' Class representing artifacts repository. @@ -543,14 +520,44 @@ def repo_factory(uri): except HTTPError: return False - if get_repo_type(uri) == 'file': - return FileRepository(uri) - elif get_repo_type(uri) == 'eos': - return EosRepository(uri) - elif get_repo_type(uri) == 'http': - return HttpRepository(uri) - else: - raise ValueError("Unrecognised repository type for: {}".format(uri)) + +def repo_type(uri): + ''' + Returns type of repository based on uri provided in the argument. + It may return 'eos', 'file', 'http' or None + ''' + try: + from urlparse import urlparse + except ImportError(): + from urllib.parse import urlparse + + result = urlparse(uri) + if result.scheme == '': + if result.path.startswith('/eos/'): + return 'eos' + else: + return 'file' + elif result.scheme == 'root': + return 'eos' + elif result.scheme == 'http': + return 'http' + return None + + +def repo_factory(url): + ''' + Factory function creating artifacts repository + object based on the URI provided in the argument. + ''' + _repos = { + 'file': FileRepository, + 'eos': EosRepository, + 'http': HttpRepository, + } + repo = _repos.get(repo_type(url)) + if not repo: + raise ValueError('Unrecognised repository type: {}'.format(url)) + return repo(url) class Dashboard(object): -- GitLab From dc03c12c447edbe37b2fe473d1ac6005b6563cac Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 10:47:12 +0100 Subject: [PATCH 23/34] remove from methods name redundant artifacts word --- python/LbNightlyTools/Utils.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index d93d463b..b56da743 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -386,21 +386,18 @@ class ArtifactsRepository(object): ''' self.uri = uri - def pull_artifacts(self, artifacts_name, dest): + def pull(self, artifacts_name): ''' - Gets artifacts from the repository and unzips them + Returns a file object in the read mode Arguments: artifacts_name: name of the artifacts file - Keyword arguments: - dest: to specify destination directory for unzipping - - Returns: True if the artifacts have been extracted, False otherwise + Returns: File object ''' raise NotImplementedError('Should be implemented in the inheriting class') - def push_artifacts(self, artifacts_name): + def push(self, artifacts_name): ''' Pushes artifacts to the repository @@ -411,7 +408,7 @@ class ArtifactsRepository(object): ''' raise NotImplementedError('Should be implemented in the inheriting class') - def artifacts_exist(self, artifacts_name): + def exist(self, artifacts_name): ''' Checks if artifacts exist -- GitLab From 17c8c83e1b546b1a28f851bb107b4689c3c586fe Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 10:47:53 +0100 Subject: [PATCH 24/34] dont unzip in pull --- python/LbNightlyTools/Utils.py | 107 +++++++++++++++------------------ 1 file changed, 48 insertions(+), 59 deletions(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index b56da743..609f14af 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -420,76 +420,65 @@ class ArtifactsRepository(object): raise NotImplementedError('Should be implemented in the inheriting class') -def repo_factory(uri): +class FileRepository(ArtifactsRepository): ''' - Factory function creating artifacts repository - object based on the URI provided in the argument. + Class defining repository in the local file system. ''' - class FileRepository(ArtifactsRepository): - ''' - Class defining repository in the local file system. - ''' + def pull(self, artifacts_name): + artifact = open(os.path.join(self.uri, artifacts_name), 'r') + return artifact - def pull_artifacts(self, artifacts_name, dest): - try: - subprocess.check_call( - ['unzip', '-q', os.path.join(self.uri, artifacts_name)], cwd=dest) - except (subprocess.CalledProcessError, OSError) as ex: - logging.debug('Unzipping failed: {}'.format(str(ex))) - return False - return True + def push(self, artifacts_name): + try: + subprocess.call(['cp', artifacts_name, self.uri]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pushing artifacts to the repository failed: {}' + .format(str(ex))) + return False + return True - def push_artifacts(self, artifacts_name): - try: - logging.debug('pushing {} to {}'.format(artifacts_name, self.uri)) - subprocess.call(['cp', artifacts_name, self.uri]) - except (subprocess.CalledProcessError, OSError) as ex: - logging.debug('Pushing artifacts to the repository failed: {}' - .format(str(ex))) - return False - return True + def exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) - def artifacts_exist(self, artifacts_name): - return os.path.exists(os.path.join(self.uri, artifacts_name)) - class EosRepository(ArtifactsRepository): - ''' - Class defining repository on EOS. - ''' +class EosRepository(ArtifactsRepository): + ''' + Class defining repository on EOS. + ''' - def pull_artifacts(self, artifacts_name, dest): - try: - subprocess.check_call( - ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name], - cwd=dest) - subprocess.check_call(['unzip', '-q', artifacts_name], - cwd=dest) - except (subprocess.CalledProcessError, OSError) as ex: - logging.debug('Pulling failed: {}'.format(str(ex))) - return False - return True + def pull(self, artifacts_name): + try: + subprocess.check_call( + ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name], + ) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pulling failed: {}'.format(str(ex))) + return None + artifact = open(artifacts_name, 'r') + return artifact - def push_artifacts(self, artifacts_name): - try: - subprocess.check_call([ - 'xrdcp', artifacts_name, os.path.join( - self.uri, - os.path.basename(artifacts_name)) - ]) - except (subprocess.CalledProcessError, OSError) as ex: - logging.debug('Pushing artifacts to the repository failed: {}' - .format(str(ex))) - return False - return True + def push(self, artifacts_name): + try: + subprocess.check_call([ + 'xrdcp', artifacts_name, os.path.join( + self.uri, + os.path.basename(artifacts_name)) + ]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pushing artifacts to the repository failed: {}' + .format(str(ex))) + return False + return True - def artifacts_exist(self, artifacts_name): - return os.path.exists(os.path.join(self.uri, artifacts_name)) + def exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) - class HttpRepository(ArtifactsRepository): - ''' - Class defining repository through HTTP request methods. - ''' + +class HttpRepository(ArtifactsRepository): + ''' + Class defining repository through HTTP request methods. + ''' def pull(self, artifacts_name): with urlopen( -- GitLab From 2c8ab7d2938aec37f3e9de91ab3e3457869725df Mon Sep 17 00:00:00 2001 From: Maciej Pawel Szymanski <maciej.szymanski@cern.ch> Date: Thu, 20 Feb 2020 08:35:32 +0000 Subject: [PATCH 25/34] Apply suggestion to python/LbNightlyTools/Utils.py --- python/LbNightlyTools/Utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index 609f14af..f002125b 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -525,7 +525,7 @@ def repo_type(uri): return 'file' elif result.scheme == 'root': return 'eos' - elif result.scheme == 'http': + elif result.scheme in ('http', 'https'): return 'http' return None -- GitLab From c87e19e35788259a8112d847ae79b18b575c576a Mon Sep 17 00:00:00 2001 From: Maciej Pawel Szymanski <maciej.szymanski@cern.ch> Date: Thu, 20 Feb 2020 08:37:14 +0000 Subject: [PATCH 26/34] Apply suggestion to python/LbNightlyTools/Utils.py --- python/LbNightlyTools/Utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index f002125b..38102ac6 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -518,7 +518,7 @@ def repo_type(uri): from urllib.parse import urlparse result = urlparse(uri) - if result.scheme == '': + if not result.scheme or result.scheme == 'file': if result.path.startswith('/eos/'): return 'eos' else: -- GitLab From 0a7f98ab93c18e9bd705100fa229c5752d57f2e3 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 10:50:47 +0100 Subject: [PATCH 27/34] fix function name --- python/LbNightlyTools/Utils.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index 38102ac6..daf8a426 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -375,6 +375,29 @@ def recursive_update(dest, data): return dest +def repo_type(uri): + ''' + Returns type of repository based on uri provided in the argument. + It may return 'eos', 'file', 'http' or None + ''' + try: + from urlparse import urlparse + except ImportError(): + from urllib.parse import urlparse + + result = urlparse(uri) + if not result.scheme or result.scheme == 'file': + if result.path.startswith('/eos/'): + return 'eos' + else: + return 'file' + elif result.scheme == 'root': + return 'eos' + elif result.scheme in ('http', 'https'): + return 'http' + return None + + class ArtifactsRepository(object): ''' Class representing artifacts repository. -- GitLab From 668cfc8946f4ca9cef71dd079a415a0309ad67ac Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 11:13:28 +0100 Subject: [PATCH 28/34] remove redundant definition --- python/LbNightlyTools/Utils.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index daf8a426..38102ac6 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -375,29 +375,6 @@ def recursive_update(dest, data): return dest -def repo_type(uri): - ''' - Returns type of repository based on uri provided in the argument. - It may return 'eos', 'file', 'http' or None - ''' - try: - from urlparse import urlparse - except ImportError(): - from urllib.parse import urlparse - - result = urlparse(uri) - if not result.scheme or result.scheme == 'file': - if result.path.startswith('/eos/'): - return 'eos' - else: - return 'file' - elif result.scheme == 'root': - return 'eos' - elif result.scheme in ('http', 'https'): - return 'http' - return None - - class ArtifactsRepository(object): ''' Class representing artifacts repository. -- GitLab From 3e3b756881a579246cde072bdd5fd186f92010f3 Mon Sep 17 00:00:00 2001 From: Gitlab CI <noreply@cern.ch> Date: Fri, 21 Feb 2020 10:18:45 +0000 Subject: [PATCH 29/34] Fixed formatting patch generated by https://gitlab.cern.ch/lhcb-core/LbNightlyTools/-/jobs/7285194 --- python/LbNightlyTools/Utils.py | 41 +++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index 38102ac6..d7ca6660 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -395,7 +395,8 @@ class ArtifactsRepository(object): Returns: File object ''' - raise NotImplementedError('Should be implemented in the inheriting class') + raise NotImplementedError( + 'Should be implemented in the inheriting class') def push(self, artifacts_name): ''' @@ -406,7 +407,8 @@ class ArtifactsRepository(object): Returns: True if the artifacts have been pushed, False otherwise ''' - raise NotImplementedError('Should be implemented in the inheriting class') + raise NotImplementedError( + 'Should be implemented in the inheriting class') def exist(self, artifacts_name): ''' @@ -417,7 +419,8 @@ class ArtifactsRepository(object): Returns: True if the artifacts exist, False otherwise ''' - raise NotImplementedError('Should be implemented in the inheriting class') + raise NotImplementedError( + 'Should be implemented in the inheriting class') class FileRepository(ArtifactsRepository): @@ -433,8 +436,9 @@ class FileRepository(ArtifactsRepository): try: subprocess.call(['cp', artifacts_name, self.uri]) except (subprocess.CalledProcessError, OSError) as ex: - logging.debug('Pushing artifacts to the repository failed: {}' - .format(str(ex))) + logging.debug( + 'Pushing artifacts to the repository failed: {}'.format( + str(ex))) return False return True @@ -449,9 +453,10 @@ class EosRepository(ArtifactsRepository): def pull(self, artifacts_name): try: - subprocess.check_call( - ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name], - ) + subprocess.check_call([ + 'xrdcp', + os.path.join(self.uri, artifacts_name), artifacts_name + ], ) except (subprocess.CalledProcessError, OSError) as ex: logging.debug('Pulling failed: {}'.format(str(ex))) return None @@ -461,13 +466,13 @@ class EosRepository(ArtifactsRepository): def push(self, artifacts_name): try: subprocess.check_call([ - 'xrdcp', artifacts_name, os.path.join( - self.uri, - os.path.basename(artifacts_name)) + 'xrdcp', artifacts_name, + os.path.join(self.uri, os.path.basename(artifacts_name)) ]) except (subprocess.CalledProcessError, OSError) as ex: - logging.debug('Pushing artifacts to the repository failed: {}' - .format(str(ex))) + logging.debug( + 'Pushing artifacts to the repository failed: {}'.format( + str(ex))) return False return True @@ -481,16 +486,16 @@ class HttpRepository(ArtifactsRepository): ''' def pull(self, artifacts_name): - with urlopen( - os.path.join(self.uri, artifacts_name)) as response, open(artifacts_name, 'wb') as out_file: + with urlopen(os.path.join(self.uri, artifacts_name)) as response, open( + artifacts_name, 'wb') as out_file: shutil.copyfileobj(response, out_file) return out_file def push(self, artifacts_name, auth=('', '')): req = Request( - url=os.path.join(self.uri, artifacts_name), - data=open(artifacts_name, 'rb').read(), - method='PUT') + url=os.path.join(self.uri, artifacts_name), + data=open(artifacts_name, 'rb').read(), + method='PUT') with urlopen(req) as f: pass if f.status != 201: -- GitLab From 06ceb4fca10b05c10dba5de66e5b79e376e90aa5 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 14:24:54 +0100 Subject: [PATCH 30/34] add module handling interactions with artifacts repository --- python/LbNightlyTools/Repository.py | 182 ++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 python/LbNightlyTools/Repository.py diff --git a/python/LbNightlyTools/Repository.py b/python/LbNightlyTools/Repository.py new file mode 100644 index 00000000..77a0e8fc --- /dev/null +++ b/python/LbNightlyTools/Repository.py @@ -0,0 +1,182 @@ +############################################################################### +# (c) Copyright 2013 CERN # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +''' +Code handling interactions with artifacts repository +''' + +_repo_handlers = {} + +def register_for(*schemes): + def _reg(cls): + global _repo_handlers + _repo_handlers.update((s, cls) for s in schemes) + return cls + return _reg + +def get_repo_type(uri): + ''' + Returns type of repository based on uri provided in the argument. + It may return 'eos', 'file', 'http' or None + ''' + from urllib.parse import urlparse + + result = urlparse(uri) + if not result.scheme or result.scheme == 'file': + if result.path.startswith('/eos/'): + return 'eos' + else: + return 'file' + elif result.scheme == 'root': + return 'eos' + elif result.scheme in ('http', 'https'): + return 'http' + return None + +class ArtifactsRepository(object): + ''' + Class representing artifacts repository. + ''' + + def __init__(self, uri): + ''' + Initialises the repository based on its URI + ''' + self.uri = uri + + def pull(self, artifacts_name): + ''' + Returns a file object in the read mode + + Arguments: + artifacts_name: name of the artifacts file + + Returns: File object + ''' + raise NotImplementedError( + 'Should be implemented in the inheriting class') + + def push(self, artifacts_name): + ''' + Pushes artifacts to the repository + + Arguments: + artifacts_name: name of the artifcats file + + Returns: True if the artifacts have been pushed, False otherwise + ''' + raise NotImplementedError( + 'Should be implemented in the inheriting class') + + def exist(self, artifacts_name): + ''' + Checks if artifacts exist + + Arguments: + artifacts_name: name of the artifcats file + + Returns: True if the artifacts exist, False otherwise + ''' + raise NotImplementedError( + 'Should be implemented in the inheriting class') + +@register_for('file') +class FileRepository(ArtifactsRepository): + ''' + Class defining repository in the local file system. + ''' + + def pull(self, artifacts_name): + artifact = open(os.path.join(self.uri, artifacts_name), 'r') + return artifact + + def push(self, artifacts_name): + try: + subprocess.call(['cp', artifacts_name, self.uri]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug( + 'Pushing artifacts to the repository failed: {}'.format( + str(ex))) + return False + return True + + def exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) + +@register_for('eos') +class EosRepository(ArtifactsRepository): + ''' + Class defining repository on EOS. + ''' + + def pull(self, artifacts_name): + try: + subprocess.check_call([ + 'xrdcp', + os.path.join(self.uri, artifacts_name), artifacts_name + ], ) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pulling failed: {}'.format(str(ex))) + return None + artifact = open(artifacts_name, 'r') + return artifact + + def push(self, artifacts_name): + try: + subprocess.check_call([ + 'xrdcp', artifacts_name, + os.path.join(self.uri, os.path.basename(artifacts_name)) + ]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug( + 'Pushing artifacts to the repository failed: {}'.format( + str(ex))) + return False + return True + + def exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) + +@register_for('http') +class HttpRepository(ArtifactsRepository): + ''' + Class defining repository through HTTP request methods. + ''' + + def pull(self, artifacts_name): + with urlopen(os.path.join(self.uri, artifacts_name)) as response, open( + artifacts_name, 'wb') as out_file: + shutil.copyfileobj(response, out_file) + return out_file + + def push(self, artifacts_name, auth=('', '')): + req = Request( + url=os.path.join(self.uri, artifacts_name), + data=open(artifacts_name, 'rb').read(), + method='PUT') + with urlopen(req) as f: + pass + if f.status != 201: + logging.debug('Pushing artifacts to the repository failed') + return False + return True + + def exist(self, artifacts_name): + from urllib.error import HTTPError + try: + urlopen(os.path.join(self.uri, artifacts_name)).close() + return True + except HTTPError: + return False + +def connect(uri, *args, **kwargs): + global _repo_handlers + return _repo_handlers[get_repo_type(uri)](uri, *args, **kwargs) + -- GitLab From 2f23809bdb9944974c532e2e8b388e01dec845e6 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 14:39:23 +0100 Subject: [PATCH 31/34] cleaning up merge request --- python/LbNightlyTools/Configuration.py | 61 +------------------ python/LbNightlyTools/Scripts/Build.py | 7 +-- python/LbNightlyTools/Scripts/Checkout.py | 11 +--- python/LbNightlyTools/Scripts/Common.py | 26 +++----- python/LbNightlyTools/Scripts/EnabledSlots.py | 18 ------ 5 files changed, 14 insertions(+), 109 deletions(-) diff --git a/python/LbNightlyTools/Configuration.py b/python/LbNightlyTools/Configuration.py index 22a94a40..1cd94610 100644 --- a/python/LbNightlyTools/Configuration.py +++ b/python/LbNightlyTools/Configuration.py @@ -22,7 +22,6 @@ import copy import urllib2 import urllib import json -from subprocess import call from datetime import datetime from collections import OrderedDict from StringIO import StringIO @@ -483,8 +482,6 @@ class Project(object): self.ignore_slot_build_tool = kwargs['ignore_slot_build_tool'] self.build_results = None - self.hash = self.calc_hash() - self.checkedout = False def toDict(self): ''' @@ -946,35 +943,6 @@ class Project(object): self._fixCMT(patchfile, dryrun=dryrun) self._fixProjectConfigJSON(patchfile, dryrun=dryrun) - - def calc_hash(self): - ''' - Returns the hash of the project which is the concatenation - of the commit id, and optionally merges. - ''' - # TODO: what about tagged versions and data packages? - hash = '' - try: - hash += self.checkout_opts['commit'][:7] - for merge in self.checkout_opts['merges']: - hash += merge[1][:7] - except KeyError: - __log__.debug('no commit id and merges for: ' + str(self.toDict())) - pass - return hash - - - def checkout_artifact_exists(self, artifacts_repo): - ''' - Checks if the project with this hash has already been checked out - ''' - file = os.path.join(artifacts_repo, - self.name + self.hash + '.zip') - if os.path.exists(file): - return True - else: - return False - class Package(object): ''' @@ -997,7 +965,6 @@ class Package(object): self.container = None self.checkout = kwargs.get('checkout') self.checkout_opts = kwargs.get('checkout_opts', {}) - self.hash = self.calc_hash() @property def slot(self): @@ -1105,11 +1072,6 @@ class Package(object): return "{0}/{1}".format(self.name, self.version) - def calc_hash(self): - # TODO: to be improved ? - return self.version - - class _ContainedList(object): ''' Helper class to handle a list of projects bound to a slot. @@ -1758,33 +1720,14 @@ class Slot(object): pass context = kwargs.pop('context', NullContext) - artifacts_repo = kwargs.pop('artifacts_repo') + results = OrderedDict() for project in self.activeProjects: if projects is None or project.name in projects: results[project.name] = {} try: with context(project) as ctx: - checkout_metadata_file = os.path.join( - artifacts_repo, - project.name + - project.hash + - '.txt') - if project.checkout_artifact_exists(artifacts_repo): - __log__.debug('project {} with hash {} ' - 'has already been checked out' - .format(project.name, project.hash)) - call(['unzip','-q', checkout_metadata_file.replace('.txt', '.zip')]) - with open(checkout_metadata_file, 'r') as infile: - checkout_metadata = json.load(infile) - results[project.name] = checkout_metadata - project.checkedout = True - continue - else: - results[project.name] = project.checkout(**kwargs) - __log__.debug('saving checkout metadata to: ' + checkout_metadata_file) - with open(checkout_metadata_file, 'w') as outfile: - json.dump(results, outfile) + results[project.name] = project.checkout(**kwargs) ctx.result = results[project.name] except (RuntimeError, AssertionError) as x: msg = 'failed to checkout {}: {}: {}'.format( diff --git a/python/LbNightlyTools/Scripts/Build.py b/python/LbNightlyTools/Scripts/Build.py index 847b774a..213c14b1 100644 --- a/python/LbNightlyTools/Scripts/Build.py +++ b/python/LbNightlyTools/Scripts/Build.py @@ -100,11 +100,6 @@ def unpackArtifacts(src, dest): # --clean option) call(['unzip', '-n', '-q', f], cwd=dest) - # TODO: copy and unpack only projects in slot, remove copying from eos in build.sh - # for project in slot: - # get_checkout_artifact - # unpack_artifact - try: from LbEnv import which @@ -290,7 +285,7 @@ class Script(BaseScript): os.path.join(self.artifacts_dir, 'ccache'), self.build_dir) if os.path.exists(os.path.join(self.artifacts_dir, 'slot.patch')): - self.log.warning('Applying patch file: %s' % os.path.join( + self.log.warning('Applaying patch file: %s' % os.path.join( self.artifacts_dir, 'slot.patch')) log_call([ 'patch', '-p1', '-i', diff --git a/python/LbNightlyTools/Scripts/Checkout.py b/python/LbNightlyTools/Scripts/Checkout.py index 78fcd0a4..99b0a2de 100644 --- a/python/LbNightlyTools/Scripts/Checkout.py +++ b/python/LbNightlyTools/Scripts/Checkout.py @@ -135,8 +135,7 @@ class Script(BaseScript): element, 'src', build_id=self.options.build_id, - artifacts_dir=self.options.artifacts_repo, - hash=element.hash) + artifacts_dir=self.artifacts_dir) def main(self): """ Main logic of the script """ @@ -201,8 +200,7 @@ class Script(BaseScript): slot.checkout( projects=opts.projects, ignore_errors=opts.ignore_checkout_errors, - context=CheckoutContext, - artifacts_repo=opts.artifacts_repo) + context=CheckoutContext) with open(join(self.artifacts_dir, 'slot.patch'), 'w') as patchfile: @@ -295,13 +293,10 @@ class Script(BaseScript): # ignore missing directories # (the project may not have been checked out) if not os.path.exists(join(self.build_dir, element.baseDir)): - # TODO fix this (correct path ?) self.log.warning('no sources for %s, skip packing', element) continue if isinstance(element, DataProject): continue # ignore DataProjects, because we pack packages - if project.checkedout: - continue self.log.info('packing %s %s...', element.name, element.version) @@ -317,7 +312,7 @@ class Script(BaseScript): contname.append(self.options.build_id) contname.append('src.zip') pack([container], - join(self.options.artifacts_repo, 'packs', 'src', '.'.join(contname)), + join(self.artifacts_dir, 'packs', 'src', '.'.join(contname)), cwd=self.build_dir, checksum='md5', dereference=False, diff --git a/python/LbNightlyTools/Scripts/Common.py b/python/LbNightlyTools/Scripts/Common.py index b1b72d35..ad5acc00 100755 --- a/python/LbNightlyTools/Scripts/Common.py +++ b/python/LbNightlyTools/Scripts/Common.py @@ -81,12 +81,6 @@ def addBasicOptions(parser): metavar='DIR', help='directory where to store the artifacts') - parser.add_option( - '--artifacts-repo', - action='store', - metavar='DIR', - help='repository with the checkout artifacts') - parser.add_option( '--projects', action='store', @@ -792,7 +786,7 @@ class BaseScript(PlainScript): cwd=self._buildDir(proj)).communicate() -def genPackageName(proj, platform, build_id=None, artifacts_dir=None, hash=None): +def genPackageName(proj, platform, build_id=None, artifacts_dir=None): ''' Generate the source/binary tarball name for a project/package. @@ -807,17 +801,13 @@ def genPackageName(proj, platform, build_id=None, artifacts_dir=None, hash=None) ... build_id='dummy', artifacts_dir='artifacts') 'artifacts/packs/x86_64-slc6-gcc48-dbg/Gaudi.v25r0.dummy.x86_64-slc6-gcc48-dbg.zip' ''' - if hash: - packname = proj.name.replace('/', '_') + hash + '.zip' - else: - packname = [proj.name.replace('/', '_'), proj.version] - if build_id: - packname.append(build_id) - packname.append(platform) - packname.append('zip') - packname = '.'.join(packname) - packname = os.path.join('packs', platform, packname) + packname = [proj.name.replace('/', '_'), proj.version] + if build_id: + packname.append(build_id) + packname.append(platform) + packname.append('zip') + packname = '.'.join(packname) + packname = os.path.join('packs', platform, packname) if artifacts_dir: packname = os.path.join(artifacts_dir, packname) - print('packname is: ' + packname) return packname diff --git a/python/LbNightlyTools/Scripts/EnabledSlots.py b/python/LbNightlyTools/Scripts/EnabledSlots.py index cedda2bb..eba09f96 100644 --- a/python/LbNightlyTools/Scripts/EnabledSlots.py +++ b/python/LbNightlyTools/Scripts/EnabledSlots.py @@ -97,24 +97,6 @@ class Script(PlainScript): 'config': slot.toDict(), 'date': self.options.date, } - for project in slot.toDict()['projects']: - filename = 'project-params-' + project['name'] - try: - filename += project['checkout_opts']['commit'][:7] - for merge in project['checkout_opts']['merges']: - filename += merge[1][:7] - except KeyError: - pass - filename += '.txt' - if os.path.isfile(filename): - self.log.info('project: {} already marked to be checked out' - .format(project['name'])) - else: - with open(filename, 'w') as f: - self.log.info('new project to checkout: {}' - .format(project['name'])) - f.write(str(project)) - if not self.options.submit: self.log.debug(' slot info: {}\n{}'.format( key, json.dumps(value, indent=2))) -- GitLab From 9dfeaa69902529a188d29a4053d70e323ff33751 Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 14:41:30 +0100 Subject: [PATCH 32/34] cleaning up merge request --- python/LbNightlyTools/Scripts/Build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/LbNightlyTools/Scripts/Build.py b/python/LbNightlyTools/Scripts/Build.py index 213c14b1..faa844a3 100644 --- a/python/LbNightlyTools/Scripts/Build.py +++ b/python/LbNightlyTools/Scripts/Build.py @@ -285,7 +285,7 @@ class Script(BaseScript): os.path.join(self.artifacts_dir, 'ccache'), self.build_dir) if os.path.exists(os.path.join(self.artifacts_dir, 'slot.patch')): - self.log.warning('Applaying patch file: %s' % os.path.join( + self.log.warning('Applying patch file: %s' % os.path.join( self.artifacts_dir, 'slot.patch')) log_call([ 'patch', '-p1', '-i', @@ -487,7 +487,7 @@ string(REPLACE "$${NIGHTLY_BUILD_ROOT}" "$${CMAKE_CURRENT_LIST_DIR}" }) if os.path.exists(os.path.join(self.artifacts_dir, 'slot.patch')): - self.log.warning('Applying patch file') + self.log.warning('Applaying patch file') log_call([ 'patch', '-p1', '-i', os.path.join(self.artifacts_dir, 'slot.patch') -- GitLab From 0d123eec5833f84a5dad5b2ee5ea9ccac3ef01b9 Mon Sep 17 00:00:00 2001 From: Gitlab CI <noreply@cern.ch> Date: Fri, 21 Feb 2020 12:39:39 +0000 Subject: [PATCH 33/34] Fixed formatting patch generated by https://gitlab.cern.ch/lhcb-core/LbNightlyTools/-/jobs/7287692 --- python/LbNightlyTools/Repository.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/LbNightlyTools/Repository.py b/python/LbNightlyTools/Repository.py index 77a0e8fc..6affa201 100644 --- a/python/LbNightlyTools/Repository.py +++ b/python/LbNightlyTools/Repository.py @@ -14,13 +14,16 @@ Code handling interactions with artifacts repository _repo_handlers = {} + def register_for(*schemes): def _reg(cls): global _repo_handlers _repo_handlers.update((s, cls) for s in schemes) return cls + return _reg + def get_repo_type(uri): ''' Returns type of repository based on uri provided in the argument. @@ -40,6 +43,7 @@ def get_repo_type(uri): return 'http' return None + class ArtifactsRepository(object): ''' Class representing artifacts repository. @@ -87,6 +91,7 @@ class ArtifactsRepository(object): raise NotImplementedError( 'Should be implemented in the inheriting class') + @register_for('file') class FileRepository(ArtifactsRepository): ''' @@ -110,6 +115,7 @@ class FileRepository(ArtifactsRepository): def exist(self, artifacts_name): return os.path.exists(os.path.join(self.uri, artifacts_name)) + @register_for('eos') class EosRepository(ArtifactsRepository): ''' @@ -144,6 +150,7 @@ class EosRepository(ArtifactsRepository): def exist(self, artifacts_name): return os.path.exists(os.path.join(self.uri, artifacts_name)) + @register_for('http') class HttpRepository(ArtifactsRepository): ''' @@ -176,7 +183,7 @@ class HttpRepository(ArtifactsRepository): except HTTPError: return False + def connect(uri, *args, **kwargs): global _repo_handlers return _repo_handlers[get_repo_type(uri)](uri, *args, **kwargs) - -- GitLab From f41ca7e2a2321737cece0cb0e50921410f544c1a Mon Sep 17 00:00:00 2001 From: Maciej Szymanski <maszyman@cern.ch> Date: Fri, 21 Feb 2020 13:56:10 +0100 Subject: [PATCH 34/34] add doc --- python/LbNightlyTools/Repository.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/LbNightlyTools/Repository.py b/python/LbNightlyTools/Repository.py index 6affa201..8a80ffe4 100644 --- a/python/LbNightlyTools/Repository.py +++ b/python/LbNightlyTools/Repository.py @@ -16,6 +16,9 @@ _repo_handlers = {} def register_for(*schemes): + ''' + Decorator used to register the concrete type of repository + ''' def _reg(cls): global _repo_handlers _repo_handlers.update((s, cls) for s in schemes) @@ -185,5 +188,9 @@ class HttpRepository(ArtifactsRepository): def connect(uri, *args, **kwargs): + ''' + Function returning the artifacts repository + object based on the URI provided in the argument. + ''' global _repo_handlers return _repo_handlers[get_repo_type(uri)](uri, *args, **kwargs) -- GitLab