diff --git a/python/LbNightlyTools/Repository.py b/python/LbNightlyTools/Repository.py new file mode 100644 index 0000000000000000000000000000000000000000..8a80ffe4ffbe699f7807ac7df325f86344162e11 --- /dev/null +++ b/python/LbNightlyTools/Repository.py @@ -0,0 +1,196 @@ +############################################################################### +# (c) Copyright 2013 CERN # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +''' +Code handling interactions with artifacts repository +''' + +_repo_handlers = {} + + +def register_for(*schemes): + ''' + Decorator used to register the concrete type of repository + ''' + def _reg(cls): + global _repo_handlers + _repo_handlers.update((s, cls) for s in schemes) + return cls + + return _reg + + +def get_repo_type(uri): + ''' + Returns type of repository based on uri provided in the argument. + It may return 'eos', 'file', 'http' or None + ''' + from urllib.parse import urlparse + + result = urlparse(uri) + if not result.scheme or result.scheme == 'file': + if result.path.startswith('/eos/'): + return 'eos' + else: + return 'file' + elif result.scheme == 'root': + return 'eos' + elif result.scheme in ('http', 'https'): + return 'http' + return None + + +class ArtifactsRepository(object): + ''' + Class representing artifacts repository. + ''' + + def __init__(self, uri): + ''' + Initialises the repository based on its URI + ''' + self.uri = uri + + def pull(self, artifacts_name): + ''' + Returns a file object in the read mode + + Arguments: + artifacts_name: name of the artifacts file + + Returns: File object + ''' + raise NotImplementedError( + 'Should be implemented in the inheriting class') + + def push(self, artifacts_name): + ''' + Pushes artifacts to the repository + + Arguments: + artifacts_name: name of the artifcats file + + Returns: True if the artifacts have been pushed, False otherwise + ''' + raise NotImplementedError( + 'Should be implemented in the inheriting class') + + def exist(self, artifacts_name): + ''' + Checks if artifacts exist + + Arguments: + artifacts_name: name of the artifcats file + + Returns: True if the artifacts exist, False otherwise + ''' + raise NotImplementedError( + 'Should be implemented in the inheriting class') + + +@register_for('file') +class FileRepository(ArtifactsRepository): + ''' + Class defining repository in the local file system. + ''' + + def pull(self, artifacts_name): + artifact = open(os.path.join(self.uri, artifacts_name), 'r') + return artifact + + def push(self, artifacts_name): + try: + subprocess.call(['cp', artifacts_name, self.uri]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug( + 'Pushing artifacts to the repository failed: {}'.format( + str(ex))) + return False + return True + + def exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) + + +@register_for('eos') +class EosRepository(ArtifactsRepository): + ''' + Class defining repository on EOS. + ''' + + def pull(self, artifacts_name): + try: + subprocess.check_call([ + 'xrdcp', + os.path.join(self.uri, artifacts_name), artifacts_name + ], ) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pulling failed: {}'.format(str(ex))) + return None + artifact = open(artifacts_name, 'r') + return artifact + + def push(self, artifacts_name): + try: + subprocess.check_call([ + 'xrdcp', artifacts_name, + os.path.join(self.uri, os.path.basename(artifacts_name)) + ]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug( + 'Pushing artifacts to the repository failed: {}'.format( + str(ex))) + return False + return True + + def exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) + + +@register_for('http') +class HttpRepository(ArtifactsRepository): + ''' + Class defining repository through HTTP request methods. + ''' + + def pull(self, artifacts_name): + with urlopen(os.path.join(self.uri, artifacts_name)) as response, open( + artifacts_name, 'wb') as out_file: + shutil.copyfileobj(response, out_file) + return out_file + + def push(self, artifacts_name, auth=('', '')): + req = Request( + url=os.path.join(self.uri, artifacts_name), + data=open(artifacts_name, 'rb').read(), + method='PUT') + with urlopen(req) as f: + pass + if f.status != 201: + logging.debug('Pushing artifacts to the repository failed') + return False + return True + + def exist(self, artifacts_name): + from urllib.error import HTTPError + try: + urlopen(os.path.join(self.uri, artifacts_name)).close() + return True + except HTTPError: + return False + + +def connect(uri, *args, **kwargs): + ''' + Function returning the artifacts repository + object based on the URI provided in the argument. + ''' + global _repo_handlers + return _repo_handlers[get_repo_type(uri)](uri, *args, **kwargs) diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index 15e467138f6898fdd06da01402b76af641213aa2..d7ca666065984acfc9852481425b617bb4cdd513 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -23,6 +23,7 @@ import subprocess import time import urllib2 import re +from urllib.request import urlopen, Request from datetime import datetime import gitlab @@ -374,6 +375,182 @@ def recursive_update(dest, data): return dest +class ArtifactsRepository(object): + ''' + Class representing artifacts repository. + ''' + + def __init__(self, uri): + ''' + Initialises the repository based on its URI + ''' + self.uri = uri + + def pull(self, artifacts_name): + ''' + Returns a file object in the read mode + + Arguments: + artifacts_name: name of the artifacts file + + Returns: File object + ''' + raise NotImplementedError( + 'Should be implemented in the inheriting class') + + def push(self, artifacts_name): + ''' + Pushes artifacts to the repository + + Arguments: + artifacts_name: name of the artifcats file + + Returns: True if the artifacts have been pushed, False otherwise + ''' + raise NotImplementedError( + 'Should be implemented in the inheriting class') + + def exist(self, artifacts_name): + ''' + Checks if artifacts exist + + Arguments: + artifacts_name: name of the artifcats file + + Returns: True if the artifacts exist, False otherwise + ''' + raise NotImplementedError( + 'Should be implemented in the inheriting class') + + +class FileRepository(ArtifactsRepository): + ''' + Class defining repository in the local file system. + ''' + + def pull(self, artifacts_name): + artifact = open(os.path.join(self.uri, artifacts_name), 'r') + return artifact + + def push(self, artifacts_name): + try: + subprocess.call(['cp', artifacts_name, self.uri]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug( + 'Pushing artifacts to the repository failed: {}'.format( + str(ex))) + return False + return True + + def exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) + + +class EosRepository(ArtifactsRepository): + ''' + Class defining repository on EOS. + ''' + + def pull(self, artifacts_name): + try: + subprocess.check_call([ + 'xrdcp', + os.path.join(self.uri, artifacts_name), artifacts_name + ], ) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug('Pulling failed: {}'.format(str(ex))) + return None + artifact = open(artifacts_name, 'r') + return artifact + + def push(self, artifacts_name): + try: + subprocess.check_call([ + 'xrdcp', artifacts_name, + os.path.join(self.uri, os.path.basename(artifacts_name)) + ]) + except (subprocess.CalledProcessError, OSError) as ex: + logging.debug( + 'Pushing artifacts to the repository failed: {}'.format( + str(ex))) + return False + return True + + def exist(self, artifacts_name): + return os.path.exists(os.path.join(self.uri, artifacts_name)) + + +class HttpRepository(ArtifactsRepository): + ''' + Class defining repository through HTTP request methods. + ''' + + def pull(self, artifacts_name): + with urlopen(os.path.join(self.uri, artifacts_name)) as response, open( + artifacts_name, 'wb') as out_file: + shutil.copyfileobj(response, out_file) + return out_file + + def push(self, artifacts_name, auth=('', '')): + req = Request( + url=os.path.join(self.uri, artifacts_name), + data=open(artifacts_name, 'rb').read(), + method='PUT') + with urlopen(req) as f: + pass + if f.status != 201: + logging.debug('Pushing artifacts to the repository failed') + return False + return True + + def exist(self, artifacts_name): + from urllib.error import HTTPError + try: + urlopen(os.path.join(self.uri, artifacts_name)).close() + return True + except HTTPError: + return False + + +def repo_type(uri): + ''' + Returns type of repository based on uri provided in the argument. + It may return 'eos', 'file', 'http' or None + ''' + try: + from urlparse import urlparse + except ImportError(): + from urllib.parse import urlparse + + result = urlparse(uri) + if not result.scheme or result.scheme == 'file': + if result.path.startswith('/eos/'): + return 'eos' + else: + return 'file' + elif result.scheme == 'root': + return 'eos' + elif result.scheme in ('http', 'https'): + return 'http' + return None + + +def repo_factory(url): + ''' + Factory function creating artifacts repository + object based on the URI provided in the argument. + ''' + _repos = { + 'file': FileRepository, + 'eos': EosRepository, + 'http': HttpRepository, + } + repo = _repos.get(repo_type(url)) + if not repo: + raise ValueError('Unrecognised repository type: {}'.format(url)) + return repo(url) + + class Dashboard(object): ''' Wrapper for the CouchDB-based dashboard.