From 1b97dca71249b4a8aa959dc51c20c6e9e1c54f4f Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Wed, 22 Jan 2020 09:46:50 +0100
Subject: [PATCH 01/34] saving and getting checkout artifacts from repo

---
 python/LbNightlyTools/Configuration.py        | 61 ++++++++++++++++++-
 python/LbNightlyTools/Scripts/Build.py        |  7 ++-
 python/LbNightlyTools/Scripts/Checkout.py     | 11 +++-
 python/LbNightlyTools/Scripts/Common.py       | 26 +++++---
 python/LbNightlyTools/Scripts/EnabledSlots.py | 18 ++++++
 5 files changed, 109 insertions(+), 14 deletions(-)

diff --git a/python/LbNightlyTools/Configuration.py b/python/LbNightlyTools/Configuration.py
index 1cd94610..22a94a40 100644
--- a/python/LbNightlyTools/Configuration.py
+++ b/python/LbNightlyTools/Configuration.py
@@ -22,6 +22,7 @@ import copy
 import urllib2
 import urllib
 import json
+from subprocess import call
 from datetime import datetime
 from collections import OrderedDict
 from StringIO import StringIO
@@ -482,6 +483,8 @@ class Project(object):
             self.ignore_slot_build_tool = kwargs['ignore_slot_build_tool']
 
         self.build_results = None
+        self.hash = self.calc_hash()
+        self.checkedout = False
 
     def toDict(self):
         '''
@@ -943,6 +946,35 @@ class Project(object):
         self._fixCMT(patchfile, dryrun=dryrun)
         self._fixProjectConfigJSON(patchfile, dryrun=dryrun)
 
+    
+    def calc_hash(self):
+        '''
+        Returns the hash of the project which is the concatenation
+        of the commit id, and optionally merges.                            
+        '''
+        # TODO: what about tagged versions and data packages?
+        hash = ''
+        try:
+            hash += self.checkout_opts['commit'][:7]
+            for merge in self.checkout_opts['merges']:
+                hash += merge[1][:7]
+        except KeyError:
+            __log__.debug('no commit id and merges for: ' + str(self.toDict()))
+            pass
+        return hash
+
+
+    def checkout_artifact_exists(self, artifacts_repo):
+        '''
+        Checks if the project with this hash has already been checked out
+        '''
+        file = os.path.join(artifacts_repo, 
+                            self.name + self.hash + '.zip')
+        if os.path.exists(file):
+            return True
+        else:
+            return False
+
 
 class Package(object):
     '''
@@ -965,6 +997,7 @@ class Package(object):
         self.container = None
         self.checkout = kwargs.get('checkout')
         self.checkout_opts = kwargs.get('checkout_opts', {})
+        self.hash = self.calc_hash()
 
     @property
     def slot(self):
@@ -1072,6 +1105,11 @@ class Package(object):
         return "{0}/{1}".format(self.name, self.version)
 
 
+    def calc_hash(self):
+        # TODO: to be improved ?
+        return self.version
+
+
 class _ContainedList(object):
     '''
     Helper class to handle a list of projects bound to a slot.
@@ -1720,14 +1758,33 @@ class Slot(object):
                 pass
 
         context = kwargs.pop('context', NullContext)
-
+        artifacts_repo = kwargs.pop('artifacts_repo')
         results = OrderedDict()
         for project in self.activeProjects:
             if projects is None or project.name in projects:
                 results[project.name] = {}
                 try:
                     with context(project) as ctx:
-                        results[project.name] = project.checkout(**kwargs)
+                        checkout_metadata_file = os.path.join(
+                                                    artifacts_repo,
+                                                    project.name + 
+                                                    project.hash +
+                                                    '.txt')
+                        if project.checkout_artifact_exists(artifacts_repo):
+                            __log__.debug('project {} with hash {} '
+                                          'has already been checked out'
+                                          .format(project.name, project.hash))
+                            call(['unzip','-q', checkout_metadata_file.replace('.txt', '.zip')])
+                            with open(checkout_metadata_file, 'r') as infile:
+                                checkout_metadata = json.load(infile)
+                            results[project.name] = checkout_metadata
+                            project.checkedout = True
+                            continue
+                        else:
+                            results[project.name] = project.checkout(**kwargs)
+                            __log__.debug('saving checkout metadata to: ' + checkout_metadata_file)
+                            with open(checkout_metadata_file, 'w') as outfile:
+                                json.dump(results, outfile)
                         ctx.result = results[project.name]
                 except (RuntimeError, AssertionError) as x:
                     msg = 'failed to checkout {}: {}: {}'.format(
diff --git a/python/LbNightlyTools/Scripts/Build.py b/python/LbNightlyTools/Scripts/Build.py
index faa844a3..847b774a 100644
--- a/python/LbNightlyTools/Scripts/Build.py
+++ b/python/LbNightlyTools/Scripts/Build.py
@@ -100,6 +100,11 @@ def unpackArtifacts(src, dest):
             # --clean option)
             call(['unzip', '-n', '-q', f], cwd=dest)
 
+    # TODO: copy and unpack only projects in slot, remove copying from eos in build.sh 
+    # for project in slot:
+    #     get_checkout_artifact
+    #     unpack_artifact
+
 
 try:
     from LbEnv import which
@@ -487,7 +492,7 @@ string(REPLACE "$${NIGHTLY_BUILD_ROOT}" "$${CMAKE_CURRENT_LIST_DIR}"
                 })
 
             if os.path.exists(os.path.join(self.artifacts_dir, 'slot.patch')):
-                self.log.warning('Applaying patch file')
+                self.log.warning('Applying patch file')
                 log_call([
                     'patch', '-p1', '-i',
                     os.path.join(self.artifacts_dir, 'slot.patch')
diff --git a/python/LbNightlyTools/Scripts/Checkout.py b/python/LbNightlyTools/Scripts/Checkout.py
index 99b0a2de..78fcd0a4 100644
--- a/python/LbNightlyTools/Scripts/Checkout.py
+++ b/python/LbNightlyTools/Scripts/Checkout.py
@@ -135,7 +135,8 @@ class Script(BaseScript):
             element,
             'src',
             build_id=self.options.build_id,
-            artifacts_dir=self.artifacts_dir)
+            artifacts_dir=self.options.artifacts_repo,
+            hash=element.hash)
 
     def main(self):
         """ Main logic of the script """
@@ -200,7 +201,8 @@ class Script(BaseScript):
             slot.checkout(
                 projects=opts.projects,
                 ignore_errors=opts.ignore_checkout_errors,
-                context=CheckoutContext)
+                context=CheckoutContext,
+                artifacts_repo=opts.artifacts_repo)
 
             with open(join(self.artifacts_dir, 'slot.patch'),
                       'w') as patchfile:
@@ -293,10 +295,13 @@ class Script(BaseScript):
             # ignore missing directories
             # (the project may not have been checked out)
             if not os.path.exists(join(self.build_dir, element.baseDir)):
+                # TODO fix this (correct path ?)
                 self.log.warning('no sources for %s, skip packing', element)
                 continue
             if isinstance(element, DataProject):
                 continue  # ignore DataProjects, because we pack packages
+            if project.checkedout:
+                continue
 
             self.log.info('packing %s %s...', element.name, element.version)
 
@@ -312,7 +317,7 @@ class Script(BaseScript):
                 contname.append(self.options.build_id)
             contname.append('src.zip')
             pack([container],
-                 join(self.artifacts_dir, 'packs', 'src', '.'.join(contname)),
+                 join(self.options.artifacts_repo, 'packs', 'src', '.'.join(contname)),
                  cwd=self.build_dir,
                  checksum='md5',
                  dereference=False,
diff --git a/python/LbNightlyTools/Scripts/Common.py b/python/LbNightlyTools/Scripts/Common.py
index 1f5ce6ff..f9c3d781 100755
--- a/python/LbNightlyTools/Scripts/Common.py
+++ b/python/LbNightlyTools/Scripts/Common.py
@@ -81,6 +81,12 @@ def addBasicOptions(parser):
         metavar='DIR',
         help='directory where to store the artifacts')
 
+    parser.add_option(
+        '--artifacts-repo',
+        action='store',
+        metavar='DIR',
+        help='repository with the checkout artifacts')
+
     parser.add_option(
         '--projects',
         action='store',
@@ -786,7 +792,7 @@ class BaseScript(PlainScript):
                       cwd=self._buildDir(proj)).communicate()
 
 
-def genPackageName(proj, platform, build_id=None, artifacts_dir=None):
+def genPackageName(proj, platform, build_id=None, artifacts_dir=None, hash=None):
     '''
     Generate the source/binary tarball name for a project/package.
 
@@ -801,13 +807,17 @@ def genPackageName(proj, platform, build_id=None, artifacts_dir=None):
     ...                build_id='dummy', artifacts_dir='artifacts')
     'artifacts/packs/x86_64-slc6-gcc48-dbg/Gaudi.v25r0.dummy.x86_64-slc6-gcc48-dbg.zip'
     '''
-    packname = [proj.name.replace('/', '_'), proj.version]
-    if build_id:
-        packname.append(build_id)
-    packname.append(platform)
-    packname.append('zip')
-    packname = '.'.join(packname)
-    packname = os.path.join('packs', platform, packname)
+    if hash:
+        packname = proj.name.replace('/', '_') + hash + '.zip'
+    else:
+        packname = [proj.name.replace('/', '_'), proj.version]
+        if build_id:
+            packname.append(build_id)
+        packname.append(platform)
+        packname.append('zip')
+        packname = '.'.join(packname)
+        packname = os.path.join('packs', platform, packname)
     if artifacts_dir:
         packname = os.path.join(artifacts_dir, packname)
+    print('packname is: ' + packname)
     return packname
diff --git a/python/LbNightlyTools/Scripts/EnabledSlots.py b/python/LbNightlyTools/Scripts/EnabledSlots.py
index eba09f96..cedda2bb 100644
--- a/python/LbNightlyTools/Scripts/EnabledSlots.py
+++ b/python/LbNightlyTools/Scripts/EnabledSlots.py
@@ -97,6 +97,24 @@ class Script(PlainScript):
                     'config': slot.toDict(),
                     'date': self.options.date,
                 }
+                for project in slot.toDict()['projects']:
+                    filename = 'project-params-' + project['name']
+                    try:
+                        filename += project['checkout_opts']['commit'][:7]
+                        for merge in project['checkout_opts']['merges']:
+                            filename += merge[1][:7]
+                    except KeyError:
+                        pass
+                    filename += '.txt'
+                    if os.path.isfile(filename):
+                        self.log.info('project: {} already marked to be checked out'
+                                      .format(project['name']))
+                    else:
+                        with open(filename, 'w') as f:
+                            self.log.info('new project to checkout: {}'
+                                          .format(project['name']))
+                            f.write(str(project))
+
                 if not self.options.submit:
                     self.log.debug('   slot info: {}\n{}'.format(
                         key, json.dumps(value, indent=2)))
-- 
GitLab


From 1dba6b1108ebf216dc21872857011394e3b6a775 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Wed, 19 Feb 2020 17:45:35 +0100
Subject: [PATCH 02/34] add class representing ArtifactsRepository

---
 python/LbNightlyTools/Utils.py | 180 +++++++++++++++++++++++++++++++++
 1 file changed, 180 insertions(+)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index 15e46713..f6002f66 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -23,6 +23,7 @@ import subprocess
 import time
 import urllib2
 import re
+import requests
 from datetime import datetime
 
 import gitlab
@@ -374,6 +375,185 @@ def recursive_update(dest, data):
     return dest
 
 
+def get_repo_type(uri):
+    '''
+    Returns type of repository based on uri provided in the argument.
+    It may return 'eos', 'file', 'http' or None
+    '''
+    try:
+        from urlparse import urlparse
+    except ImportError():
+        from urllib.parse import urlparse
+
+    result = urlparse(uri)
+    if result.scheme == '':
+        if result.path.startswith('/eos/'):
+            return 'eos'
+        else:
+            return 'file'
+    elif result.scheme == 'root':
+        return 'eos'
+    elif result.scheme == 'http':
+        return 'http'
+    return None
+
+
+class ArtifactsRepository(object):
+    '''
+    Class representing artifacts repository.
+    '''
+
+    def __init__(self, uri):
+        '''
+        Initialises the repository based on its URI
+        '''
+        self.uri = uri
+
+    def pull_artifacts(self, artifacts_name, dest):
+        '''
+        Gets artifacts from the repository and unzips them
+
+        Arguments:
+        artifacts_name: name of the artifacts file
+
+        Keyword arguments:
+        dest: to specify destination directory for unzipping
+
+        Returns: True if the artifacts have been extracted, False otherwise
+        '''
+        raise NotImplementedError('Should be implemented in the inheriting class')
+
+    def push_artifacts(self, artifacts_name):
+        '''
+        Pushes artifacts to the repository
+
+        Arguments:
+        artifacts_name: name of the artifcats file
+
+        Returns: True if the artifacts have been pushed, False otherwise
+        '''
+        raise NotImplementedError('Should be implemented in the inheriting class')
+
+    def artifacts_exist(self, artifacts_name):
+        '''
+        Checks if artifacts exist
+
+        Arguments:
+        artifacts_name: name of the artifcats file
+
+        Returns: True if the artifacts exist, False otherwise
+        '''
+        raise NotImplementedError('Should be implemented in the inheriting class')
+
+
+def repo_factory(uri):
+    '''
+    Factory function creating artifacts repository
+    object based on the URI provided in the argument.
+    '''
+
+    class FileRepository(ArtifactsRepository):
+        '''
+        Class defining repository in the local file system.
+        '''
+
+        def pull_artifacts(self, artifacts_name, dest):
+            try:
+                subprocess.check_call(
+                    ['unzip', '-q', os.path.join(self.uri, artifacts_name)], cwd=dest)
+            except (subprocess.CalledProcessError, OSError) as ex:
+                logging.debug('Unzipping failed: {}'.format(str(ex)))
+                return False
+            return True
+
+        def push_artifacts(self, artifacts_name):
+            try:
+                logging.debug('pushing {} to {}'.format(artifacts_name, self.uri))
+                subprocess.call(['cp', artifacts_name, self.uri])
+            except (subprocess.CalledProcessError, OSError) as ex:
+                logging.debug('Pushing artifacts to the repository failed: {}'
+                            .format(str(ex)))
+                return False
+            return True
+
+        def artifacts_exist(self, artifacts_name):
+            return os.path.exists(os.path.join(self.uri, artifacts_name))
+
+    class EosRepository(ArtifactsRepository):
+        '''
+        Class defining repository on EOS.
+        '''
+
+        def pull_artifacts(self, artifacts_name, dest):
+            try:
+                subprocess.check_call(
+                    ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name],
+                    cwd=dest)
+                subprocess.check_call(['unzip', '-q', artifacts_name],
+                                  cwd=dest)
+            except (subprocess.CalledProcessError, OSError) as ex:
+                logging.debug('Pulling failed: {}'.format(str(ex)))
+                return False
+            return True
+
+        def push_artifacts(self, artifacts_name):
+            try:
+                subprocess.check_call([
+                    'xrdcp', artifacts_name, os.path.join(
+                                                self.uri,
+                                                os.path.basename(artifacts_name))
+                ])
+            except (subprocess.CalledProcessError, OSError) as ex:
+                logging.debug('Pushing artifacts to the repository failed: {}'
+                            .format(str(ex)))
+                return False
+            return True
+
+        def artifacts_exist(self, artifacts_name):
+            return os.path.exists(os.path.join(self.uri, artifacts_name))
+
+    class HttpRepository(ArtifactsRepository):
+        '''
+        Class defining repository through HTTP request methods.
+        '''
+
+        def pull_artifacts(self, artifacts_name, dest):
+            r = requests.get(os.path.join(self.uri, artifacts_name))
+            if r.status_code != 200:
+                logging.debug('Resource does not exist')
+                return False
+            with open(os.path.join(dest, artifacts_name), 'wb') as f:
+                f.write(r.content)
+            subprocess.check_call(['unzip','-q', os.path.join(dest, artifacts_name)],
+                                  cwd=dest)
+            return True
+
+        def push_artifacts(self, artifacts_name, auth=('', '')):
+            r = requests.put(os.path.join(self.uri, artifacts_name),
+                             data=open(artifacts_name, 'rb').read(),
+                             headers={'content-type': 'application/zip'},
+                             auth=auth)
+            if r.status_code != 201:
+                logging.debug('Pushing artifacts to the repository failed')
+                return False
+            return True
+
+        def artifacts_exist(self, artifacts_name):
+            r = requests.get(os.path.join(self.uri, artifacts_name))
+            if r.status_code == 200:
+                return True
+            return False
+
+    if get_repo_type(uri) == 'file':
+        return FileRepository(uri)
+    elif get_repo_type(uri) == 'eos':
+        return EosRepository(uri)
+    elif get_repo_type(uri) == 'http':
+        return HttpRepository(uri)
+    else:
+        raise ValueError("Unrecognised repository type for: {}".format(uri))
+
+
 class Dashboard(object):
     '''
     Wrapper for the CouchDB-based dashboard.
-- 
GitLab


From d7d5b5565aba234f84f6e569ead40f8cd9b8ebac Mon Sep 17 00:00:00 2001
From: Maciej Pawel Szymanski <maciej.szymanski@cern.ch>
Date: Thu, 20 Feb 2020 08:35:32 +0000
Subject: [PATCH 03/34] Apply suggestion to python/LbNightlyTools/Utils.py

---
 python/LbNightlyTools/Utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index f6002f66..505cf97d 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -393,7 +393,7 @@ def get_repo_type(uri):
             return 'file'
     elif result.scheme == 'root':
         return 'eos'
-    elif result.scheme == 'http':
+    elif result.scheme in ('http', 'https'):
         return 'http'
     return None
 
-- 
GitLab


From 18bc92149cc0f13924cd056593fba1be811397bc Mon Sep 17 00:00:00 2001
From: Maciej Pawel Szymanski <maciej.szymanski@cern.ch>
Date: Thu, 20 Feb 2020 08:37:14 +0000
Subject: [PATCH 04/34] Apply suggestion to python/LbNightlyTools/Utils.py

---
 python/LbNightlyTools/Utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index 505cf97d..61f70264 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -386,7 +386,7 @@ def get_repo_type(uri):
         from urllib.parse import urlparse
 
     result = urlparse(uri)
-    if result.scheme == '':
+    if not result.scheme or result.scheme == 'file':
         if result.path.startswith('/eos/'):
             return 'eos'
         else:
-- 
GitLab


From 8c25eeb07b4431f29d3c22ed4b69da517d6a5dd7 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 10:45:38 +0100
Subject: [PATCH 05/34] use urllib instead of requests

---
 python/LbNightlyTools/Utils.py | 47 +++++++++++++++++-----------------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index f6002f66..4625155d 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -23,7 +23,7 @@ import subprocess
 import time
 import urllib2
 import re
-import requests
+from urllib.request import urlopen, Request
 from datetime import datetime
 
 import gitlab
@@ -517,31 +517,30 @@ def repo_factory(uri):
         Class defining repository through HTTP request methods.
         '''
 
-        def pull_artifacts(self, artifacts_name, dest):
-            r = requests.get(os.path.join(self.uri, artifacts_name))
-            if r.status_code != 200:
-                logging.debug('Resource does not exist')
-                return False
-            with open(os.path.join(dest, artifacts_name), 'wb') as f:
-                f.write(r.content)
-            subprocess.check_call(['unzip','-q', os.path.join(dest, artifacts_name)],
-                                  cwd=dest)
-            return True
+    def pull(self, artifacts_name):
+        with urlopen(
+                os.path.join(self.uri, artifacts_name)) as response, open(artifacts_name, 'wb') as out_file:
+            shutil.copyfileobj(response, out_file)
+        return out_file
+
+    def push(self, artifacts_name, auth=('', '')):
+        req = Request(
+                url=os.path.join(self.uri, artifacts_name),
+                data=open(artifacts_name, 'rb').read(),
+                method='PUT')
+        with urlopen(req) as f:
+            pass
+        if f.status != 201:
+            logging.debug('Pushing artifacts to the repository failed')
+            return False
+        return True
 
-        def push_artifacts(self, artifacts_name, auth=('', '')):
-            r = requests.put(os.path.join(self.uri, artifacts_name),
-                             data=open(artifacts_name, 'rb').read(),
-                             headers={'content-type': 'application/zip'},
-                             auth=auth)
-            if r.status_code != 201:
-                logging.debug('Pushing artifacts to the repository failed')
-                return False
+    def exist(self, artifacts_name):
+        from urllib.error import HTTPError
+        try:
+            urlopen(os.path.join(self.uri, artifacts_name)).close()
             return True
-
-        def artifacts_exist(self, artifacts_name):
-            r = requests.get(os.path.join(self.uri, artifacts_name))
-            if r.status_code == 200:
-                return True
+        except HTTPError:
             return False
 
     if get_repo_type(uri) == 'file':
-- 
GitLab


From aa1689221dc9d087db3462cf01e0b840cb302ec4 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 10:46:15 +0100
Subject: [PATCH 06/34] refactor factory method

---
 python/LbNightlyTools/Utils.py | 69 +++++++++++++++++++---------------
 1 file changed, 38 insertions(+), 31 deletions(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index 4625155d..d93d463b 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -375,29 +375,6 @@ def recursive_update(dest, data):
     return dest
 
 
-def get_repo_type(uri):
-    '''
-    Returns type of repository based on uri provided in the argument.
-    It may return 'eos', 'file', 'http' or None
-    '''
-    try:
-        from urlparse import urlparse
-    except ImportError():
-        from urllib.parse import urlparse
-
-    result = urlparse(uri)
-    if result.scheme == '':
-        if result.path.startswith('/eos/'):
-            return 'eos'
-        else:
-            return 'file'
-    elif result.scheme == 'root':
-        return 'eos'
-    elif result.scheme == 'http':
-        return 'http'
-    return None
-
-
 class ArtifactsRepository(object):
     '''
     Class representing artifacts repository.
@@ -543,14 +520,44 @@ def repo_factory(uri):
         except HTTPError:
             return False
 
-    if get_repo_type(uri) == 'file':
-        return FileRepository(uri)
-    elif get_repo_type(uri) == 'eos':
-        return EosRepository(uri)
-    elif get_repo_type(uri) == 'http':
-        return HttpRepository(uri)
-    else:
-        raise ValueError("Unrecognised repository type for: {}".format(uri))
+
+def repo_type(uri):
+    '''
+    Returns type of repository based on uri provided in the argument.
+    It may return 'eos', 'file', 'http' or None
+    '''
+    try:
+        from urlparse import urlparse
+    except ImportError():
+        from urllib.parse import urlparse
+
+    result = urlparse(uri)
+    if result.scheme == '':
+        if result.path.startswith('/eos/'):
+            return 'eos'
+        else:
+            return 'file'
+    elif result.scheme == 'root':
+        return 'eos'
+    elif result.scheme == 'http':
+        return 'http'
+    return None
+
+
+def repo_factory(url):
+    '''
+    Factory function creating artifacts repository
+    object based on the URI provided in the argument.
+    '''
+    _repos = {
+        'file': FileRepository,
+        'eos': EosRepository,
+        'http': HttpRepository,
+    }
+    repo = _repos.get(repo_type(url))
+    if not repo:
+        raise ValueError('Unrecognised repository type: {}'.format(url))
+    return repo(url)
 
 
 class Dashboard(object):
-- 
GitLab


From beafb62df3083378a5f7bc6d09c5851370758776 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 10:47:12 +0100
Subject: [PATCH 07/34] remove from methods name redundant artifacts word

---
 python/LbNightlyTools/Utils.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index d93d463b..b56da743 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -386,21 +386,18 @@ class ArtifactsRepository(object):
         '''
         self.uri = uri
 
-    def pull_artifacts(self, artifacts_name, dest):
+    def pull(self, artifacts_name):
         '''
-        Gets artifacts from the repository and unzips them
+        Returns a file object in the read mode
 
         Arguments:
         artifacts_name: name of the artifacts file
 
-        Keyword arguments:
-        dest: to specify destination directory for unzipping
-
-        Returns: True if the artifacts have been extracted, False otherwise
+        Returns: File object
         '''
         raise NotImplementedError('Should be implemented in the inheriting class')
 
-    def push_artifacts(self, artifacts_name):
+    def push(self, artifacts_name):
         '''
         Pushes artifacts to the repository
 
@@ -411,7 +408,7 @@ class ArtifactsRepository(object):
         '''
         raise NotImplementedError('Should be implemented in the inheriting class')
 
-    def artifacts_exist(self, artifacts_name):
+    def exist(self, artifacts_name):
         '''
         Checks if artifacts exist
 
-- 
GitLab


From 50a923edd26ae13a043e8df2ae57a29788f62f56 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 10:47:53 +0100
Subject: [PATCH 08/34] dont unzip in pull

---
 python/LbNightlyTools/Utils.py | 107 +++++++++++++++------------------
 1 file changed, 48 insertions(+), 59 deletions(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index b56da743..609f14af 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -420,76 +420,65 @@ class ArtifactsRepository(object):
         raise NotImplementedError('Should be implemented in the inheriting class')
 
 
-def repo_factory(uri):
+class FileRepository(ArtifactsRepository):
     '''
-    Factory function creating artifacts repository
-    object based on the URI provided in the argument.
+    Class defining repository in the local file system.
     '''
 
-    class FileRepository(ArtifactsRepository):
-        '''
-        Class defining repository in the local file system.
-        '''
+    def pull(self, artifacts_name):
+        artifact = open(os.path.join(self.uri, artifacts_name), 'r')
+        return artifact
 
-        def pull_artifacts(self, artifacts_name, dest):
-            try:
-                subprocess.check_call(
-                    ['unzip', '-q', os.path.join(self.uri, artifacts_name)], cwd=dest)
-            except (subprocess.CalledProcessError, OSError) as ex:
-                logging.debug('Unzipping failed: {}'.format(str(ex)))
-                return False
-            return True
+    def push(self, artifacts_name):
+        try:
+            subprocess.call(['cp', artifacts_name, self.uri])
+        except (subprocess.CalledProcessError, OSError) as ex:
+            logging.debug('Pushing artifacts to the repository failed: {}'
+                        .format(str(ex)))
+            return False
+        return True
 
-        def push_artifacts(self, artifacts_name):
-            try:
-                logging.debug('pushing {} to {}'.format(artifacts_name, self.uri))
-                subprocess.call(['cp', artifacts_name, self.uri])
-            except (subprocess.CalledProcessError, OSError) as ex:
-                logging.debug('Pushing artifacts to the repository failed: {}'
-                            .format(str(ex)))
-                return False
-            return True
+    def exist(self, artifacts_name):
+        return os.path.exists(os.path.join(self.uri, artifacts_name))
 
-        def artifacts_exist(self, artifacts_name):
-            return os.path.exists(os.path.join(self.uri, artifacts_name))
 
-    class EosRepository(ArtifactsRepository):
-        '''
-        Class defining repository on EOS.
-        '''
+class EosRepository(ArtifactsRepository):
+    '''
+    Class defining repository on EOS.
+    '''
 
-        def pull_artifacts(self, artifacts_name, dest):
-            try:
-                subprocess.check_call(
-                    ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name],
-                    cwd=dest)
-                subprocess.check_call(['unzip', '-q', artifacts_name],
-                                  cwd=dest)
-            except (subprocess.CalledProcessError, OSError) as ex:
-                logging.debug('Pulling failed: {}'.format(str(ex)))
-                return False
-            return True
+    def pull(self, artifacts_name):
+        try:
+            subprocess.check_call(
+                ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name],
+            )
+        except (subprocess.CalledProcessError, OSError) as ex:
+            logging.debug('Pulling failed: {}'.format(str(ex)))
+            return None
+        artifact = open(artifacts_name, 'r')
+        return artifact
 
-        def push_artifacts(self, artifacts_name):
-            try:
-                subprocess.check_call([
-                    'xrdcp', artifacts_name, os.path.join(
-                                                self.uri,
-                                                os.path.basename(artifacts_name))
-                ])
-            except (subprocess.CalledProcessError, OSError) as ex:
-                logging.debug('Pushing artifacts to the repository failed: {}'
-                            .format(str(ex)))
-                return False
-            return True
+    def push(self, artifacts_name):
+        try:
+            subprocess.check_call([
+                'xrdcp', artifacts_name, os.path.join(
+                                            self.uri,
+                                            os.path.basename(artifacts_name))
+            ])
+        except (subprocess.CalledProcessError, OSError) as ex:
+            logging.debug('Pushing artifacts to the repository failed: {}'
+                        .format(str(ex)))
+            return False
+        return True
 
-        def artifacts_exist(self, artifacts_name):
-            return os.path.exists(os.path.join(self.uri, artifacts_name))
+    def exist(self, artifacts_name):
+        return os.path.exists(os.path.join(self.uri, artifacts_name))
 
-    class HttpRepository(ArtifactsRepository):
-        '''
-        Class defining repository through HTTP request methods.
-        '''
+
+class HttpRepository(ArtifactsRepository):
+    '''
+    Class defining repository through HTTP request methods.
+    '''
 
     def pull(self, artifacts_name):
         with urlopen(
-- 
GitLab


From 8ec53449b34c55ea5ff91cf2485afd6922c2ef07 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 10:50:47 +0100
Subject: [PATCH 09/34] fix function name

---
 python/LbNightlyTools/Utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index 7c8afaf3..a4264e92 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -375,7 +375,7 @@ def recursive_update(dest, data):
     return dest
 
 
-def get_repo_type(uri):
+def repo_type(uri):
     '''
     Returns type of repository based on uri provided in the argument.
     It may return 'eos', 'file', 'http' or None
-- 
GitLab


From ffe0383232f36cc1f77002b80b1d76d0ac9d8225 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 11:13:28 +0100
Subject: [PATCH 10/34] remove redundant definition

---
 python/LbNightlyTools/Utils.py | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index a4264e92..609f14af 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -375,29 +375,6 @@ def recursive_update(dest, data):
     return dest
 
 
-def repo_type(uri):
-    '''
-    Returns type of repository based on uri provided in the argument.
-    It may return 'eos', 'file', 'http' or None
-    '''
-    try:
-        from urlparse import urlparse
-    except ImportError():
-        from urllib.parse import urlparse
-
-    result = urlparse(uri)
-    if not result.scheme or result.scheme == 'file':
-        if result.path.startswith('/eos/'):
-            return 'eos'
-        else:
-            return 'file'
-    elif result.scheme == 'root':
-        return 'eos'
-    elif result.scheme in ('http', 'https'):
-        return 'http'
-    return None
-
-
 class ArtifactsRepository(object):
     '''
     Class representing artifacts repository.
-- 
GitLab


From e2434e053a46d3b06135a36de905e5da2857bdaf Mon Sep 17 00:00:00 2001
From: Gitlab CI <noreply@cern.ch>
Date: Fri, 21 Feb 2020 10:18:45 +0000
Subject: [PATCH 11/34] Fixed formatting

patch generated by https://gitlab.cern.ch/lhcb-core/LbNightlyTools/-/jobs/7285194
---
 python/LbNightlyTools/Utils.py | 41 +++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index 609f14af..20949e12 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -395,7 +395,8 @@ class ArtifactsRepository(object):
 
         Returns: File object
         '''
-        raise NotImplementedError('Should be implemented in the inheriting class')
+        raise NotImplementedError(
+            'Should be implemented in the inheriting class')
 
     def push(self, artifacts_name):
         '''
@@ -406,7 +407,8 @@ class ArtifactsRepository(object):
 
         Returns: True if the artifacts have been pushed, False otherwise
         '''
-        raise NotImplementedError('Should be implemented in the inheriting class')
+        raise NotImplementedError(
+            'Should be implemented in the inheriting class')
 
     def exist(self, artifacts_name):
         '''
@@ -417,7 +419,8 @@ class ArtifactsRepository(object):
 
         Returns: True if the artifacts exist, False otherwise
         '''
-        raise NotImplementedError('Should be implemented in the inheriting class')
+        raise NotImplementedError(
+            'Should be implemented in the inheriting class')
 
 
 class FileRepository(ArtifactsRepository):
@@ -433,8 +436,9 @@ class FileRepository(ArtifactsRepository):
         try:
             subprocess.call(['cp', artifacts_name, self.uri])
         except (subprocess.CalledProcessError, OSError) as ex:
-            logging.debug('Pushing artifacts to the repository failed: {}'
-                        .format(str(ex)))
+            logging.debug(
+                'Pushing artifacts to the repository failed: {}'.format(
+                    str(ex)))
             return False
         return True
 
@@ -449,9 +453,10 @@ class EosRepository(ArtifactsRepository):
 
     def pull(self, artifacts_name):
         try:
-            subprocess.check_call(
-                ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name],
-            )
+            subprocess.check_call([
+                'xrdcp',
+                os.path.join(self.uri, artifacts_name), artifacts_name
+            ], )
         except (subprocess.CalledProcessError, OSError) as ex:
             logging.debug('Pulling failed: {}'.format(str(ex)))
             return None
@@ -461,13 +466,13 @@ class EosRepository(ArtifactsRepository):
     def push(self, artifacts_name):
         try:
             subprocess.check_call([
-                'xrdcp', artifacts_name, os.path.join(
-                                            self.uri,
-                                            os.path.basename(artifacts_name))
+                'xrdcp', artifacts_name,
+                os.path.join(self.uri, os.path.basename(artifacts_name))
             ])
         except (subprocess.CalledProcessError, OSError) as ex:
-            logging.debug('Pushing artifacts to the repository failed: {}'
-                        .format(str(ex)))
+            logging.debug(
+                'Pushing artifacts to the repository failed: {}'.format(
+                    str(ex)))
             return False
         return True
 
@@ -481,16 +486,16 @@ class HttpRepository(ArtifactsRepository):
     '''
 
     def pull(self, artifacts_name):
-        with urlopen(
-                os.path.join(self.uri, artifacts_name)) as response, open(artifacts_name, 'wb') as out_file:
+        with urlopen(os.path.join(self.uri, artifacts_name)) as response, open(
+                artifacts_name, 'wb') as out_file:
             shutil.copyfileobj(response, out_file)
         return out_file
 
     def push(self, artifacts_name, auth=('', '')):
         req = Request(
-                url=os.path.join(self.uri, artifacts_name),
-                data=open(artifacts_name, 'rb').read(),
-                method='PUT')
+            url=os.path.join(self.uri, artifacts_name),
+            data=open(artifacts_name, 'rb').read(),
+            method='PUT')
         with urlopen(req) as f:
             pass
         if f.status != 201:
-- 
GitLab


From 4eeefca55beca91af4feb83951621335d2097fb2 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 12:36:50 +0100
Subject: [PATCH 12/34] re-apply suggestions

---
 python/LbNightlyTools/Utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index 20949e12..d7ca6660 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -523,14 +523,14 @@ def repo_type(uri):
         from urllib.parse import urlparse
 
     result = urlparse(uri)
-    if result.scheme == '':
+    if not result.scheme or result.scheme == 'file':
         if result.path.startswith('/eos/'):
             return 'eos'
         else:
             return 'file'
     elif result.scheme == 'root':
         return 'eos'
-    elif result.scheme == 'http':
+    elif result.scheme in ('http', 'https'):
         return 'http'
     return None
 
-- 
GitLab


From 72202d4334c975a2277d9d51d0035855e4f95bd6 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 14:24:54 +0100
Subject: [PATCH 13/34] add module handling interactions with artifacts
 repository

---
 python/LbNightlyTools/Repository.py | 182 ++++++++++++++++++++++++++++
 1 file changed, 182 insertions(+)
 create mode 100644 python/LbNightlyTools/Repository.py

diff --git a/python/LbNightlyTools/Repository.py b/python/LbNightlyTools/Repository.py
new file mode 100644
index 00000000..77a0e8fc
--- /dev/null
+++ b/python/LbNightlyTools/Repository.py
@@ -0,0 +1,182 @@
+###############################################################################
+# (c) Copyright 2013 CERN                                                     #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+'''
+Code handling interactions with artifacts repository
+'''
+
+_repo_handlers = {}
+
+def register_for(*schemes):
+    def _reg(cls):
+        global _repo_handlers
+        _repo_handlers.update((s, cls) for s in schemes)
+        return cls
+    return _reg
+
+def get_repo_type(uri):
+    '''
+    Returns type of repository based on uri provided in the argument.
+    It may return 'eos', 'file', 'http' or None
+    '''
+    from urllib.parse import urlparse
+
+    result = urlparse(uri)
+    if not result.scheme or result.scheme == 'file':
+        if result.path.startswith('/eos/'):
+            return 'eos'
+        else:
+            return 'file'
+    elif result.scheme == 'root':
+        return 'eos'
+    elif result.scheme in ('http', 'https'):
+        return 'http'
+    return None
+
+class ArtifactsRepository(object):
+    '''
+    Class representing artifacts repository.
+    '''
+
+    def __init__(self, uri):
+        '''
+        Initialises the repository based on its URI
+        '''
+        self.uri = uri
+
+    def pull(self, artifacts_name):
+        '''
+        Returns a file object in the read mode
+
+        Arguments:
+        artifacts_name: name of the artifacts file
+
+        Returns: File object
+        '''
+        raise NotImplementedError(
+            'Should be implemented in the inheriting class')
+
+    def push(self, artifacts_name):
+        '''
+        Pushes artifacts to the repository
+
+        Arguments:
+        artifacts_name: name of the artifcats file
+
+        Returns: True if the artifacts have been pushed, False otherwise
+        '''
+        raise NotImplementedError(
+            'Should be implemented in the inheriting class')
+
+    def exist(self, artifacts_name):
+        '''
+        Checks if artifacts exist
+
+        Arguments:
+        artifacts_name: name of the artifcats file
+
+        Returns: True if the artifacts exist, False otherwise
+        '''
+        raise NotImplementedError(
+            'Should be implemented in the inheriting class')
+
+@register_for('file')
+class FileRepository(ArtifactsRepository):
+    '''
+    Class defining repository in the local file system.
+    '''
+
+    def pull(self, artifacts_name):
+        artifact = open(os.path.join(self.uri, artifacts_name), 'r')
+        return artifact
+
+    def push(self, artifacts_name):
+        try:
+            subprocess.call(['cp', artifacts_name, self.uri])
+        except (subprocess.CalledProcessError, OSError) as ex:
+            logging.debug(
+                'Pushing artifacts to the repository failed: {}'.format(
+                    str(ex)))
+            return False
+        return True
+
+    def exist(self, artifacts_name):
+        return os.path.exists(os.path.join(self.uri, artifacts_name))
+
+@register_for('eos')
+class EosRepository(ArtifactsRepository):
+    '''
+    Class defining repository on EOS.
+    '''
+
+    def pull(self, artifacts_name):
+        try:
+            subprocess.check_call([
+                'xrdcp',
+                os.path.join(self.uri, artifacts_name), artifacts_name
+            ], )
+        except (subprocess.CalledProcessError, OSError) as ex:
+            logging.debug('Pulling failed: {}'.format(str(ex)))
+            return None
+        artifact = open(artifacts_name, 'r')
+        return artifact
+
+    def push(self, artifacts_name):
+        try:
+            subprocess.check_call([
+                'xrdcp', artifacts_name,
+                os.path.join(self.uri, os.path.basename(artifacts_name))
+            ])
+        except (subprocess.CalledProcessError, OSError) as ex:
+            logging.debug(
+                'Pushing artifacts to the repository failed: {}'.format(
+                    str(ex)))
+            return False
+        return True
+
+    def exist(self, artifacts_name):
+        return os.path.exists(os.path.join(self.uri, artifacts_name))
+
+@register_for('http')
+class HttpRepository(ArtifactsRepository):
+    '''
+    Class defining repository through HTTP request methods.
+    '''
+
+    def pull(self, artifacts_name):
+        with urlopen(os.path.join(self.uri, artifacts_name)) as response, open(
+                artifacts_name, 'wb') as out_file:
+            shutil.copyfileobj(response, out_file)
+        return out_file
+
+    def push(self, artifacts_name, auth=('', '')):
+        req = Request(
+            url=os.path.join(self.uri, artifacts_name),
+            data=open(artifacts_name, 'rb').read(),
+            method='PUT')
+        with urlopen(req) as f:
+            pass
+        if f.status != 201:
+            logging.debug('Pushing artifacts to the repository failed')
+            return False
+        return True
+
+    def exist(self, artifacts_name):
+        from urllib.error import HTTPError
+        try:
+            urlopen(os.path.join(self.uri, artifacts_name)).close()
+            return True
+        except HTTPError:
+            return False
+
+def connect(uri, *args, **kwargs):
+    global _repo_handlers
+    return _repo_handlers[get_repo_type(uri)](uri, *args, **kwargs)
+    
-- 
GitLab


From d40fd1dd21acd38de0b915801292d4988a66f38d Mon Sep 17 00:00:00 2001
From: Maciej Pawel Szymanski <maciej.szymanski@cern.ch>
Date: Fri, 21 Feb 2020 12:27:41 +0000
Subject: [PATCH 14/34] Revert "Merge branch 'maszyman-artifacts-repository'
 of..."

This reverts commit 268274adb987f856f2e924750fe07fa554ec1a9e
---
 python/LbNightlyTools/BuildMethods.py       |   3 +-
 python/LbNightlyTools/MergeRequestBuilds.py |   9 +-
 python/LbNightlyTools/Scripts/Common.py     |   4 +-
 python/LbNightlyTools/Scripts/Test.py       |  17 +-
 python/LbNightlyTools/Utils.py              | 177 --------------------
 setup.py                                    |   2 +-
 6 files changed, 15 insertions(+), 197 deletions(-)

diff --git a/python/LbNightlyTools/BuildMethods.py b/python/LbNightlyTools/BuildMethods.py
index 6986c1b8..b4756cab 100644
--- a/python/LbNightlyTools/BuildMethods.py
+++ b/python/LbNightlyTools/BuildMethods.py
@@ -210,7 +210,8 @@ class cmt(make):
                                 or os.environ.get('BINARY_TAG'))
         if 'GAUDI_QMTEST_HTML_OUTPUT' not in env:
             bin_dir = os.path.join(
-                os.path.abspath(proj.baseDir), 'build', 'html')
+                os.path.abspath(proj.baseDir),
+                'build.{CMTCONFIG}'.format(**env), 'html')
             env['GAUDI_QMTEST_HTML_OUTPUT'] = bin_dir
         kwargs['env'] = env
         return self._make('test', proj, **kwargs)
diff --git a/python/LbNightlyTools/MergeRequestBuilds.py b/python/LbNightlyTools/MergeRequestBuilds.py
index a2279270..d5a0ce78 100644
--- a/python/LbNightlyTools/MergeRequestBuilds.py
+++ b/python/LbNightlyTools/MergeRequestBuilds.py
@@ -65,13 +65,10 @@ def mr_getrefdate(mrs):
     for mr, project in mrs:
         # Parse the date of the time when the MR was committed
         ref_sha1 = mr.attributes['diff_refs']['start_sha']
-        committed_date = project.commits.get(
-            ref_sha1).attributes['committed_date']
-        committed_date = committed_date.rstrip('Z')
-        if '+' in committed_date:
-            committed_date = committed_date.split('+', 1)[0]
         mr_ref_dates.append(
-            datetime.datetime.strptime(committed_date, "%Y-%m-%dT%H:%M:%S.%f"))
+            datetime.datetime.strptime(
+                project.commits.get(ref_sha1).attributes['committed_date'],
+                "%Y-%m-%dT%H:%M:%S.%fZ"))
 
     # Return the latest time corresponsing to the last MR branched from it's project
     return sorted(mr_ref_dates)[-1].isoformat()
diff --git a/python/LbNightlyTools/Scripts/Common.py b/python/LbNightlyTools/Scripts/Common.py
index b1b72d35..f9c3d781 100755
--- a/python/LbNightlyTools/Scripts/Common.py
+++ b/python/LbNightlyTools/Scripts/Common.py
@@ -101,7 +101,7 @@ def addBasicOptions(parser):
 
     parser.set_defaults(
         build_id='{slot}',
-        slot_build_id=None,
+        slot_build_id=0,
         artifacts_dir='artifacts',
         summary_prefix='')
     return parser
@@ -720,7 +720,7 @@ class BaseScript(PlainScript):
             if self.summary_base:
                 ensureDirs([self._summaryDir()])
 
-        if opts.slot_build_id is not None:
+        if opts.slot_build_id:
             self.slot.build_id = self.options.slot_build_id
         elif not self.slot.build_id:
             self.slot.build_id = int(os.environ.get('slot_build_id', 0))
diff --git a/python/LbNightlyTools/Scripts/Test.py b/python/LbNightlyTools/Scripts/Test.py
index 7e38db3a..8498597b 100644
--- a/python/LbNightlyTools/Scripts/Test.py
+++ b/python/LbNightlyTools/Scripts/Test.py
@@ -199,17 +199,14 @@ class Script(BaseScript):
 
             for proj, _ in self.slot.testGen(
                     projects=opts.projects, before=before, jobs=opts.jobs):
-                html_src = self._buildDir(proj, 'build', 'html')
                 # Always use the most recent CTEST_CONVERTER, if available
-                # and we are in a CMake project
-                if os.path.exists(self._buildDir(proj, 'build', 'Testing')):
-                    if CTEST_CONVERTER:
-                        if os.path.exists(html_src):
-                            shutil.rmtree(html_src)
-                        call([CTEST_CONVERTER],
-                             cwd=self._buildDir(proj, 'build'))
-                    summary_json = os.path.join(html_src, 'summary.json')
-                    fixFailureCauses(summary_json)
+                html_src = self._buildDir(proj, 'build', 'html')
+                if CTEST_CONVERTER:
+                    if os.path.exists(html_src):
+                        shutil.rmtree(html_src)
+                    call([CTEST_CONVERTER], cwd=self._buildDir(proj, 'build'))
+                summary_json = os.path.join(html_src, 'summary.json')
+                fixFailureCauses(summary_json)
 
                 # update annotations with cpuinfo summary
                 try:
diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index d7ca6660..15e46713 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -23,7 +23,6 @@ import subprocess
 import time
 import urllib2
 import re
-from urllib.request import urlopen, Request
 from datetime import datetime
 
 import gitlab
@@ -375,182 +374,6 @@ def recursive_update(dest, data):
     return dest
 
 
-class ArtifactsRepository(object):
-    '''
-    Class representing artifacts repository.
-    '''
-
-    def __init__(self, uri):
-        '''
-        Initialises the repository based on its URI
-        '''
-        self.uri = uri
-
-    def pull(self, artifacts_name):
-        '''
-        Returns a file object in the read mode
-
-        Arguments:
-        artifacts_name: name of the artifacts file
-
-        Returns: File object
-        '''
-        raise NotImplementedError(
-            'Should be implemented in the inheriting class')
-
-    def push(self, artifacts_name):
-        '''
-        Pushes artifacts to the repository
-
-        Arguments:
-        artifacts_name: name of the artifcats file
-
-        Returns: True if the artifacts have been pushed, False otherwise
-        '''
-        raise NotImplementedError(
-            'Should be implemented in the inheriting class')
-
-    def exist(self, artifacts_name):
-        '''
-        Checks if artifacts exist
-
-        Arguments:
-        artifacts_name: name of the artifcats file
-
-        Returns: True if the artifacts exist, False otherwise
-        '''
-        raise NotImplementedError(
-            'Should be implemented in the inheriting class')
-
-
-class FileRepository(ArtifactsRepository):
-    '''
-    Class defining repository in the local file system.
-    '''
-
-    def pull(self, artifacts_name):
-        artifact = open(os.path.join(self.uri, artifacts_name), 'r')
-        return artifact
-
-    def push(self, artifacts_name):
-        try:
-            subprocess.call(['cp', artifacts_name, self.uri])
-        except (subprocess.CalledProcessError, OSError) as ex:
-            logging.debug(
-                'Pushing artifacts to the repository failed: {}'.format(
-                    str(ex)))
-            return False
-        return True
-
-    def exist(self, artifacts_name):
-        return os.path.exists(os.path.join(self.uri, artifacts_name))
-
-
-class EosRepository(ArtifactsRepository):
-    '''
-    Class defining repository on EOS.
-    '''
-
-    def pull(self, artifacts_name):
-        try:
-            subprocess.check_call([
-                'xrdcp',
-                os.path.join(self.uri, artifacts_name), artifacts_name
-            ], )
-        except (subprocess.CalledProcessError, OSError) as ex:
-            logging.debug('Pulling failed: {}'.format(str(ex)))
-            return None
-        artifact = open(artifacts_name, 'r')
-        return artifact
-
-    def push(self, artifacts_name):
-        try:
-            subprocess.check_call([
-                'xrdcp', artifacts_name,
-                os.path.join(self.uri, os.path.basename(artifacts_name))
-            ])
-        except (subprocess.CalledProcessError, OSError) as ex:
-            logging.debug(
-                'Pushing artifacts to the repository failed: {}'.format(
-                    str(ex)))
-            return False
-        return True
-
-    def exist(self, artifacts_name):
-        return os.path.exists(os.path.join(self.uri, artifacts_name))
-
-
-class HttpRepository(ArtifactsRepository):
-    '''
-    Class defining repository through HTTP request methods.
-    '''
-
-    def pull(self, artifacts_name):
-        with urlopen(os.path.join(self.uri, artifacts_name)) as response, open(
-                artifacts_name, 'wb') as out_file:
-            shutil.copyfileobj(response, out_file)
-        return out_file
-
-    def push(self, artifacts_name, auth=('', '')):
-        req = Request(
-            url=os.path.join(self.uri, artifacts_name),
-            data=open(artifacts_name, 'rb').read(),
-            method='PUT')
-        with urlopen(req) as f:
-            pass
-        if f.status != 201:
-            logging.debug('Pushing artifacts to the repository failed')
-            return False
-        return True
-
-    def exist(self, artifacts_name):
-        from urllib.error import HTTPError
-        try:
-            urlopen(os.path.join(self.uri, artifacts_name)).close()
-            return True
-        except HTTPError:
-            return False
-
-
-def repo_type(uri):
-    '''
-    Returns type of repository based on uri provided in the argument.
-    It may return 'eos', 'file', 'http' or None
-    '''
-    try:
-        from urlparse import urlparse
-    except ImportError():
-        from urllib.parse import urlparse
-
-    result = urlparse(uri)
-    if not result.scheme or result.scheme == 'file':
-        if result.path.startswith('/eos/'):
-            return 'eos'
-        else:
-            return 'file'
-    elif result.scheme == 'root':
-        return 'eos'
-    elif result.scheme in ('http', 'https'):
-        return 'http'
-    return None
-
-
-def repo_factory(url):
-    '''
-    Factory function creating artifacts repository
-    object based on the URI provided in the argument.
-    '''
-    _repos = {
-        'file': FileRepository,
-        'eos': EosRepository,
-        'http': HttpRepository,
-    }
-    repo = _repos.get(repo_type(url))
-    if not repo:
-        raise ValueError('Unrecognised repository type: {}'.format(url))
-    return repo(url)
-
-
 class Dashboard(object):
     '''
     Wrapper for the CouchDB-based dashboard.
diff --git a/setup.py b/setup.py
index 655e5e12..219cc126 100644
--- a/setup.py
+++ b/setup.py
@@ -135,7 +135,7 @@ setup(
         "LbCommon>=0.0.7",
         "LbSoftConfDb2Clients",
         "LbDevTools",
-        "python-gitlab" + ('<2' if version_info < (3, 0) else ''),
+        "python-gitlab",
         "pika>=1.0.0",
         "CouchDB",
         "tabulate",
-- 
GitLab


From f22a9024bed64ec362118846ade7bd12d147e1b1 Mon Sep 17 00:00:00 2001
From: Marco Clemencic <marco.clemencic@cern.ch>
Date: Tue, 28 Jan 2020 17:53:18 +0100
Subject: [PATCH 15/34] Allow override of --slot-build-id with 0

---
 python/LbNightlyTools/Scripts/Common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/LbNightlyTools/Scripts/Common.py b/python/LbNightlyTools/Scripts/Common.py
index f9c3d781..b1b72d35 100755
--- a/python/LbNightlyTools/Scripts/Common.py
+++ b/python/LbNightlyTools/Scripts/Common.py
@@ -101,7 +101,7 @@ def addBasicOptions(parser):
 
     parser.set_defaults(
         build_id='{slot}',
-        slot_build_id=0,
+        slot_build_id=None,
         artifacts_dir='artifacts',
         summary_prefix='')
     return parser
@@ -720,7 +720,7 @@ class BaseScript(PlainScript):
             if self.summary_base:
                 ensureDirs([self._summaryDir()])
 
-        if opts.slot_build_id:
+        if opts.slot_build_id is not None:
             self.slot.build_id = self.options.slot_build_id
         elif not self.slot.build_id:
             self.slot.build_id = int(os.environ.get('slot_build_id', 0))
-- 
GitLab


From 1fac548d80dfc1fe2828da17079055fa91fdd258 Mon Sep 17 00:00:00 2001
From: Marco Clemencic <marco.clemencic@cern.ch>
Date: Tue, 28 Jan 2020 18:01:47 +0100
Subject: [PATCH 16/34] Constrain python-gitlab version on Python2

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 219cc126..655e5e12 100644
--- a/setup.py
+++ b/setup.py
@@ -135,7 +135,7 @@ setup(
         "LbCommon>=0.0.7",
         "LbSoftConfDb2Clients",
         "LbDevTools",
-        "python-gitlab",
+        "python-gitlab" + ('<2' if version_info < (3, 0) else ''),
         "pika>=1.0.0",
         "CouchDB",
         "tabulate",
-- 
GitLab


From 582cdb5437a9ed3fbf2656b7112832e04c55c273 Mon Sep 17 00:00:00 2001
From: Marco Clemencic <marco.clemencic@cern.ch>
Date: Wed, 29 Jan 2020 22:27:14 +0100
Subject: [PATCH 17/34] Strip timezone part from committed_date before parsing

---
 python/LbNightlyTools/MergeRequestBuilds.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/python/LbNightlyTools/MergeRequestBuilds.py b/python/LbNightlyTools/MergeRequestBuilds.py
index d5a0ce78..a2279270 100644
--- a/python/LbNightlyTools/MergeRequestBuilds.py
+++ b/python/LbNightlyTools/MergeRequestBuilds.py
@@ -65,10 +65,13 @@ def mr_getrefdate(mrs):
     for mr, project in mrs:
         # Parse the date of the time when the MR was committed
         ref_sha1 = mr.attributes['diff_refs']['start_sha']
+        committed_date = project.commits.get(
+            ref_sha1).attributes['committed_date']
+        committed_date = committed_date.rstrip('Z')
+        if '+' in committed_date:
+            committed_date = committed_date.split('+', 1)[0]
         mr_ref_dates.append(
-            datetime.datetime.strptime(
-                project.commits.get(ref_sha1).attributes['committed_date'],
-                "%Y-%m-%dT%H:%M:%S.%fZ"))
+            datetime.datetime.strptime(committed_date, "%Y-%m-%dT%H:%M:%S.%f"))
 
     # Return the latest time corresponsing to the last MR branched from it's project
     return sorted(mr_ref_dates)[-1].isoformat()
-- 
GitLab


From 327817b9d7c2847539024d8688cd85b36da6ed19 Mon Sep 17 00:00:00 2001
From: Marco Clemencic <marco.clemencic@cern.ch>
Date: Fri, 31 Jan 2020 10:10:09 +0100
Subject: [PATCH 18/34] Do not try to run CTEST_CONVERTER for non-CMake builds

---
 python/LbNightlyTools/Scripts/Test.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/python/LbNightlyTools/Scripts/Test.py b/python/LbNightlyTools/Scripts/Test.py
index 8498597b..7e38db3a 100644
--- a/python/LbNightlyTools/Scripts/Test.py
+++ b/python/LbNightlyTools/Scripts/Test.py
@@ -199,14 +199,17 @@ class Script(BaseScript):
 
             for proj, _ in self.slot.testGen(
                     projects=opts.projects, before=before, jobs=opts.jobs):
-                # Always use the most recent CTEST_CONVERTER, if available
                 html_src = self._buildDir(proj, 'build', 'html')
-                if CTEST_CONVERTER:
-                    if os.path.exists(html_src):
-                        shutil.rmtree(html_src)
-                    call([CTEST_CONVERTER], cwd=self._buildDir(proj, 'build'))
-                summary_json = os.path.join(html_src, 'summary.json')
-                fixFailureCauses(summary_json)
+                # Always use the most recent CTEST_CONVERTER, if available
+                # and we are in a CMake project
+                if os.path.exists(self._buildDir(proj, 'build', 'Testing')):
+                    if CTEST_CONVERTER:
+                        if os.path.exists(html_src):
+                            shutil.rmtree(html_src)
+                        call([CTEST_CONVERTER],
+                             cwd=self._buildDir(proj, 'build'))
+                    summary_json = os.path.join(html_src, 'summary.json')
+                    fixFailureCauses(summary_json)
 
                 # update annotations with cpuinfo summary
                 try:
-- 
GitLab


From cc6728e959f0d69fe67c0cecc6925a2649b64847 Mon Sep 17 00:00:00 2001
From: Marco Clemencic <marco.clemencic@cern.ch>
Date: Fri, 31 Jan 2020 10:25:09 +0100
Subject: [PATCH 19/34] Make CMT test output location consistent with CMake

---
 python/LbNightlyTools/BuildMethods.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/python/LbNightlyTools/BuildMethods.py b/python/LbNightlyTools/BuildMethods.py
index b4756cab..6986c1b8 100644
--- a/python/LbNightlyTools/BuildMethods.py
+++ b/python/LbNightlyTools/BuildMethods.py
@@ -210,8 +210,7 @@ class cmt(make):
                                 or os.environ.get('BINARY_TAG'))
         if 'GAUDI_QMTEST_HTML_OUTPUT' not in env:
             bin_dir = os.path.join(
-                os.path.abspath(proj.baseDir),
-                'build.{CMTCONFIG}'.format(**env), 'html')
+                os.path.abspath(proj.baseDir), 'build', 'html')
             env['GAUDI_QMTEST_HTML_OUTPUT'] = bin_dir
         kwargs['env'] = env
         return self._make('test', proj, **kwargs)
-- 
GitLab


From 72863e77b1c2184f9ce9827f300a701c5d2641a3 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Wed, 19 Feb 2020 17:45:35 +0100
Subject: [PATCH 20/34] add class representing ArtifactsRepository

---
 python/LbNightlyTools/Utils.py | 180 +++++++++++++++++++++++++++++++++
 1 file changed, 180 insertions(+)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index 15e46713..f6002f66 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -23,6 +23,7 @@ import subprocess
 import time
 import urllib2
 import re
+import requests
 from datetime import datetime
 
 import gitlab
@@ -374,6 +375,185 @@ def recursive_update(dest, data):
     return dest
 
 
+def get_repo_type(uri):
+    '''
+    Returns type of repository based on uri provided in the argument.
+    It may return 'eos', 'file', 'http' or None
+    '''
+    try:
+        from urlparse import urlparse
+    except ImportError():
+        from urllib.parse import urlparse
+
+    result = urlparse(uri)
+    if result.scheme == '':
+        if result.path.startswith('/eos/'):
+            return 'eos'
+        else:
+            return 'file'
+    elif result.scheme == 'root':
+        return 'eos'
+    elif result.scheme == 'http':
+        return 'http'
+    return None
+
+
+class ArtifactsRepository(object):
+    '''
+    Class representing artifacts repository.
+    '''
+
+    def __init__(self, uri):
+        '''
+        Initialises the repository based on its URI
+        '''
+        self.uri = uri
+
+    def pull_artifacts(self, artifacts_name, dest):
+        '''
+        Gets artifacts from the repository and unzips them
+
+        Arguments:
+        artifacts_name: name of the artifacts file
+
+        Keyword arguments:
+        dest: to specify destination directory for unzipping
+
+        Returns: True if the artifacts have been extracted, False otherwise
+        '''
+        raise NotImplementedError('Should be implemented in the inheriting class')
+
+    def push_artifacts(self, artifacts_name):
+        '''
+        Pushes artifacts to the repository
+
+        Arguments:
+        artifacts_name: name of the artifcats file
+
+        Returns: True if the artifacts have been pushed, False otherwise
+        '''
+        raise NotImplementedError('Should be implemented in the inheriting class')
+
+    def artifacts_exist(self, artifacts_name):
+        '''
+        Checks if artifacts exist
+
+        Arguments:
+        artifacts_name: name of the artifcats file
+
+        Returns: True if the artifacts exist, False otherwise
+        '''
+        raise NotImplementedError('Should be implemented in the inheriting class')
+
+
+def repo_factory(uri):
+    '''
+    Factory function creating artifacts repository
+    object based on the URI provided in the argument.
+    '''
+
+    class FileRepository(ArtifactsRepository):
+        '''
+        Class defining repository in the local file system.
+        '''
+
+        def pull_artifacts(self, artifacts_name, dest):
+            try:
+                subprocess.check_call(
+                    ['unzip', '-q', os.path.join(self.uri, artifacts_name)], cwd=dest)
+            except (subprocess.CalledProcessError, OSError) as ex:
+                logging.debug('Unzipping failed: {}'.format(str(ex)))
+                return False
+            return True
+
+        def push_artifacts(self, artifacts_name):
+            try:
+                logging.debug('pushing {} to {}'.format(artifacts_name, self.uri))
+                subprocess.call(['cp', artifacts_name, self.uri])
+            except (subprocess.CalledProcessError, OSError) as ex:
+                logging.debug('Pushing artifacts to the repository failed: {}'
+                            .format(str(ex)))
+                return False
+            return True
+
+        def artifacts_exist(self, artifacts_name):
+            return os.path.exists(os.path.join(self.uri, artifacts_name))
+
+    class EosRepository(ArtifactsRepository):
+        '''
+        Class defining repository on EOS.
+        '''
+
+        def pull_artifacts(self, artifacts_name, dest):
+            try:
+                subprocess.check_call(
+                    ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name],
+                    cwd=dest)
+                subprocess.check_call(['unzip', '-q', artifacts_name],
+                                  cwd=dest)
+            except (subprocess.CalledProcessError, OSError) as ex:
+                logging.debug('Pulling failed: {}'.format(str(ex)))
+                return False
+            return True
+
+        def push_artifacts(self, artifacts_name):
+            try:
+                subprocess.check_call([
+                    'xrdcp', artifacts_name, os.path.join(
+                                                self.uri,
+                                                os.path.basename(artifacts_name))
+                ])
+            except (subprocess.CalledProcessError, OSError) as ex:
+                logging.debug('Pushing artifacts to the repository failed: {}'
+                            .format(str(ex)))
+                return False
+            return True
+
+        def artifacts_exist(self, artifacts_name):
+            return os.path.exists(os.path.join(self.uri, artifacts_name))
+
+    class HttpRepository(ArtifactsRepository):
+        '''
+        Class defining repository through HTTP request methods.
+        '''
+
+        def pull_artifacts(self, artifacts_name, dest):
+            r = requests.get(os.path.join(self.uri, artifacts_name))
+            if r.status_code != 200:
+                logging.debug('Resource does not exist')
+                return False
+            with open(os.path.join(dest, artifacts_name), 'wb') as f:
+                f.write(r.content)
+            subprocess.check_call(['unzip','-q', os.path.join(dest, artifacts_name)],
+                                  cwd=dest)
+            return True
+
+        def push_artifacts(self, artifacts_name, auth=('', '')):
+            r = requests.put(os.path.join(self.uri, artifacts_name),
+                             data=open(artifacts_name, 'rb').read(),
+                             headers={'content-type': 'application/zip'},
+                             auth=auth)
+            if r.status_code != 201:
+                logging.debug('Pushing artifacts to the repository failed')
+                return False
+            return True
+
+        def artifacts_exist(self, artifacts_name):
+            r = requests.get(os.path.join(self.uri, artifacts_name))
+            if r.status_code == 200:
+                return True
+            return False
+
+    if get_repo_type(uri) == 'file':
+        return FileRepository(uri)
+    elif get_repo_type(uri) == 'eos':
+        return EosRepository(uri)
+    elif get_repo_type(uri) == 'http':
+        return HttpRepository(uri)
+    else:
+        raise ValueError("Unrecognised repository type for: {}".format(uri))
+
+
 class Dashboard(object):
     '''
     Wrapper for the CouchDB-based dashboard.
-- 
GitLab


From 24a777e6d90f47361184c56cdacb60192c072914 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 10:45:38 +0100
Subject: [PATCH 21/34] use urllib instead of requests

---
 python/LbNightlyTools/Utils.py | 47 +++++++++++++++++-----------------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index f6002f66..4625155d 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -23,7 +23,7 @@ import subprocess
 import time
 import urllib2
 import re
-import requests
+from urllib.request import urlopen, Request
 from datetime import datetime
 
 import gitlab
@@ -517,31 +517,30 @@ def repo_factory(uri):
         Class defining repository through HTTP request methods.
         '''
 
-        def pull_artifacts(self, artifacts_name, dest):
-            r = requests.get(os.path.join(self.uri, artifacts_name))
-            if r.status_code != 200:
-                logging.debug('Resource does not exist')
-                return False
-            with open(os.path.join(dest, artifacts_name), 'wb') as f:
-                f.write(r.content)
-            subprocess.check_call(['unzip','-q', os.path.join(dest, artifacts_name)],
-                                  cwd=dest)
-            return True
+    def pull(self, artifacts_name):
+        with urlopen(
+                os.path.join(self.uri, artifacts_name)) as response, open(artifacts_name, 'wb') as out_file:
+            shutil.copyfileobj(response, out_file)
+        return out_file
+
+    def push(self, artifacts_name, auth=('', '')):
+        req = Request(
+                url=os.path.join(self.uri, artifacts_name),
+                data=open(artifacts_name, 'rb').read(),
+                method='PUT')
+        with urlopen(req) as f:
+            pass
+        if f.status != 201:
+            logging.debug('Pushing artifacts to the repository failed')
+            return False
+        return True
 
-        def push_artifacts(self, artifacts_name, auth=('', '')):
-            r = requests.put(os.path.join(self.uri, artifacts_name),
-                             data=open(artifacts_name, 'rb').read(),
-                             headers={'content-type': 'application/zip'},
-                             auth=auth)
-            if r.status_code != 201:
-                logging.debug('Pushing artifacts to the repository failed')
-                return False
+    def exist(self, artifacts_name):
+        from urllib.error import HTTPError
+        try:
+            urlopen(os.path.join(self.uri, artifacts_name)).close()
             return True
-
-        def artifacts_exist(self, artifacts_name):
-            r = requests.get(os.path.join(self.uri, artifacts_name))
-            if r.status_code == 200:
-                return True
+        except HTTPError:
             return False
 
     if get_repo_type(uri) == 'file':
-- 
GitLab


From 1236057df4efafceb58a1c23bd878ac83551a695 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 10:46:15 +0100
Subject: [PATCH 22/34] refactor factory method

---
 python/LbNightlyTools/Utils.py | 69 +++++++++++++++++++---------------
 1 file changed, 38 insertions(+), 31 deletions(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index 4625155d..d93d463b 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -375,29 +375,6 @@ def recursive_update(dest, data):
     return dest
 
 
-def get_repo_type(uri):
-    '''
-    Returns type of repository based on uri provided in the argument.
-    It may return 'eos', 'file', 'http' or None
-    '''
-    try:
-        from urlparse import urlparse
-    except ImportError():
-        from urllib.parse import urlparse
-
-    result = urlparse(uri)
-    if result.scheme == '':
-        if result.path.startswith('/eos/'):
-            return 'eos'
-        else:
-            return 'file'
-    elif result.scheme == 'root':
-        return 'eos'
-    elif result.scheme == 'http':
-        return 'http'
-    return None
-
-
 class ArtifactsRepository(object):
     '''
     Class representing artifacts repository.
@@ -543,14 +520,44 @@ def repo_factory(uri):
         except HTTPError:
             return False
 
-    if get_repo_type(uri) == 'file':
-        return FileRepository(uri)
-    elif get_repo_type(uri) == 'eos':
-        return EosRepository(uri)
-    elif get_repo_type(uri) == 'http':
-        return HttpRepository(uri)
-    else:
-        raise ValueError("Unrecognised repository type for: {}".format(uri))
+
+def repo_type(uri):
+    '''
+    Returns type of repository based on uri provided in the argument.
+    It may return 'eos', 'file', 'http' or None
+    '''
+    try:
+        from urlparse import urlparse
+    except ImportError():
+        from urllib.parse import urlparse
+
+    result = urlparse(uri)
+    if result.scheme == '':
+        if result.path.startswith('/eos/'):
+            return 'eos'
+        else:
+            return 'file'
+    elif result.scheme == 'root':
+        return 'eos'
+    elif result.scheme == 'http':
+        return 'http'
+    return None
+
+
+def repo_factory(url):
+    '''
+    Factory function creating artifacts repository
+    object based on the URI provided in the argument.
+    '''
+    _repos = {
+        'file': FileRepository,
+        'eos': EosRepository,
+        'http': HttpRepository,
+    }
+    repo = _repos.get(repo_type(url))
+    if not repo:
+        raise ValueError('Unrecognised repository type: {}'.format(url))
+    return repo(url)
 
 
 class Dashboard(object):
-- 
GitLab


From dc03c12c447edbe37b2fe473d1ac6005b6563cac Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 10:47:12 +0100
Subject: [PATCH 23/34] remove from methods name redundant artifacts word

---
 python/LbNightlyTools/Utils.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index d93d463b..b56da743 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -386,21 +386,18 @@ class ArtifactsRepository(object):
         '''
         self.uri = uri
 
-    def pull_artifacts(self, artifacts_name, dest):
+    def pull(self, artifacts_name):
         '''
-        Gets artifacts from the repository and unzips them
+        Returns a file object in the read mode
 
         Arguments:
         artifacts_name: name of the artifacts file
 
-        Keyword arguments:
-        dest: to specify destination directory for unzipping
-
-        Returns: True if the artifacts have been extracted, False otherwise
+        Returns: File object
         '''
         raise NotImplementedError('Should be implemented in the inheriting class')
 
-    def push_artifacts(self, artifacts_name):
+    def push(self, artifacts_name):
         '''
         Pushes artifacts to the repository
 
@@ -411,7 +408,7 @@ class ArtifactsRepository(object):
         '''
         raise NotImplementedError('Should be implemented in the inheriting class')
 
-    def artifacts_exist(self, artifacts_name):
+    def exist(self, artifacts_name):
         '''
         Checks if artifacts exist
 
-- 
GitLab


From 17c8c83e1b546b1a28f851bb107b4689c3c586fe Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 10:47:53 +0100
Subject: [PATCH 24/34] dont unzip in pull

---
 python/LbNightlyTools/Utils.py | 107 +++++++++++++++------------------
 1 file changed, 48 insertions(+), 59 deletions(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index b56da743..609f14af 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -420,76 +420,65 @@ class ArtifactsRepository(object):
         raise NotImplementedError('Should be implemented in the inheriting class')
 
 
-def repo_factory(uri):
+class FileRepository(ArtifactsRepository):
     '''
-    Factory function creating artifacts repository
-    object based on the URI provided in the argument.
+    Class defining repository in the local file system.
     '''
 
-    class FileRepository(ArtifactsRepository):
-        '''
-        Class defining repository in the local file system.
-        '''
+    def pull(self, artifacts_name):
+        artifact = open(os.path.join(self.uri, artifacts_name), 'r')
+        return artifact
 
-        def pull_artifacts(self, artifacts_name, dest):
-            try:
-                subprocess.check_call(
-                    ['unzip', '-q', os.path.join(self.uri, artifacts_name)], cwd=dest)
-            except (subprocess.CalledProcessError, OSError) as ex:
-                logging.debug('Unzipping failed: {}'.format(str(ex)))
-                return False
-            return True
+    def push(self, artifacts_name):
+        try:
+            subprocess.call(['cp', artifacts_name, self.uri])
+        except (subprocess.CalledProcessError, OSError) as ex:
+            logging.debug('Pushing artifacts to the repository failed: {}'
+                        .format(str(ex)))
+            return False
+        return True
 
-        def push_artifacts(self, artifacts_name):
-            try:
-                logging.debug('pushing {} to {}'.format(artifacts_name, self.uri))
-                subprocess.call(['cp', artifacts_name, self.uri])
-            except (subprocess.CalledProcessError, OSError) as ex:
-                logging.debug('Pushing artifacts to the repository failed: {}'
-                            .format(str(ex)))
-                return False
-            return True
+    def exist(self, artifacts_name):
+        return os.path.exists(os.path.join(self.uri, artifacts_name))
 
-        def artifacts_exist(self, artifacts_name):
-            return os.path.exists(os.path.join(self.uri, artifacts_name))
 
-    class EosRepository(ArtifactsRepository):
-        '''
-        Class defining repository on EOS.
-        '''
+class EosRepository(ArtifactsRepository):
+    '''
+    Class defining repository on EOS.
+    '''
 
-        def pull_artifacts(self, artifacts_name, dest):
-            try:
-                subprocess.check_call(
-                    ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name],
-                    cwd=dest)
-                subprocess.check_call(['unzip', '-q', artifacts_name],
-                                  cwd=dest)
-            except (subprocess.CalledProcessError, OSError) as ex:
-                logging.debug('Pulling failed: {}'.format(str(ex)))
-                return False
-            return True
+    def pull(self, artifacts_name):
+        try:
+            subprocess.check_call(
+                ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name],
+            )
+        except (subprocess.CalledProcessError, OSError) as ex:
+            logging.debug('Pulling failed: {}'.format(str(ex)))
+            return None
+        artifact = open(artifacts_name, 'r')
+        return artifact
 
-        def push_artifacts(self, artifacts_name):
-            try:
-                subprocess.check_call([
-                    'xrdcp', artifacts_name, os.path.join(
-                                                self.uri,
-                                                os.path.basename(artifacts_name))
-                ])
-            except (subprocess.CalledProcessError, OSError) as ex:
-                logging.debug('Pushing artifacts to the repository failed: {}'
-                            .format(str(ex)))
-                return False
-            return True
+    def push(self, artifacts_name):
+        try:
+            subprocess.check_call([
+                'xrdcp', artifacts_name, os.path.join(
+                                            self.uri,
+                                            os.path.basename(artifacts_name))
+            ])
+        except (subprocess.CalledProcessError, OSError) as ex:
+            logging.debug('Pushing artifacts to the repository failed: {}'
+                        .format(str(ex)))
+            return False
+        return True
 
-        def artifacts_exist(self, artifacts_name):
-            return os.path.exists(os.path.join(self.uri, artifacts_name))
+    def exist(self, artifacts_name):
+        return os.path.exists(os.path.join(self.uri, artifacts_name))
 
-    class HttpRepository(ArtifactsRepository):
-        '''
-        Class defining repository through HTTP request methods.
-        '''
+
+class HttpRepository(ArtifactsRepository):
+    '''
+    Class defining repository through HTTP request methods.
+    '''
 
     def pull(self, artifacts_name):
         with urlopen(
-- 
GitLab


From 2c8ab7d2938aec37f3e9de91ab3e3457869725df Mon Sep 17 00:00:00 2001
From: Maciej Pawel Szymanski <maciej.szymanski@cern.ch>
Date: Thu, 20 Feb 2020 08:35:32 +0000
Subject: [PATCH 25/34] Apply suggestion to python/LbNightlyTools/Utils.py

---
 python/LbNightlyTools/Utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index 609f14af..f002125b 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -525,7 +525,7 @@ def repo_type(uri):
             return 'file'
     elif result.scheme == 'root':
         return 'eos'
-    elif result.scheme == 'http':
+    elif result.scheme in ('http', 'https'):
         return 'http'
     return None
 
-- 
GitLab


From c87e19e35788259a8112d847ae79b18b575c576a Mon Sep 17 00:00:00 2001
From: Maciej Pawel Szymanski <maciej.szymanski@cern.ch>
Date: Thu, 20 Feb 2020 08:37:14 +0000
Subject: [PATCH 26/34] Apply suggestion to python/LbNightlyTools/Utils.py

---
 python/LbNightlyTools/Utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index f002125b..38102ac6 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -518,7 +518,7 @@ def repo_type(uri):
         from urllib.parse import urlparse
 
     result = urlparse(uri)
-    if result.scheme == '':
+    if not result.scheme or result.scheme == 'file':
         if result.path.startswith('/eos/'):
             return 'eos'
         else:
-- 
GitLab


From 0a7f98ab93c18e9bd705100fa229c5752d57f2e3 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 10:50:47 +0100
Subject: [PATCH 27/34] fix function name

---
 python/LbNightlyTools/Utils.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index 38102ac6..daf8a426 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -375,6 +375,29 @@ def recursive_update(dest, data):
     return dest
 
 
+def repo_type(uri):
+    '''
+    Returns type of repository based on uri provided in the argument.
+    It may return 'eos', 'file', 'http' or None
+    '''
+    try:
+        from urlparse import urlparse
+    except ImportError():
+        from urllib.parse import urlparse
+
+    result = urlparse(uri)
+    if not result.scheme or result.scheme == 'file':
+        if result.path.startswith('/eos/'):
+            return 'eos'
+        else:
+            return 'file'
+    elif result.scheme == 'root':
+        return 'eos'
+    elif result.scheme in ('http', 'https'):
+        return 'http'
+    return None
+
+
 class ArtifactsRepository(object):
     '''
     Class representing artifacts repository.
-- 
GitLab


From 668cfc8946f4ca9cef71dd079a415a0309ad67ac Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 11:13:28 +0100
Subject: [PATCH 28/34] remove redundant definition

---
 python/LbNightlyTools/Utils.py | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index daf8a426..38102ac6 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -375,29 +375,6 @@ def recursive_update(dest, data):
     return dest
 
 
-def repo_type(uri):
-    '''
-    Returns type of repository based on uri provided in the argument.
-    It may return 'eos', 'file', 'http' or None
-    '''
-    try:
-        from urlparse import urlparse
-    except ImportError():
-        from urllib.parse import urlparse
-
-    result = urlparse(uri)
-    if not result.scheme or result.scheme == 'file':
-        if result.path.startswith('/eos/'):
-            return 'eos'
-        else:
-            return 'file'
-    elif result.scheme == 'root':
-        return 'eos'
-    elif result.scheme in ('http', 'https'):
-        return 'http'
-    return None
-
-
 class ArtifactsRepository(object):
     '''
     Class representing artifacts repository.
-- 
GitLab


From 3e3b756881a579246cde072bdd5fd186f92010f3 Mon Sep 17 00:00:00 2001
From: Gitlab CI <noreply@cern.ch>
Date: Fri, 21 Feb 2020 10:18:45 +0000
Subject: [PATCH 29/34] Fixed formatting

patch generated by https://gitlab.cern.ch/lhcb-core/LbNightlyTools/-/jobs/7285194
---
 python/LbNightlyTools/Utils.py | 41 +++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py
index 38102ac6..d7ca6660 100644
--- a/python/LbNightlyTools/Utils.py
+++ b/python/LbNightlyTools/Utils.py
@@ -395,7 +395,8 @@ class ArtifactsRepository(object):
 
         Returns: File object
         '''
-        raise NotImplementedError('Should be implemented in the inheriting class')
+        raise NotImplementedError(
+            'Should be implemented in the inheriting class')
 
     def push(self, artifacts_name):
         '''
@@ -406,7 +407,8 @@ class ArtifactsRepository(object):
 
         Returns: True if the artifacts have been pushed, False otherwise
         '''
-        raise NotImplementedError('Should be implemented in the inheriting class')
+        raise NotImplementedError(
+            'Should be implemented in the inheriting class')
 
     def exist(self, artifacts_name):
         '''
@@ -417,7 +419,8 @@ class ArtifactsRepository(object):
 
         Returns: True if the artifacts exist, False otherwise
         '''
-        raise NotImplementedError('Should be implemented in the inheriting class')
+        raise NotImplementedError(
+            'Should be implemented in the inheriting class')
 
 
 class FileRepository(ArtifactsRepository):
@@ -433,8 +436,9 @@ class FileRepository(ArtifactsRepository):
         try:
             subprocess.call(['cp', artifacts_name, self.uri])
         except (subprocess.CalledProcessError, OSError) as ex:
-            logging.debug('Pushing artifacts to the repository failed: {}'
-                        .format(str(ex)))
+            logging.debug(
+                'Pushing artifacts to the repository failed: {}'.format(
+                    str(ex)))
             return False
         return True
 
@@ -449,9 +453,10 @@ class EosRepository(ArtifactsRepository):
 
     def pull(self, artifacts_name):
         try:
-            subprocess.check_call(
-                ['xrdcp', os.path.join(self.uri, artifacts_name), artifacts_name],
-            )
+            subprocess.check_call([
+                'xrdcp',
+                os.path.join(self.uri, artifacts_name), artifacts_name
+            ], )
         except (subprocess.CalledProcessError, OSError) as ex:
             logging.debug('Pulling failed: {}'.format(str(ex)))
             return None
@@ -461,13 +466,13 @@ class EosRepository(ArtifactsRepository):
     def push(self, artifacts_name):
         try:
             subprocess.check_call([
-                'xrdcp', artifacts_name, os.path.join(
-                                            self.uri,
-                                            os.path.basename(artifacts_name))
+                'xrdcp', artifacts_name,
+                os.path.join(self.uri, os.path.basename(artifacts_name))
             ])
         except (subprocess.CalledProcessError, OSError) as ex:
-            logging.debug('Pushing artifacts to the repository failed: {}'
-                        .format(str(ex)))
+            logging.debug(
+                'Pushing artifacts to the repository failed: {}'.format(
+                    str(ex)))
             return False
         return True
 
@@ -481,16 +486,16 @@ class HttpRepository(ArtifactsRepository):
     '''
 
     def pull(self, artifacts_name):
-        with urlopen(
-                os.path.join(self.uri, artifacts_name)) as response, open(artifacts_name, 'wb') as out_file:
+        with urlopen(os.path.join(self.uri, artifacts_name)) as response, open(
+                artifacts_name, 'wb') as out_file:
             shutil.copyfileobj(response, out_file)
         return out_file
 
     def push(self, artifacts_name, auth=('', '')):
         req = Request(
-                url=os.path.join(self.uri, artifacts_name),
-                data=open(artifacts_name, 'rb').read(),
-                method='PUT')
+            url=os.path.join(self.uri, artifacts_name),
+            data=open(artifacts_name, 'rb').read(),
+            method='PUT')
         with urlopen(req) as f:
             pass
         if f.status != 201:
-- 
GitLab


From 06ceb4fca10b05c10dba5de66e5b79e376e90aa5 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 14:24:54 +0100
Subject: [PATCH 30/34] add module handling interactions with artifacts
 repository

---
 python/LbNightlyTools/Repository.py | 182 ++++++++++++++++++++++++++++
 1 file changed, 182 insertions(+)
 create mode 100644 python/LbNightlyTools/Repository.py

diff --git a/python/LbNightlyTools/Repository.py b/python/LbNightlyTools/Repository.py
new file mode 100644
index 00000000..77a0e8fc
--- /dev/null
+++ b/python/LbNightlyTools/Repository.py
@@ -0,0 +1,182 @@
+###############################################################################
+# (c) Copyright 2013 CERN                                                     #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+'''
+Code handling interactions with artifacts repository
+'''
+
+_repo_handlers = {}
+
+def register_for(*schemes):
+    def _reg(cls):
+        global _repo_handlers
+        _repo_handlers.update((s, cls) for s in schemes)
+        return cls
+    return _reg
+
+def get_repo_type(uri):
+    '''
+    Returns type of repository based on uri provided in the argument.
+    It may return 'eos', 'file', 'http' or None
+    '''
+    from urllib.parse import urlparse
+
+    result = urlparse(uri)
+    if not result.scheme or result.scheme == 'file':
+        if result.path.startswith('/eos/'):
+            return 'eos'
+        else:
+            return 'file'
+    elif result.scheme == 'root':
+        return 'eos'
+    elif result.scheme in ('http', 'https'):
+        return 'http'
+    return None
+
+class ArtifactsRepository(object):
+    '''
+    Class representing artifacts repository.
+    '''
+
+    def __init__(self, uri):
+        '''
+        Initialises the repository based on its URI
+        '''
+        self.uri = uri
+
+    def pull(self, artifacts_name):
+        '''
+        Returns a file object in the read mode
+
+        Arguments:
+        artifacts_name: name of the artifacts file
+
+        Returns: File object
+        '''
+        raise NotImplementedError(
+            'Should be implemented in the inheriting class')
+
+    def push(self, artifacts_name):
+        '''
+        Pushes artifacts to the repository
+
+        Arguments:
+        artifacts_name: name of the artifcats file
+
+        Returns: True if the artifacts have been pushed, False otherwise
+        '''
+        raise NotImplementedError(
+            'Should be implemented in the inheriting class')
+
+    def exist(self, artifacts_name):
+        '''
+        Checks if artifacts exist
+
+        Arguments:
+        artifacts_name: name of the artifcats file
+
+        Returns: True if the artifacts exist, False otherwise
+        '''
+        raise NotImplementedError(
+            'Should be implemented in the inheriting class')
+
+@register_for('file')
+class FileRepository(ArtifactsRepository):
+    '''
+    Class defining repository in the local file system.
+    '''
+
+    def pull(self, artifacts_name):
+        artifact = open(os.path.join(self.uri, artifacts_name), 'r')
+        return artifact
+
+    def push(self, artifacts_name):
+        try:
+            subprocess.call(['cp', artifacts_name, self.uri])
+        except (subprocess.CalledProcessError, OSError) as ex:
+            logging.debug(
+                'Pushing artifacts to the repository failed: {}'.format(
+                    str(ex)))
+            return False
+        return True
+
+    def exist(self, artifacts_name):
+        return os.path.exists(os.path.join(self.uri, artifacts_name))
+
+@register_for('eos')
+class EosRepository(ArtifactsRepository):
+    '''
+    Class defining repository on EOS.
+    '''
+
+    def pull(self, artifacts_name):
+        try:
+            subprocess.check_call([
+                'xrdcp',
+                os.path.join(self.uri, artifacts_name), artifacts_name
+            ], )
+        except (subprocess.CalledProcessError, OSError) as ex:
+            logging.debug('Pulling failed: {}'.format(str(ex)))
+            return None
+        artifact = open(artifacts_name, 'r')
+        return artifact
+
+    def push(self, artifacts_name):
+        try:
+            subprocess.check_call([
+                'xrdcp', artifacts_name,
+                os.path.join(self.uri, os.path.basename(artifacts_name))
+            ])
+        except (subprocess.CalledProcessError, OSError) as ex:
+            logging.debug(
+                'Pushing artifacts to the repository failed: {}'.format(
+                    str(ex)))
+            return False
+        return True
+
+    def exist(self, artifacts_name):
+        return os.path.exists(os.path.join(self.uri, artifacts_name))
+
+@register_for('http')
+class HttpRepository(ArtifactsRepository):
+    '''
+    Class defining repository through HTTP request methods.
+    '''
+
+    def pull(self, artifacts_name):
+        with urlopen(os.path.join(self.uri, artifacts_name)) as response, open(
+                artifacts_name, 'wb') as out_file:
+            shutil.copyfileobj(response, out_file)
+        return out_file
+
+    def push(self, artifacts_name, auth=('', '')):
+        req = Request(
+            url=os.path.join(self.uri, artifacts_name),
+            data=open(artifacts_name, 'rb').read(),
+            method='PUT')
+        with urlopen(req) as f:
+            pass
+        if f.status != 201:
+            logging.debug('Pushing artifacts to the repository failed')
+            return False
+        return True
+
+    def exist(self, artifacts_name):
+        from urllib.error import HTTPError
+        try:
+            urlopen(os.path.join(self.uri, artifacts_name)).close()
+            return True
+        except HTTPError:
+            return False
+
+def connect(uri, *args, **kwargs):
+    global _repo_handlers
+    return _repo_handlers[get_repo_type(uri)](uri, *args, **kwargs)
+    
-- 
GitLab


From 2f23809bdb9944974c532e2e8b388e01dec845e6 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 14:39:23 +0100
Subject: [PATCH 31/34] cleaning up merge request

---
 python/LbNightlyTools/Configuration.py        | 61 +------------------
 python/LbNightlyTools/Scripts/Build.py        |  7 +--
 python/LbNightlyTools/Scripts/Checkout.py     | 11 +---
 python/LbNightlyTools/Scripts/Common.py       | 26 +++-----
 python/LbNightlyTools/Scripts/EnabledSlots.py | 18 ------
 5 files changed, 14 insertions(+), 109 deletions(-)

diff --git a/python/LbNightlyTools/Configuration.py b/python/LbNightlyTools/Configuration.py
index 22a94a40..1cd94610 100644
--- a/python/LbNightlyTools/Configuration.py
+++ b/python/LbNightlyTools/Configuration.py
@@ -22,7 +22,6 @@ import copy
 import urllib2
 import urllib
 import json
-from subprocess import call
 from datetime import datetime
 from collections import OrderedDict
 from StringIO import StringIO
@@ -483,8 +482,6 @@ class Project(object):
             self.ignore_slot_build_tool = kwargs['ignore_slot_build_tool']
 
         self.build_results = None
-        self.hash = self.calc_hash()
-        self.checkedout = False
 
     def toDict(self):
         '''
@@ -946,35 +943,6 @@ class Project(object):
         self._fixCMT(patchfile, dryrun=dryrun)
         self._fixProjectConfigJSON(patchfile, dryrun=dryrun)
 
-    
-    def calc_hash(self):
-        '''
-        Returns the hash of the project which is the concatenation
-        of the commit id, and optionally merges.                            
-        '''
-        # TODO: what about tagged versions and data packages?
-        hash = ''
-        try:
-            hash += self.checkout_opts['commit'][:7]
-            for merge in self.checkout_opts['merges']:
-                hash += merge[1][:7]
-        except KeyError:
-            __log__.debug('no commit id and merges for: ' + str(self.toDict()))
-            pass
-        return hash
-
-
-    def checkout_artifact_exists(self, artifacts_repo):
-        '''
-        Checks if the project with this hash has already been checked out
-        '''
-        file = os.path.join(artifacts_repo, 
-                            self.name + self.hash + '.zip')
-        if os.path.exists(file):
-            return True
-        else:
-            return False
-
 
 class Package(object):
     '''
@@ -997,7 +965,6 @@ class Package(object):
         self.container = None
         self.checkout = kwargs.get('checkout')
         self.checkout_opts = kwargs.get('checkout_opts', {})
-        self.hash = self.calc_hash()
 
     @property
     def slot(self):
@@ -1105,11 +1072,6 @@ class Package(object):
         return "{0}/{1}".format(self.name, self.version)
 
 
-    def calc_hash(self):
-        # TODO: to be improved ?
-        return self.version
-
-
 class _ContainedList(object):
     '''
     Helper class to handle a list of projects bound to a slot.
@@ -1758,33 +1720,14 @@ class Slot(object):
                 pass
 
         context = kwargs.pop('context', NullContext)
-        artifacts_repo = kwargs.pop('artifacts_repo')
+
         results = OrderedDict()
         for project in self.activeProjects:
             if projects is None or project.name in projects:
                 results[project.name] = {}
                 try:
                     with context(project) as ctx:
-                        checkout_metadata_file = os.path.join(
-                                                    artifacts_repo,
-                                                    project.name + 
-                                                    project.hash +
-                                                    '.txt')
-                        if project.checkout_artifact_exists(artifacts_repo):
-                            __log__.debug('project {} with hash {} '
-                                          'has already been checked out'
-                                          .format(project.name, project.hash))
-                            call(['unzip','-q', checkout_metadata_file.replace('.txt', '.zip')])
-                            with open(checkout_metadata_file, 'r') as infile:
-                                checkout_metadata = json.load(infile)
-                            results[project.name] = checkout_metadata
-                            project.checkedout = True
-                            continue
-                        else:
-                            results[project.name] = project.checkout(**kwargs)
-                            __log__.debug('saving checkout metadata to: ' + checkout_metadata_file)
-                            with open(checkout_metadata_file, 'w') as outfile:
-                                json.dump(results, outfile)
+                        results[project.name] = project.checkout(**kwargs)
                         ctx.result = results[project.name]
                 except (RuntimeError, AssertionError) as x:
                     msg = 'failed to checkout {}: {}: {}'.format(
diff --git a/python/LbNightlyTools/Scripts/Build.py b/python/LbNightlyTools/Scripts/Build.py
index 847b774a..213c14b1 100644
--- a/python/LbNightlyTools/Scripts/Build.py
+++ b/python/LbNightlyTools/Scripts/Build.py
@@ -100,11 +100,6 @@ def unpackArtifacts(src, dest):
             # --clean option)
             call(['unzip', '-n', '-q', f], cwd=dest)
 
-    # TODO: copy and unpack only projects in slot, remove copying from eos in build.sh 
-    # for project in slot:
-    #     get_checkout_artifact
-    #     unpack_artifact
-
 
 try:
     from LbEnv import which
@@ -290,7 +285,7 @@ class Script(BaseScript):
                 os.path.join(self.artifacts_dir, 'ccache'), self.build_dir)
 
             if os.path.exists(os.path.join(self.artifacts_dir, 'slot.patch')):
-                self.log.warning('Applying patch file: %s' % os.path.join(
+                self.log.warning('Applaying patch file: %s' % os.path.join(
                     self.artifacts_dir, 'slot.patch'))
                 log_call([
                     'patch', '-p1', '-i',
diff --git a/python/LbNightlyTools/Scripts/Checkout.py b/python/LbNightlyTools/Scripts/Checkout.py
index 78fcd0a4..99b0a2de 100644
--- a/python/LbNightlyTools/Scripts/Checkout.py
+++ b/python/LbNightlyTools/Scripts/Checkout.py
@@ -135,8 +135,7 @@ class Script(BaseScript):
             element,
             'src',
             build_id=self.options.build_id,
-            artifacts_dir=self.options.artifacts_repo,
-            hash=element.hash)
+            artifacts_dir=self.artifacts_dir)
 
     def main(self):
         """ Main logic of the script """
@@ -201,8 +200,7 @@ class Script(BaseScript):
             slot.checkout(
                 projects=opts.projects,
                 ignore_errors=opts.ignore_checkout_errors,
-                context=CheckoutContext,
-                artifacts_repo=opts.artifacts_repo)
+                context=CheckoutContext)
 
             with open(join(self.artifacts_dir, 'slot.patch'),
                       'w') as patchfile:
@@ -295,13 +293,10 @@ class Script(BaseScript):
             # ignore missing directories
             # (the project may not have been checked out)
             if not os.path.exists(join(self.build_dir, element.baseDir)):
-                # TODO fix this (correct path ?)
                 self.log.warning('no sources for %s, skip packing', element)
                 continue
             if isinstance(element, DataProject):
                 continue  # ignore DataProjects, because we pack packages
-            if project.checkedout:
-                continue
 
             self.log.info('packing %s %s...', element.name, element.version)
 
@@ -317,7 +312,7 @@ class Script(BaseScript):
                 contname.append(self.options.build_id)
             contname.append('src.zip')
             pack([container],
-                 join(self.options.artifacts_repo, 'packs', 'src', '.'.join(contname)),
+                 join(self.artifacts_dir, 'packs', 'src', '.'.join(contname)),
                  cwd=self.build_dir,
                  checksum='md5',
                  dereference=False,
diff --git a/python/LbNightlyTools/Scripts/Common.py b/python/LbNightlyTools/Scripts/Common.py
index b1b72d35..ad5acc00 100755
--- a/python/LbNightlyTools/Scripts/Common.py
+++ b/python/LbNightlyTools/Scripts/Common.py
@@ -81,12 +81,6 @@ def addBasicOptions(parser):
         metavar='DIR',
         help='directory where to store the artifacts')
 
-    parser.add_option(
-        '--artifacts-repo',
-        action='store',
-        metavar='DIR',
-        help='repository with the checkout artifacts')
-
     parser.add_option(
         '--projects',
         action='store',
@@ -792,7 +786,7 @@ class BaseScript(PlainScript):
                       cwd=self._buildDir(proj)).communicate()
 
 
-def genPackageName(proj, platform, build_id=None, artifacts_dir=None, hash=None):
+def genPackageName(proj, platform, build_id=None, artifacts_dir=None):
     '''
     Generate the source/binary tarball name for a project/package.
 
@@ -807,17 +801,13 @@ def genPackageName(proj, platform, build_id=None, artifacts_dir=None, hash=None)
     ...                build_id='dummy', artifacts_dir='artifacts')
     'artifacts/packs/x86_64-slc6-gcc48-dbg/Gaudi.v25r0.dummy.x86_64-slc6-gcc48-dbg.zip'
     '''
-    if hash:
-        packname = proj.name.replace('/', '_') + hash + '.zip'
-    else:
-        packname = [proj.name.replace('/', '_'), proj.version]
-        if build_id:
-            packname.append(build_id)
-        packname.append(platform)
-        packname.append('zip')
-        packname = '.'.join(packname)
-        packname = os.path.join('packs', platform, packname)
+    packname = [proj.name.replace('/', '_'), proj.version]
+    if build_id:
+        packname.append(build_id)
+    packname.append(platform)
+    packname.append('zip')
+    packname = '.'.join(packname)
+    packname = os.path.join('packs', platform, packname)
     if artifacts_dir:
         packname = os.path.join(artifacts_dir, packname)
-    print('packname is: ' + packname)
     return packname
diff --git a/python/LbNightlyTools/Scripts/EnabledSlots.py b/python/LbNightlyTools/Scripts/EnabledSlots.py
index cedda2bb..eba09f96 100644
--- a/python/LbNightlyTools/Scripts/EnabledSlots.py
+++ b/python/LbNightlyTools/Scripts/EnabledSlots.py
@@ -97,24 +97,6 @@ class Script(PlainScript):
                     'config': slot.toDict(),
                     'date': self.options.date,
                 }
-                for project in slot.toDict()['projects']:
-                    filename = 'project-params-' + project['name']
-                    try:
-                        filename += project['checkout_opts']['commit'][:7]
-                        for merge in project['checkout_opts']['merges']:
-                            filename += merge[1][:7]
-                    except KeyError:
-                        pass
-                    filename += '.txt'
-                    if os.path.isfile(filename):
-                        self.log.info('project: {} already marked to be checked out'
-                                      .format(project['name']))
-                    else:
-                        with open(filename, 'w') as f:
-                            self.log.info('new project to checkout: {}'
-                                          .format(project['name']))
-                            f.write(str(project))
-
                 if not self.options.submit:
                     self.log.debug('   slot info: {}\n{}'.format(
                         key, json.dumps(value, indent=2)))
-- 
GitLab


From 9dfeaa69902529a188d29a4053d70e323ff33751 Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 14:41:30 +0100
Subject: [PATCH 32/34] cleaning up merge request

---
 python/LbNightlyTools/Scripts/Build.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/LbNightlyTools/Scripts/Build.py b/python/LbNightlyTools/Scripts/Build.py
index 213c14b1..faa844a3 100644
--- a/python/LbNightlyTools/Scripts/Build.py
+++ b/python/LbNightlyTools/Scripts/Build.py
@@ -285,7 +285,7 @@ class Script(BaseScript):
                 os.path.join(self.artifacts_dir, 'ccache'), self.build_dir)
 
             if os.path.exists(os.path.join(self.artifacts_dir, 'slot.patch')):
-                self.log.warning('Applaying patch file: %s' % os.path.join(
+                self.log.warning('Applying patch file: %s' % os.path.join(
                     self.artifacts_dir, 'slot.patch'))
                 log_call([
                     'patch', '-p1', '-i',
@@ -487,7 +487,7 @@ string(REPLACE "$${NIGHTLY_BUILD_ROOT}" "$${CMAKE_CURRENT_LIST_DIR}"
                 })
 
             if os.path.exists(os.path.join(self.artifacts_dir, 'slot.patch')):
-                self.log.warning('Applying patch file')
+                self.log.warning('Applaying patch file')
                 log_call([
                     'patch', '-p1', '-i',
                     os.path.join(self.artifacts_dir, 'slot.patch')
-- 
GitLab


From 0d123eec5833f84a5dad5b2ee5ea9ccac3ef01b9 Mon Sep 17 00:00:00 2001
From: Gitlab CI <noreply@cern.ch>
Date: Fri, 21 Feb 2020 12:39:39 +0000
Subject: [PATCH 33/34] Fixed formatting

patch generated by https://gitlab.cern.ch/lhcb-core/LbNightlyTools/-/jobs/7287692
---
 python/LbNightlyTools/Repository.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/python/LbNightlyTools/Repository.py b/python/LbNightlyTools/Repository.py
index 77a0e8fc..6affa201 100644
--- a/python/LbNightlyTools/Repository.py
+++ b/python/LbNightlyTools/Repository.py
@@ -14,13 +14,16 @@ Code handling interactions with artifacts repository
 
 _repo_handlers = {}
 
+
 def register_for(*schemes):
     def _reg(cls):
         global _repo_handlers
         _repo_handlers.update((s, cls) for s in schemes)
         return cls
+
     return _reg
 
+
 def get_repo_type(uri):
     '''
     Returns type of repository based on uri provided in the argument.
@@ -40,6 +43,7 @@ def get_repo_type(uri):
         return 'http'
     return None
 
+
 class ArtifactsRepository(object):
     '''
     Class representing artifacts repository.
@@ -87,6 +91,7 @@ class ArtifactsRepository(object):
         raise NotImplementedError(
             'Should be implemented in the inheriting class')
 
+
 @register_for('file')
 class FileRepository(ArtifactsRepository):
     '''
@@ -110,6 +115,7 @@ class FileRepository(ArtifactsRepository):
     def exist(self, artifacts_name):
         return os.path.exists(os.path.join(self.uri, artifacts_name))
 
+
 @register_for('eos')
 class EosRepository(ArtifactsRepository):
     '''
@@ -144,6 +150,7 @@ class EosRepository(ArtifactsRepository):
     def exist(self, artifacts_name):
         return os.path.exists(os.path.join(self.uri, artifacts_name))
 
+
 @register_for('http')
 class HttpRepository(ArtifactsRepository):
     '''
@@ -176,7 +183,7 @@ class HttpRepository(ArtifactsRepository):
         except HTTPError:
             return False
 
+
 def connect(uri, *args, **kwargs):
     global _repo_handlers
     return _repo_handlers[get_repo_type(uri)](uri, *args, **kwargs)
-    
-- 
GitLab


From f41ca7e2a2321737cece0cb0e50921410f544c1a Mon Sep 17 00:00:00 2001
From: Maciej Szymanski <maszyman@cern.ch>
Date: Fri, 21 Feb 2020 13:56:10 +0100
Subject: [PATCH 34/34] add doc

---
 python/LbNightlyTools/Repository.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/python/LbNightlyTools/Repository.py b/python/LbNightlyTools/Repository.py
index 6affa201..8a80ffe4 100644
--- a/python/LbNightlyTools/Repository.py
+++ b/python/LbNightlyTools/Repository.py
@@ -16,6 +16,9 @@ _repo_handlers = {}
 
 
 def register_for(*schemes):
+    '''
+    Decorator used to register the concrete type of repository
+    '''
     def _reg(cls):
         global _repo_handlers
         _repo_handlers.update((s, cls) for s in schemes)
@@ -185,5 +188,9 @@ class HttpRepository(ArtifactsRepository):
 
 
 def connect(uri, *args, **kwargs):
+    '''
+    Function returning the artifacts repository
+    object based on the URI provided in the argument.
+    '''
     global _repo_handlers
     return _repo_handlers[get_repo_type(uri)](uri, *args, **kwargs)
-- 
GitLab