From 9b37c79bc9c43b7c41530474a1286b35436e5b2d Mon Sep 17 00:00:00 2001
From: Marco Clemencic <marco.clemencic@cern.ch>
Date: Mon, 17 Oct 2016 14:20:51 +0200
Subject: [PATCH 1/5] minor refactoring of lbn-gen-old-dist

---
 python/LbNightlyTools/Scripts/GenOldDist.py | 51 +++++++++++----------
 1 file changed, 27 insertions(+), 24 deletions(-)

diff --git a/python/LbNightlyTools/Scripts/GenOldDist.py b/python/LbNightlyTools/Scripts/GenOldDist.py
index c5b8b2bc..f03cbb4f 100644
--- a/python/LbNightlyTools/Scripts/GenOldDist.py
+++ b/python/LbNightlyTools/Scripts/GenOldDist.py
@@ -223,13 +223,7 @@ class Script(BaseScript):
             html.write(HTML_TAIL)
         mkHash(html_fn)
 
-    def _repack(self, project):
-        tarname = genPackageName(project, self.platform or 'src',
-                                 build_id=self.options.build_id,
-                                 artifacts_dir=self.artifacts_dir)
-        oldtar = genOldPackageName(project, self.platform,
-                                   artifacts_dir=self.old_artifacts_dir)
-
+    def _repack(self, tarname, oldtar):
         ensureDirs([os.path.dirname(oldtar)])
         # repackage to .tar.gz
         self.log.debug('repacking %s to %s', tarname, oldtar)
@@ -244,6 +238,30 @@ class Script(BaseScript):
                 raise SystemExit(1)
         mkHash(oldtar)
 
+    def processProject(self, project):
+        # we repack platform independent only if --shared, while
+        # platform dependent shoud be always repacked
+        if not self.platform or not project.platform_independent:
+            tarname = genPackageName(project, self.platform or 'src',
+                                     build_id=self.options.build_id,
+                                     artifacts_dir=self.artifacts_dir)
+            oldtar = genOldPackageName(project, self.platform,
+                                       artifacts_dir=self.old_artifacts_dir)
+
+            self._repack(tarname, oldtar)
+
+        # The HTML file generation must be delayed until we have a platform
+        # to make sure we get the correct dependencies between projects in
+        # the slot.
+        if self.platform:
+            self._genHTML(project, shared=True)
+            if not project.platform_independent:
+                self._genHTML(project, shared=False)
+        # Unless we are not going to build for any platform, in which case
+        # we are sure that the dependencies will be correct anyway
+        elif not self.slot.platforms and project.platform_independent:
+            self._genHTML(project, shared=True)
+
     def main(self):
         '''
         Main function of the script.
@@ -275,22 +293,7 @@ class Script(BaseScript):
             if isinstance(project, DataProject):
                 self.log.error('data projects not supported yet (%s)', project)
                 continue
-
-            # we repack platform independent only if --shared, while
-            # platform dependent shoud be always repacked
-            if not self.platform or not project.platform_independent:
-                self._repack(project)
-
-            # The HTML file generation must be delayed until we have a platform
-            # to make sure we get the correct dependencies between projects in
-            # the slot.
-            if self.platform:
-                self._genHTML(project, shared=True)
-                if not project.platform_independent:
-                    self._genHTML(project, shared=False)
-            # Unless we are not going to build for any platform, in which case
-            # we are sure that the dependencies will be correct anyway
-            elif not self.slot.platforms and project.platform_independent:
-                self._genHTML(project, shared=True)
+            else:
+                self.processProject(project)
 
         return 0
-- 
GitLab


From c961bf5a10d733a7648b366cd9a7af3c33f112ff Mon Sep 17 00:00:00 2001
From: Marco Clemencic <marco.clemencic@cern.ch>
Date: Mon, 17 Oct 2016 15:08:44 +0200
Subject: [PATCH 2/5] refactored code to generate the tarfile names

---
 python/LbNightlyTools/Scripts/Build.py    | 25 +---------------------
 python/LbNightlyTools/Scripts/Checkout.py | 18 ++++++----------
 python/LbNightlyTools/Scripts/Common.py   | 26 +++++++++++++++++++++++
 3 files changed, 34 insertions(+), 35 deletions(-)

diff --git a/python/LbNightlyTools/Scripts/Build.py b/python/LbNightlyTools/Scripts/Build.py
index 75238859..dc29e110 100644
--- a/python/LbNightlyTools/Scripts/Build.py
+++ b/python/LbNightlyTools/Scripts/Build.py
@@ -28,6 +28,7 @@ from LbNightlyTools.Utils import ensureDirs, pack, chdir, wipeDir
 from LbNightlyTools.Utils import cpuinfo, write_json
 
 from LbNightlyTools.Scripts.CollectBuildLogs import Script as CBLScript
+from LbNightlyTools.Scripts.Common import BaseScript, genPackageName
 
 from string import Template
 from datetime import datetime
@@ -86,29 +87,6 @@ def unpackArtifacts(src, dest):
                   '--no-overwrite-dir', '--keep-old-files',
                   '-f', f], cwd=dest)
 
-def genPackageName(proj, platform, build_id=None, artifacts_dir=None):
-    '''
-    Generate the binary tarball name for a project.
-
-    >>> from LbNightlyTools.Configuration import Project
-    >>> genPackageName(Project('Gaudi', 'HEAD'),
-    ...                'x86_64-slc6-gcc48-opt')
-    'Gaudi.HEAD.x86_64-slc6-gcc48-opt.tar.bz2'
-    >>> genPackageName(Project('Gaudi', 'v25r0'),
-    ...                'x86_64-slc6-gcc48-dbg',
-    ...                build_id='dummy', artifacts_dir='artifacts')
-    'artifacts/Gaudi.v25r0.dummy.x86_64-slc6-gcc48-dbg.tar.bz2'
-    '''
-    packname = [proj.name, proj.version]
-    if build_id:
-        packname.append(build_id)
-    packname.append(platform)
-    packname.append('tar.bz2')
-    packname = '.'.join(packname)
-    if artifacts_dir:
-        packname = os.path.join(artifacts_dir, packname)
-    return packname
-
 
 def which(cmd):
     '''
@@ -127,7 +105,6 @@ def which(cmd):
     return None
 
 
-from LbNightlyTools.Scripts.Common import BaseScript
 class Script(BaseScript):
     '''
     Script to build the projects in a slot configuration.
diff --git a/python/LbNightlyTools/Scripts/Checkout.py b/python/LbNightlyTools/Scripts/Checkout.py
index 3c4676f2..ad95c605 100644
--- a/python/LbNightlyTools/Scripts/Checkout.py
+++ b/python/LbNightlyTools/Scripts/Checkout.py
@@ -24,11 +24,11 @@ from os.path import join
 from LbNightlyTools.Utils import chdir, pack
 from LbNightlyTools.HTMLUtils import XTerm2HTML
 from LbNightlyTools.Configuration import DataProject
+from LbNightlyTools.Scripts.Common import BaseScript, genPackageName
 
 __log__ = logging.getLogger(__name__)
 
 
-from LbNightlyTools.Scripts.Common import BaseScript
 class Script(BaseScript):
     '''
     Script to checkout a consistent set of projects as described in a
@@ -52,8 +52,8 @@ class Script(BaseScript):
     def defineOpts(self):
         """ User options -- has to be overridden """
         from LbNightlyTools.Scripts.Common import (addBasicOptions,
-                                                  addDashboardOptions,
-                                                  addDeploymentOptions)
+                                                   addDashboardOptions,
+                                                   addDeploymentOptions)
         addBasicOptions(self.parser)
         addDashboardOptions(self.parser)
         addDeploymentOptions(self.parser)
@@ -73,12 +73,9 @@ class Script(BaseScript):
         '''
         Return the filename of the archive (package) of the given project.
         '''
-        packname = [element.name.replace('/', '_'), element.version]
-        if self.options.build_id:
-            packname.append(self.options.build_id)
-        packname.append('src')
-        packname.append('tar.bz2')
-        return '.'.join(packname)
+        return genPackageName(element, 'src',
+                              build_id=self.options.build_id,
+                              artifacts_dir=self.artifacts_dir)
 
     def main(self):
         """ Main logic of the script """
@@ -163,8 +160,7 @@ class Script(BaseScript):
 
             self.log.info('packing %s %s...', element.name, element.version)
 
-            pack([element.baseDir], join(self.artifacts_dir,
-                                         self.packname(element)),
+            pack([element.baseDir], self.packname(element),
                  cwd=self.build_dir, checksum='md5')
         for container in containers():
             container = container.name
diff --git a/python/LbNightlyTools/Scripts/Common.py b/python/LbNightlyTools/Scripts/Common.py
index 0eadaca8..433c3b87 100644
--- a/python/LbNightlyTools/Scripts/Common.py
+++ b/python/LbNightlyTools/Scripts/Common.py
@@ -442,3 +442,29 @@ class BaseScript(PlainScript):
                 receiver.accept(msg)
         else:
             self.tasks.add(self.send, (msg, True))
+
+
+def genPackageName(proj, platform, build_id=None, artifacts_dir=None):
+    '''
+    Generate the source/binary tarball name for a project/package.
+
+    >>> from LbNightlyTools.Configuration import Project, Package
+    >>> genPackageName(Project('Gaudi', 'HEAD'),
+    ...                'x86_64-slc6-gcc48-opt')
+    'Gaudi.HEAD.x86_64-slc6-gcc48-opt.tar.bz2'
+    >>> genPackageName(Package('Gen/DecFiles', 'HEAD'), 'src')
+    'Gen_DecFiles.head.src.tar.bz2'
+    >>> genPackageName(Project('Gaudi', 'v25r0'),
+    ...                'x86_64-slc6-gcc48-dbg',
+    ...                build_id='dummy', artifacts_dir='artifacts')
+    'artifacts/Gaudi.v25r0.dummy.x86_64-slc6-gcc48-dbg.tar.bz2'
+    '''
+    packname = [proj.name.replace('/', '_'), proj.version]
+    if build_id:
+        packname.append(build_id)
+    packname.append(platform)
+    packname.append('tar.bz2')
+    packname = '.'.join(packname)
+    if artifacts_dir:
+        packname = os.path.join(artifacts_dir, packname)
+    return packname
-- 
GitLab


From ed96fb495e0ec1a4cc2241b929c4b0ed85ee0b21 Mon Sep 17 00:00:00 2001
From: Marco Clemencic <marco.clemencic@cern.ch>
Date: Mon, 17 Oct 2016 15:41:46 +0200
Subject: [PATCH 3/5] added support for data pacakges in lbn-gen-old-dist

---
 python/LbNightlyTools/Scripts/GenOldDist.py | 61 ++++++++++++++++-----
 1 file changed, 48 insertions(+), 13 deletions(-)

diff --git a/python/LbNightlyTools/Scripts/GenOldDist.py b/python/LbNightlyTools/Scripts/GenOldDist.py
index f03cbb4f..2413c0dc 100644
--- a/python/LbNightlyTools/Scripts/GenOldDist.py
+++ b/python/LbNightlyTools/Scripts/GenOldDist.py
@@ -23,7 +23,7 @@ from hashlib import md5
 from LbNightlyTools.Scripts.Common import BaseScript
 from LbNightlyTools.Scripts.Build import genPackageName
 from LbNightlyTools.Utils import ensureDirs, applyenv
-from LbNightlyTools.Configuration import DataProject
+from LbNightlyTools.Configuration import Project, DataProject, Package
 
 HTML_HEAD = \
 '''<H3>Project {project.name} version {project.version} ({type} files) </H3>
@@ -38,19 +38,29 @@ def genOldPackageName(proj, platform=None, artifacts_dir=None):
     '''
     Generate the binary tarball name for a project.
 
-    >>> from LbNightlyTools.Configuration import Project
-    >>> genPackageName(Project('Gaudi', 'HEAD'),
-    ...                'x86_64-slc6-gcc48-opt')
-    'GAUDI/GAUDI_GAUDI_HEAD_x86_64-slc6-gcc49-opt.tar.gz'
-    >>> genPackageName(Project('Gaudi', 'HEAD'), 'src')
+    >>> from LbNightlyTools.Configuration import Project, DBASE, Package, PARAM
+    >>> genOldPackageName(Project('Gaudi', 'HEAD'),
+    ...                   'x86_64-slc6-gcc48-opt')
+    'GAUDI/GAUDI_GAUDI_HEAD_x86_64-slc6-gcc48-opt.tar.gz'
+    >>> genOldPackageName(Project('Gaudi', 'HEAD'), 'src')
     'GAUDI/GAUDI_GAUDI_HEAD.tar.gz'
-    >>> genPackageName(Project('Gaudi', 'v25r0'),
-    ...                'x86_64-slc6-gcc48-dbg',
-    ...                artifacts_dir='artifacts')
+    >>> d = DBASE(packages=[Package('Gen/DecFiles', 'v27r40')])
+    >>> genOldPackageName(d.Gen_DecFiles)
+    'DBASE/DBASE_Gen_DecFiles_v27r40.tar.gz'
+    >>> d = PARAM(packages=[Package('QMTestFiles', 'v1r3')])
+    >>> genOldPackageName(d.QMTestFiles)
+    'PARAM/PARAM_QMTestFiles_v1r3.tar.gz'
+    >>> genOldPackageName(Project('Gaudi', 'v25r0'),
+    ...                   'x86_64-slc6-gcc48-dbg',
+    ...                   artifacts_dir='artifacts')
     'artifacts/GAUDI/GAUDI_GAUDI_v25r0_x86_64-slc6-gcc48-dbg.tar.gz'
     '''
-    upname = proj.name.upper()
-    packname = [upname, upname, proj.version]
+    if isinstance(proj, Project):
+        upname = proj.name.upper()
+        packname = [upname, upname, proj.version]
+    else:
+        upname = proj.container.name.upper()
+        packname = [upname, proj.name.replace('/', '_'), proj.version]
     if platform and platform not in ('src',):
         packname.append(platform)
     packname = '_'.join(packname) + '.tar.gz'
@@ -151,6 +161,19 @@ class Script(BaseScript):
         html_name = os.path.basename(tarname)[:-7]
         html_fn = os.path.join(self.html_dir, html_name + '.html')
 
+        if isinstance(project, Package):
+            self.log.debug('writing %s', html_fn)
+            with open(html_fn, 'wb') as html:
+                html.write(('<H3>{name} version {version}</H3>\n'
+                            '<A NAME={cont}/{name}/{version} HREF={cont}/{tarname}.tar.gz>{tarname}</A>\n'
+                            ).format(name=project.name,
+                                              version=project.version,
+                                              cont=project.container.name.upper(),
+                                              tarname=html_name))
+                html.write(HTML_TAIL)
+            mkHash(html_fn)
+            return
+
         manifest = [project]
         if self.platform and not project.platform_independent:
             manifest.extend(['InstallArea', self.platform])
@@ -262,6 +285,18 @@ class Script(BaseScript):
         elif not self.slot.platforms and project.platform_independent:
             self._genHTML(project, shared=True)
 
+    def processDataPackage(self, package):
+        # process data packages only if --shared
+        if self.platform:
+            return
+        tarname = genPackageName(package, 'src',
+                                 build_id=self.options.build_id,
+                                 artifacts_dir=self.artifacts_dir)
+        oldtar = genOldPackageName(package,
+                                   artifacts_dir=self.old_artifacts_dir)
+        self._repack(tarname, oldtar)
+        self._genHTML(package, shared=True)
+
     def main(self):
         '''
         Main function of the script.
@@ -291,8 +326,8 @@ class Script(BaseScript):
                 continue
 
             if isinstance(project, DataProject):
-                self.log.error('data projects not supported yet (%s)', project)
-                continue
+                for package in project.packages:
+                    self.processDataPackage(package)
             else:
                 self.processProject(project)
 
-- 
GitLab


From 05c281304e1a405f34f05ffa5daf930e5fb21974 Mon Sep 17 00:00:00 2001
From: Marco Clemencic <marco.clemencic@cern.ch>
Date: Mon, 17 Oct 2016 17:35:44 +0200
Subject: [PATCH 4/5] improved dep html generation for data packages

---
 python/LbNightlyTools/Scripts/GenOldDist.py | 23 +++++++++++----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/python/LbNightlyTools/Scripts/GenOldDist.py b/python/LbNightlyTools/Scripts/GenOldDist.py
index 2413c0dc..9f14163b 100644
--- a/python/LbNightlyTools/Scripts/GenOldDist.py
+++ b/python/LbNightlyTools/Scripts/GenOldDist.py
@@ -27,11 +27,15 @@ from LbNightlyTools.Configuration import Project, DataProject, Package
 
 HTML_HEAD = \
 '''<H3>Project {project.name} version {project.version} ({type} files) </H3>
-<A NAME={name} HREF={tarname}>{name}</A>
+<A NAME={tarname} HREF={tarname}.tar.gz>{tarname}</A>
 <MENU><LI>
 '''
-HTML_LINE = '[<A HREF={cont}/{name}.tar.gz>{name}</A>]\n'
+HTML_LINE = '[<A HREF={cont}/{tarname}.tar.gz>{tarname}</A>]\n'
 HTML_TAIL = '</MENU>\n'
+HTML_HEAD_PKG = \
+'''<H3>{project.name} version {project.version}</H3>
+<A NAME={cont}/{project.name}/{project.version} HREF={cont}/{tarname}.tar.gz>{tarname}</A>
+'''
 
 
 def genOldPackageName(proj, platform=None, artifacts_dir=None):
@@ -164,12 +168,10 @@ class Script(BaseScript):
         if isinstance(project, Package):
             self.log.debug('writing %s', html_fn)
             with open(html_fn, 'wb') as html:
-                html.write(('<H3>{name} version {version}</H3>\n'
-                            '<A NAME={cont}/{name}/{version} HREF={cont}/{tarname}.tar.gz>{tarname}</A>\n'
-                            ).format(name=project.name,
-                                              version=project.version,
-                                              cont=project.container.name.upper(),
-                                              tarname=html_name))
+                html.write(HTML_HEAD_PKG.format(
+                    project=project,
+                    cont=project.container.name.upper(),
+                    tarname=html_name))
                 html.write(HTML_TAIL)
             mkHash(html_fn)
             return
@@ -213,8 +215,7 @@ class Script(BaseScript):
                                         type='source' if shared
                                              else (self.platform +
                                                    ' binary'),
-                                        name=html_name,
-                                        tarname=tarname))
+                                        tarname=html_name))
 
             for proj, version, platf in projects:
                 if proj in ('LCGCMT', 'LCGGrid', 'GENSER'):
@@ -227,7 +228,7 @@ class Script(BaseScript):
                 # and for the special cases coming from "source"
                 if platf and (cont == 'source' or not shared):
                     name += '_' + platf
-                html.write(HTML_LINE.format(cont=cont, name=name))
+                html.write(HTML_LINE.format(cont=cont, tarname=name))
 
             pkg_used = set()
             for pkg, version in packages:
-- 
GitLab


From 72f6a381387fd3ca067538d374ede693d21ac125 Mon Sep 17 00:00:00 2001
From: Marco Clemencic <marco.clemencic@cern.ch>
Date: Mon, 17 Oct 2016 18:26:15 +0200
Subject: [PATCH 5/5] added index and shared to deps html of sources tarfile

---
 python/LbNightlyTools/Scripts/GenOldDist.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/python/LbNightlyTools/Scripts/GenOldDist.py b/python/LbNightlyTools/Scripts/GenOldDist.py
index 9f14163b..9592f411 100644
--- a/python/LbNightlyTools/Scripts/GenOldDist.py
+++ b/python/LbNightlyTools/Scripts/GenOldDist.py
@@ -230,6 +230,18 @@ class Script(BaseScript):
                     name += '_' + platf
                 html.write(HTML_LINE.format(cont=cont, tarname=name))
 
+            # we add dependencies on special (platform independent) tarballs
+            # to the "shared" tarball
+            if shared:
+                for special in ('index', 'shared'):
+                    tar = genOldPackageName(
+                        project, special,
+                        artifacts_dir=self.old_artifacts_dir)
+                    if os.path.exists(tar):
+                        cont = project.name.upper()
+                        name = os.path.basename(tar)[:-7]
+                        html.write(HTML_LINE.format(cont=cont, tarname=name))
+
             pkg_used = set()
             for pkg, version in packages:
                 pkgpath = findDataPackage(pkg, version)
@@ -271,8 +283,15 @@ class Script(BaseScript):
                                      artifacts_dir=self.artifacts_dir)
             oldtar = genOldPackageName(project, self.platform,
                                        artifacts_dir=self.old_artifacts_dir)
-
             self._repack(tarname, oldtar)
+            for special in ('index', 'shared'):
+                tar = genPackageName(project, special,
+                                     build_id=self.options.build_id,
+                                     artifacts_dir=self.artifacts_dir)
+                old = genOldPackageName(project, special,
+                                        artifacts_dir=self.old_artifacts_dir)
+                if os.path.exists(tar):
+                    self._repack(tar, old)
 
         # The HTML file generation must be delayed until we have a platform
         # to make sure we get the correct dependencies between projects in
-- 
GitLab