From 9b37c79bc9c43b7c41530474a1286b35436e5b2d Mon Sep 17 00:00:00 2001 From: Marco Clemencic <marco.clemencic@cern.ch> Date: Mon, 17 Oct 2016 14:20:51 +0200 Subject: [PATCH 1/5] minor refactoring of lbn-gen-old-dist --- python/LbNightlyTools/Scripts/GenOldDist.py | 51 +++++++++++---------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/python/LbNightlyTools/Scripts/GenOldDist.py b/python/LbNightlyTools/Scripts/GenOldDist.py index c5b8b2bc..f03cbb4f 100644 --- a/python/LbNightlyTools/Scripts/GenOldDist.py +++ b/python/LbNightlyTools/Scripts/GenOldDist.py @@ -223,13 +223,7 @@ class Script(BaseScript): html.write(HTML_TAIL) mkHash(html_fn) - def _repack(self, project): - tarname = genPackageName(project, self.platform or 'src', - build_id=self.options.build_id, - artifacts_dir=self.artifacts_dir) - oldtar = genOldPackageName(project, self.platform, - artifacts_dir=self.old_artifacts_dir) - + def _repack(self, tarname, oldtar): ensureDirs([os.path.dirname(oldtar)]) # repackage to .tar.gz self.log.debug('repacking %s to %s', tarname, oldtar) @@ -244,6 +238,30 @@ class Script(BaseScript): raise SystemExit(1) mkHash(oldtar) + def processProject(self, project): + # we repack platform independent only if --shared, while + # platform dependent shoud be always repacked + if not self.platform or not project.platform_independent: + tarname = genPackageName(project, self.platform or 'src', + build_id=self.options.build_id, + artifacts_dir=self.artifacts_dir) + oldtar = genOldPackageName(project, self.platform, + artifacts_dir=self.old_artifacts_dir) + + self._repack(tarname, oldtar) + + # The HTML file generation must be delayed until we have a platform + # to make sure we get the correct dependencies between projects in + # the slot. + if self.platform: + self._genHTML(project, shared=True) + if not project.platform_independent: + self._genHTML(project, shared=False) + # Unless we are not going to build for any platform, in which case + # we are sure that the dependencies will be correct anyway + elif not self.slot.platforms and project.platform_independent: + self._genHTML(project, shared=True) + def main(self): ''' Main function of the script. @@ -275,22 +293,7 @@ class Script(BaseScript): if isinstance(project, DataProject): self.log.error('data projects not supported yet (%s)', project) continue - - # we repack platform independent only if --shared, while - # platform dependent shoud be always repacked - if not self.platform or not project.platform_independent: - self._repack(project) - - # The HTML file generation must be delayed until we have a platform - # to make sure we get the correct dependencies between projects in - # the slot. - if self.platform: - self._genHTML(project, shared=True) - if not project.platform_independent: - self._genHTML(project, shared=False) - # Unless we are not going to build for any platform, in which case - # we are sure that the dependencies will be correct anyway - elif not self.slot.platforms and project.platform_independent: - self._genHTML(project, shared=True) + else: + self.processProject(project) return 0 -- GitLab From c961bf5a10d733a7648b366cd9a7af3c33f112ff Mon Sep 17 00:00:00 2001 From: Marco Clemencic <marco.clemencic@cern.ch> Date: Mon, 17 Oct 2016 15:08:44 +0200 Subject: [PATCH 2/5] refactored code to generate the tarfile names --- python/LbNightlyTools/Scripts/Build.py | 25 +--------------------- python/LbNightlyTools/Scripts/Checkout.py | 18 ++++++---------- python/LbNightlyTools/Scripts/Common.py | 26 +++++++++++++++++++++++ 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/python/LbNightlyTools/Scripts/Build.py b/python/LbNightlyTools/Scripts/Build.py index 75238859..dc29e110 100644 --- a/python/LbNightlyTools/Scripts/Build.py +++ b/python/LbNightlyTools/Scripts/Build.py @@ -28,6 +28,7 @@ from LbNightlyTools.Utils import ensureDirs, pack, chdir, wipeDir from LbNightlyTools.Utils import cpuinfo, write_json from LbNightlyTools.Scripts.CollectBuildLogs import Script as CBLScript +from LbNightlyTools.Scripts.Common import BaseScript, genPackageName from string import Template from datetime import datetime @@ -86,29 +87,6 @@ def unpackArtifacts(src, dest): '--no-overwrite-dir', '--keep-old-files', '-f', f], cwd=dest) -def genPackageName(proj, platform, build_id=None, artifacts_dir=None): - ''' - Generate the binary tarball name for a project. - - >>> from LbNightlyTools.Configuration import Project - >>> genPackageName(Project('Gaudi', 'HEAD'), - ... 'x86_64-slc6-gcc48-opt') - 'Gaudi.HEAD.x86_64-slc6-gcc48-opt.tar.bz2' - >>> genPackageName(Project('Gaudi', 'v25r0'), - ... 'x86_64-slc6-gcc48-dbg', - ... build_id='dummy', artifacts_dir='artifacts') - 'artifacts/Gaudi.v25r0.dummy.x86_64-slc6-gcc48-dbg.tar.bz2' - ''' - packname = [proj.name, proj.version] - if build_id: - packname.append(build_id) - packname.append(platform) - packname.append('tar.bz2') - packname = '.'.join(packname) - if artifacts_dir: - packname = os.path.join(artifacts_dir, packname) - return packname - def which(cmd): ''' @@ -127,7 +105,6 @@ def which(cmd): return None -from LbNightlyTools.Scripts.Common import BaseScript class Script(BaseScript): ''' Script to build the projects in a slot configuration. diff --git a/python/LbNightlyTools/Scripts/Checkout.py b/python/LbNightlyTools/Scripts/Checkout.py index 3c4676f2..ad95c605 100644 --- a/python/LbNightlyTools/Scripts/Checkout.py +++ b/python/LbNightlyTools/Scripts/Checkout.py @@ -24,11 +24,11 @@ from os.path import join from LbNightlyTools.Utils import chdir, pack from LbNightlyTools.HTMLUtils import XTerm2HTML from LbNightlyTools.Configuration import DataProject +from LbNightlyTools.Scripts.Common import BaseScript, genPackageName __log__ = logging.getLogger(__name__) -from LbNightlyTools.Scripts.Common import BaseScript class Script(BaseScript): ''' Script to checkout a consistent set of projects as described in a @@ -52,8 +52,8 @@ class Script(BaseScript): def defineOpts(self): """ User options -- has to be overridden """ from LbNightlyTools.Scripts.Common import (addBasicOptions, - addDashboardOptions, - addDeploymentOptions) + addDashboardOptions, + addDeploymentOptions) addBasicOptions(self.parser) addDashboardOptions(self.parser) addDeploymentOptions(self.parser) @@ -73,12 +73,9 @@ class Script(BaseScript): ''' Return the filename of the archive (package) of the given project. ''' - packname = [element.name.replace('/', '_'), element.version] - if self.options.build_id: - packname.append(self.options.build_id) - packname.append('src') - packname.append('tar.bz2') - return '.'.join(packname) + return genPackageName(element, 'src', + build_id=self.options.build_id, + artifacts_dir=self.artifacts_dir) def main(self): """ Main logic of the script """ @@ -163,8 +160,7 @@ class Script(BaseScript): self.log.info('packing %s %s...', element.name, element.version) - pack([element.baseDir], join(self.artifacts_dir, - self.packname(element)), + pack([element.baseDir], self.packname(element), cwd=self.build_dir, checksum='md5') for container in containers(): container = container.name diff --git a/python/LbNightlyTools/Scripts/Common.py b/python/LbNightlyTools/Scripts/Common.py index 0eadaca8..433c3b87 100644 --- a/python/LbNightlyTools/Scripts/Common.py +++ b/python/LbNightlyTools/Scripts/Common.py @@ -442,3 +442,29 @@ class BaseScript(PlainScript): receiver.accept(msg) else: self.tasks.add(self.send, (msg, True)) + + +def genPackageName(proj, platform, build_id=None, artifacts_dir=None): + ''' + Generate the source/binary tarball name for a project/package. + + >>> from LbNightlyTools.Configuration import Project, Package + >>> genPackageName(Project('Gaudi', 'HEAD'), + ... 'x86_64-slc6-gcc48-opt') + 'Gaudi.HEAD.x86_64-slc6-gcc48-opt.tar.bz2' + >>> genPackageName(Package('Gen/DecFiles', 'HEAD'), 'src') + 'Gen_DecFiles.head.src.tar.bz2' + >>> genPackageName(Project('Gaudi', 'v25r0'), + ... 'x86_64-slc6-gcc48-dbg', + ... build_id='dummy', artifacts_dir='artifacts') + 'artifacts/Gaudi.v25r0.dummy.x86_64-slc6-gcc48-dbg.tar.bz2' + ''' + packname = [proj.name.replace('/', '_'), proj.version] + if build_id: + packname.append(build_id) + packname.append(platform) + packname.append('tar.bz2') + packname = '.'.join(packname) + if artifacts_dir: + packname = os.path.join(artifacts_dir, packname) + return packname -- GitLab From ed96fb495e0ec1a4cc2241b929c4b0ed85ee0b21 Mon Sep 17 00:00:00 2001 From: Marco Clemencic <marco.clemencic@cern.ch> Date: Mon, 17 Oct 2016 15:41:46 +0200 Subject: [PATCH 3/5] added support for data pacakges in lbn-gen-old-dist --- python/LbNightlyTools/Scripts/GenOldDist.py | 61 ++++++++++++++++----- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/python/LbNightlyTools/Scripts/GenOldDist.py b/python/LbNightlyTools/Scripts/GenOldDist.py index f03cbb4f..2413c0dc 100644 --- a/python/LbNightlyTools/Scripts/GenOldDist.py +++ b/python/LbNightlyTools/Scripts/GenOldDist.py @@ -23,7 +23,7 @@ from hashlib import md5 from LbNightlyTools.Scripts.Common import BaseScript from LbNightlyTools.Scripts.Build import genPackageName from LbNightlyTools.Utils import ensureDirs, applyenv -from LbNightlyTools.Configuration import DataProject +from LbNightlyTools.Configuration import Project, DataProject, Package HTML_HEAD = \ '''<H3>Project {project.name} version {project.version} ({type} files) </H3> @@ -38,19 +38,29 @@ def genOldPackageName(proj, platform=None, artifacts_dir=None): ''' Generate the binary tarball name for a project. - >>> from LbNightlyTools.Configuration import Project - >>> genPackageName(Project('Gaudi', 'HEAD'), - ... 'x86_64-slc6-gcc48-opt') - 'GAUDI/GAUDI_GAUDI_HEAD_x86_64-slc6-gcc49-opt.tar.gz' - >>> genPackageName(Project('Gaudi', 'HEAD'), 'src') + >>> from LbNightlyTools.Configuration import Project, DBASE, Package, PARAM + >>> genOldPackageName(Project('Gaudi', 'HEAD'), + ... 'x86_64-slc6-gcc48-opt') + 'GAUDI/GAUDI_GAUDI_HEAD_x86_64-slc6-gcc48-opt.tar.gz' + >>> genOldPackageName(Project('Gaudi', 'HEAD'), 'src') 'GAUDI/GAUDI_GAUDI_HEAD.tar.gz' - >>> genPackageName(Project('Gaudi', 'v25r0'), - ... 'x86_64-slc6-gcc48-dbg', - ... artifacts_dir='artifacts') + >>> d = DBASE(packages=[Package('Gen/DecFiles', 'v27r40')]) + >>> genOldPackageName(d.Gen_DecFiles) + 'DBASE/DBASE_Gen_DecFiles_v27r40.tar.gz' + >>> d = PARAM(packages=[Package('QMTestFiles', 'v1r3')]) + >>> genOldPackageName(d.QMTestFiles) + 'PARAM/PARAM_QMTestFiles_v1r3.tar.gz' + >>> genOldPackageName(Project('Gaudi', 'v25r0'), + ... 'x86_64-slc6-gcc48-dbg', + ... artifacts_dir='artifacts') 'artifacts/GAUDI/GAUDI_GAUDI_v25r0_x86_64-slc6-gcc48-dbg.tar.gz' ''' - upname = proj.name.upper() - packname = [upname, upname, proj.version] + if isinstance(proj, Project): + upname = proj.name.upper() + packname = [upname, upname, proj.version] + else: + upname = proj.container.name.upper() + packname = [upname, proj.name.replace('/', '_'), proj.version] if platform and platform not in ('src',): packname.append(platform) packname = '_'.join(packname) + '.tar.gz' @@ -151,6 +161,19 @@ class Script(BaseScript): html_name = os.path.basename(tarname)[:-7] html_fn = os.path.join(self.html_dir, html_name + '.html') + if isinstance(project, Package): + self.log.debug('writing %s', html_fn) + with open(html_fn, 'wb') as html: + html.write(('<H3>{name} version {version}</H3>\n' + '<A NAME={cont}/{name}/{version} HREF={cont}/{tarname}.tar.gz>{tarname}</A>\n' + ).format(name=project.name, + version=project.version, + cont=project.container.name.upper(), + tarname=html_name)) + html.write(HTML_TAIL) + mkHash(html_fn) + return + manifest = [project] if self.platform and not project.platform_independent: manifest.extend(['InstallArea', self.platform]) @@ -262,6 +285,18 @@ class Script(BaseScript): elif not self.slot.platforms and project.platform_independent: self._genHTML(project, shared=True) + def processDataPackage(self, package): + # process data packages only if --shared + if self.platform: + return + tarname = genPackageName(package, 'src', + build_id=self.options.build_id, + artifacts_dir=self.artifacts_dir) + oldtar = genOldPackageName(package, + artifacts_dir=self.old_artifacts_dir) + self._repack(tarname, oldtar) + self._genHTML(package, shared=True) + def main(self): ''' Main function of the script. @@ -291,8 +326,8 @@ class Script(BaseScript): continue if isinstance(project, DataProject): - self.log.error('data projects not supported yet (%s)', project) - continue + for package in project.packages: + self.processDataPackage(package) else: self.processProject(project) -- GitLab From 05c281304e1a405f34f05ffa5daf930e5fb21974 Mon Sep 17 00:00:00 2001 From: Marco Clemencic <marco.clemencic@cern.ch> Date: Mon, 17 Oct 2016 17:35:44 +0200 Subject: [PATCH 4/5] improved dep html generation for data packages --- python/LbNightlyTools/Scripts/GenOldDist.py | 23 +++++++++++---------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/python/LbNightlyTools/Scripts/GenOldDist.py b/python/LbNightlyTools/Scripts/GenOldDist.py index 2413c0dc..9f14163b 100644 --- a/python/LbNightlyTools/Scripts/GenOldDist.py +++ b/python/LbNightlyTools/Scripts/GenOldDist.py @@ -27,11 +27,15 @@ from LbNightlyTools.Configuration import Project, DataProject, Package HTML_HEAD = \ '''<H3>Project {project.name} version {project.version} ({type} files) </H3> -<A NAME={name} HREF={tarname}>{name}</A> +<A NAME={tarname} HREF={tarname}.tar.gz>{tarname}</A> <MENU><LI> ''' -HTML_LINE = '[<A HREF={cont}/{name}.tar.gz>{name}</A>]\n' +HTML_LINE = '[<A HREF={cont}/{tarname}.tar.gz>{tarname}</A>]\n' HTML_TAIL = '</MENU>\n' +HTML_HEAD_PKG = \ +'''<H3>{project.name} version {project.version}</H3> +<A NAME={cont}/{project.name}/{project.version} HREF={cont}/{tarname}.tar.gz>{tarname}</A> +''' def genOldPackageName(proj, platform=None, artifacts_dir=None): @@ -164,12 +168,10 @@ class Script(BaseScript): if isinstance(project, Package): self.log.debug('writing %s', html_fn) with open(html_fn, 'wb') as html: - html.write(('<H3>{name} version {version}</H3>\n' - '<A NAME={cont}/{name}/{version} HREF={cont}/{tarname}.tar.gz>{tarname}</A>\n' - ).format(name=project.name, - version=project.version, - cont=project.container.name.upper(), - tarname=html_name)) + html.write(HTML_HEAD_PKG.format( + project=project, + cont=project.container.name.upper(), + tarname=html_name)) html.write(HTML_TAIL) mkHash(html_fn) return @@ -213,8 +215,7 @@ class Script(BaseScript): type='source' if shared else (self.platform + ' binary'), - name=html_name, - tarname=tarname)) + tarname=html_name)) for proj, version, platf in projects: if proj in ('LCGCMT', 'LCGGrid', 'GENSER'): @@ -227,7 +228,7 @@ class Script(BaseScript): # and for the special cases coming from "source" if platf and (cont == 'source' or not shared): name += '_' + platf - html.write(HTML_LINE.format(cont=cont, name=name)) + html.write(HTML_LINE.format(cont=cont, tarname=name)) pkg_used = set() for pkg, version in packages: -- GitLab From 72f6a381387fd3ca067538d374ede693d21ac125 Mon Sep 17 00:00:00 2001 From: Marco Clemencic <marco.clemencic@cern.ch> Date: Mon, 17 Oct 2016 18:26:15 +0200 Subject: [PATCH 5/5] added index and shared to deps html of sources tarfile --- python/LbNightlyTools/Scripts/GenOldDist.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/python/LbNightlyTools/Scripts/GenOldDist.py b/python/LbNightlyTools/Scripts/GenOldDist.py index 9f14163b..9592f411 100644 --- a/python/LbNightlyTools/Scripts/GenOldDist.py +++ b/python/LbNightlyTools/Scripts/GenOldDist.py @@ -230,6 +230,18 @@ class Script(BaseScript): name += '_' + platf html.write(HTML_LINE.format(cont=cont, tarname=name)) + # we add dependencies on special (platform independent) tarballs + # to the "shared" tarball + if shared: + for special in ('index', 'shared'): + tar = genOldPackageName( + project, special, + artifacts_dir=self.old_artifacts_dir) + if os.path.exists(tar): + cont = project.name.upper() + name = os.path.basename(tar)[:-7] + html.write(HTML_LINE.format(cont=cont, tarname=name)) + pkg_used = set() for pkg, version in packages: pkgpath = findDataPackage(pkg, version) @@ -271,8 +283,15 @@ class Script(BaseScript): artifacts_dir=self.artifacts_dir) oldtar = genOldPackageName(project, self.platform, artifacts_dir=self.old_artifacts_dir) - self._repack(tarname, oldtar) + for special in ('index', 'shared'): + tar = genPackageName(project, special, + build_id=self.options.build_id, + artifacts_dir=self.artifacts_dir) + old = genOldPackageName(project, special, + artifacts_dir=self.old_artifacts_dir) + if os.path.exists(tar): + self._repack(tar, old) # The HTML file generation must be delayed until we have a platform # to make sure we get the correct dependencies between projects in -- GitLab