diff --git a/python/LbNightlyTools/Scripts/Build.py b/python/LbNightlyTools/Scripts/Build.py index 75238859396c519abfc36ac78f5d6b0ec7988c8f..dc29e11082db34271bfabcfe84d201d0a668f33e 100644 --- a/python/LbNightlyTools/Scripts/Build.py +++ b/python/LbNightlyTools/Scripts/Build.py @@ -28,6 +28,7 @@ from LbNightlyTools.Utils import ensureDirs, pack, chdir, wipeDir from LbNightlyTools.Utils import cpuinfo, write_json from LbNightlyTools.Scripts.CollectBuildLogs import Script as CBLScript +from LbNightlyTools.Scripts.Common import BaseScript, genPackageName from string import Template from datetime import datetime @@ -86,29 +87,6 @@ def unpackArtifacts(src, dest): '--no-overwrite-dir', '--keep-old-files', '-f', f], cwd=dest) -def genPackageName(proj, platform, build_id=None, artifacts_dir=None): - ''' - Generate the binary tarball name for a project. - - >>> from LbNightlyTools.Configuration import Project - >>> genPackageName(Project('Gaudi', 'HEAD'), - ... 'x86_64-slc6-gcc48-opt') - 'Gaudi.HEAD.x86_64-slc6-gcc48-opt.tar.bz2' - >>> genPackageName(Project('Gaudi', 'v25r0'), - ... 'x86_64-slc6-gcc48-dbg', - ... build_id='dummy', artifacts_dir='artifacts') - 'artifacts/Gaudi.v25r0.dummy.x86_64-slc6-gcc48-dbg.tar.bz2' - ''' - packname = [proj.name, proj.version] - if build_id: - packname.append(build_id) - packname.append(platform) - packname.append('tar.bz2') - packname = '.'.join(packname) - if artifacts_dir: - packname = os.path.join(artifacts_dir, packname) - return packname - def which(cmd): ''' @@ -127,7 +105,6 @@ def which(cmd): return None -from LbNightlyTools.Scripts.Common import BaseScript class Script(BaseScript): ''' Script to build the projects in a slot configuration. diff --git a/python/LbNightlyTools/Scripts/Checkout.py b/python/LbNightlyTools/Scripts/Checkout.py index 3c4676f2a55f3114fbdbe437592af5b94631eb9e..ad95c6056911801d0dd52f23a0d27ae647123982 100644 --- a/python/LbNightlyTools/Scripts/Checkout.py +++ b/python/LbNightlyTools/Scripts/Checkout.py @@ -24,11 +24,11 @@ from os.path import join from LbNightlyTools.Utils import chdir, pack from LbNightlyTools.HTMLUtils import XTerm2HTML from LbNightlyTools.Configuration import DataProject +from LbNightlyTools.Scripts.Common import BaseScript, genPackageName __log__ = logging.getLogger(__name__) -from LbNightlyTools.Scripts.Common import BaseScript class Script(BaseScript): ''' Script to checkout a consistent set of projects as described in a @@ -52,8 +52,8 @@ class Script(BaseScript): def defineOpts(self): """ User options -- has to be overridden """ from LbNightlyTools.Scripts.Common import (addBasicOptions, - addDashboardOptions, - addDeploymentOptions) + addDashboardOptions, + addDeploymentOptions) addBasicOptions(self.parser) addDashboardOptions(self.parser) addDeploymentOptions(self.parser) @@ -73,12 +73,9 @@ class Script(BaseScript): ''' Return the filename of the archive (package) of the given project. ''' - packname = [element.name.replace('/', '_'), element.version] - if self.options.build_id: - packname.append(self.options.build_id) - packname.append('src') - packname.append('tar.bz2') - return '.'.join(packname) + return genPackageName(element, 'src', + build_id=self.options.build_id, + artifacts_dir=self.artifacts_dir) def main(self): """ Main logic of the script """ @@ -163,8 +160,7 @@ class Script(BaseScript): self.log.info('packing %s %s...', element.name, element.version) - pack([element.baseDir], join(self.artifacts_dir, - self.packname(element)), + pack([element.baseDir], self.packname(element), cwd=self.build_dir, checksum='md5') for container in containers(): container = container.name diff --git a/python/LbNightlyTools/Scripts/Common.py b/python/LbNightlyTools/Scripts/Common.py index 0eadaca8949cc9057d1617d801cbdf83f805405f..433c3b8742e9aa943846495bf1f39bc2a6e5e4ba 100644 --- a/python/LbNightlyTools/Scripts/Common.py +++ b/python/LbNightlyTools/Scripts/Common.py @@ -442,3 +442,29 @@ class BaseScript(PlainScript): receiver.accept(msg) else: self.tasks.add(self.send, (msg, True)) + + +def genPackageName(proj, platform, build_id=None, artifacts_dir=None): + ''' + Generate the source/binary tarball name for a project/package. + + >>> from LbNightlyTools.Configuration import Project, Package + >>> genPackageName(Project('Gaudi', 'HEAD'), + ... 'x86_64-slc6-gcc48-opt') + 'Gaudi.HEAD.x86_64-slc6-gcc48-opt.tar.bz2' + >>> genPackageName(Package('Gen/DecFiles', 'HEAD'), 'src') + 'Gen_DecFiles.head.src.tar.bz2' + >>> genPackageName(Project('Gaudi', 'v25r0'), + ... 'x86_64-slc6-gcc48-dbg', + ... build_id='dummy', artifacts_dir='artifacts') + 'artifacts/Gaudi.v25r0.dummy.x86_64-slc6-gcc48-dbg.tar.bz2' + ''' + packname = [proj.name.replace('/', '_'), proj.version] + if build_id: + packname.append(build_id) + packname.append(platform) + packname.append('tar.bz2') + packname = '.'.join(packname) + if artifacts_dir: + packname = os.path.join(artifacts_dir, packname) + return packname diff --git a/python/LbNightlyTools/Scripts/GenOldDist.py b/python/LbNightlyTools/Scripts/GenOldDist.py index c5b8b2bccf696e7c22b92532968b8fb9e69fe22f..9592f411dabc79b60870ae01da9354cb57c86159 100644 --- a/python/LbNightlyTools/Scripts/GenOldDist.py +++ b/python/LbNightlyTools/Scripts/GenOldDist.py @@ -23,34 +23,48 @@ from hashlib import md5 from LbNightlyTools.Scripts.Common import BaseScript from LbNightlyTools.Scripts.Build import genPackageName from LbNightlyTools.Utils import ensureDirs, applyenv -from LbNightlyTools.Configuration import DataProject +from LbNightlyTools.Configuration import Project, DataProject, Package HTML_HEAD = \ '''<H3>Project {project.name} version {project.version} ({type} files) </H3> -<A NAME={name} HREF={tarname}>{name}</A> +<A NAME={tarname} HREF={tarname}.tar.gz>{tarname}</A> <MENU><LI> ''' -HTML_LINE = '[<A HREF={cont}/{name}.tar.gz>{name}</A>]\n' +HTML_LINE = '[<A HREF={cont}/{tarname}.tar.gz>{tarname}</A>]\n' HTML_TAIL = '</MENU>\n' +HTML_HEAD_PKG = \ +'''<H3>{project.name} version {project.version}</H3> +<A NAME={cont}/{project.name}/{project.version} HREF={cont}/{tarname}.tar.gz>{tarname}</A> +''' def genOldPackageName(proj, platform=None, artifacts_dir=None): ''' Generate the binary tarball name for a project. - >>> from LbNightlyTools.Configuration import Project - >>> genPackageName(Project('Gaudi', 'HEAD'), - ... 'x86_64-slc6-gcc48-opt') - 'GAUDI/GAUDI_GAUDI_HEAD_x86_64-slc6-gcc49-opt.tar.gz' - >>> genPackageName(Project('Gaudi', 'HEAD'), 'src') + >>> from LbNightlyTools.Configuration import Project, DBASE, Package, PARAM + >>> genOldPackageName(Project('Gaudi', 'HEAD'), + ... 'x86_64-slc6-gcc48-opt') + 'GAUDI/GAUDI_GAUDI_HEAD_x86_64-slc6-gcc48-opt.tar.gz' + >>> genOldPackageName(Project('Gaudi', 'HEAD'), 'src') 'GAUDI/GAUDI_GAUDI_HEAD.tar.gz' - >>> genPackageName(Project('Gaudi', 'v25r0'), - ... 'x86_64-slc6-gcc48-dbg', - ... artifacts_dir='artifacts') + >>> d = DBASE(packages=[Package('Gen/DecFiles', 'v27r40')]) + >>> genOldPackageName(d.Gen_DecFiles) + 'DBASE/DBASE_Gen_DecFiles_v27r40.tar.gz' + >>> d = PARAM(packages=[Package('QMTestFiles', 'v1r3')]) + >>> genOldPackageName(d.QMTestFiles) + 'PARAM/PARAM_QMTestFiles_v1r3.tar.gz' + >>> genOldPackageName(Project('Gaudi', 'v25r0'), + ... 'x86_64-slc6-gcc48-dbg', + ... artifacts_dir='artifacts') 'artifacts/GAUDI/GAUDI_GAUDI_v25r0_x86_64-slc6-gcc48-dbg.tar.gz' ''' - upname = proj.name.upper() - packname = [upname, upname, proj.version] + if isinstance(proj, Project): + upname = proj.name.upper() + packname = [upname, upname, proj.version] + else: + upname = proj.container.name.upper() + packname = [upname, proj.name.replace('/', '_'), proj.version] if platform and platform not in ('src',): packname.append(platform) packname = '_'.join(packname) + '.tar.gz' @@ -151,6 +165,17 @@ class Script(BaseScript): html_name = os.path.basename(tarname)[:-7] html_fn = os.path.join(self.html_dir, html_name + '.html') + if isinstance(project, Package): + self.log.debug('writing %s', html_fn) + with open(html_fn, 'wb') as html: + html.write(HTML_HEAD_PKG.format( + project=project, + cont=project.container.name.upper(), + tarname=html_name)) + html.write(HTML_TAIL) + mkHash(html_fn) + return + manifest = [project] if self.platform and not project.platform_independent: manifest.extend(['InstallArea', self.platform]) @@ -190,8 +215,7 @@ class Script(BaseScript): type='source' if shared else (self.platform + ' binary'), - name=html_name, - tarname=tarname)) + tarname=html_name)) for proj, version, platf in projects: if proj in ('LCGCMT', 'LCGGrid', 'GENSER'): @@ -204,7 +228,19 @@ class Script(BaseScript): # and for the special cases coming from "source" if platf and (cont == 'source' or not shared): name += '_' + platf - html.write(HTML_LINE.format(cont=cont, name=name)) + html.write(HTML_LINE.format(cont=cont, tarname=name)) + + # we add dependencies on special (platform independent) tarballs + # to the "shared" tarball + if shared: + for special in ('index', 'shared'): + tar = genOldPackageName( + project, special, + artifacts_dir=self.old_artifacts_dir) + if os.path.exists(tar): + cont = project.name.upper() + name = os.path.basename(tar)[:-7] + html.write(HTML_LINE.format(cont=cont, tarname=name)) pkg_used = set() for pkg, version in packages: @@ -223,13 +259,7 @@ class Script(BaseScript): html.write(HTML_TAIL) mkHash(html_fn) - def _repack(self, project): - tarname = genPackageName(project, self.platform or 'src', - build_id=self.options.build_id, - artifacts_dir=self.artifacts_dir) - oldtar = genOldPackageName(project, self.platform, - artifacts_dir=self.old_artifacts_dir) - + def _repack(self, tarname, oldtar): ensureDirs([os.path.dirname(oldtar)]) # repackage to .tar.gz self.log.debug('repacking %s to %s', tarname, oldtar) @@ -244,6 +274,49 @@ class Script(BaseScript): raise SystemExit(1) mkHash(oldtar) + def processProject(self, project): + # we repack platform independent only if --shared, while + # platform dependent shoud be always repacked + if not self.platform or not project.platform_independent: + tarname = genPackageName(project, self.platform or 'src', + build_id=self.options.build_id, + artifacts_dir=self.artifacts_dir) + oldtar = genOldPackageName(project, self.platform, + artifacts_dir=self.old_artifacts_dir) + self._repack(tarname, oldtar) + for special in ('index', 'shared'): + tar = genPackageName(project, special, + build_id=self.options.build_id, + artifacts_dir=self.artifacts_dir) + old = genOldPackageName(project, special, + artifacts_dir=self.old_artifacts_dir) + if os.path.exists(tar): + self._repack(tar, old) + + # The HTML file generation must be delayed until we have a platform + # to make sure we get the correct dependencies between projects in + # the slot. + if self.platform: + self._genHTML(project, shared=True) + if not project.platform_independent: + self._genHTML(project, shared=False) + # Unless we are not going to build for any platform, in which case + # we are sure that the dependencies will be correct anyway + elif not self.slot.platforms and project.platform_independent: + self._genHTML(project, shared=True) + + def processDataPackage(self, package): + # process data packages only if --shared + if self.platform: + return + tarname = genPackageName(package, 'src', + build_id=self.options.build_id, + artifacts_dir=self.artifacts_dir) + oldtar = genOldPackageName(package, + artifacts_dir=self.old_artifacts_dir) + self._repack(tarname, oldtar) + self._genHTML(package, shared=True) + def main(self): ''' Main function of the script. @@ -273,24 +346,9 @@ class Script(BaseScript): continue if isinstance(project, DataProject): - self.log.error('data projects not supported yet (%s)', project) - continue - - # we repack platform independent only if --shared, while - # platform dependent shoud be always repacked - if not self.platform or not project.platform_independent: - self._repack(project) - - # The HTML file generation must be delayed until we have a platform - # to make sure we get the correct dependencies between projects in - # the slot. - if self.platform: - self._genHTML(project, shared=True) - if not project.platform_independent: - self._genHTML(project, shared=False) - # Unless we are not going to build for any platform, in which case - # we are sure that the dependencies will be correct anyway - elif not self.slot.platforms and project.platform_independent: - self._genHTML(project, shared=True) + for package in project.packages: + self.processDataPackage(package) + else: + self.processProject(project) return 0