diff --git a/LbRelease/python/LbRelease/LCGYumInstaller.py b/LbRelease/python/LbRelease/LCGYumInstaller.py
index b6dd673f1987405a86e6c10b2df6ddb44b706a7b..d9717c1d1b42248172e21e347379f6fbb463cc4d 100644
--- a/LbRelease/python/LbRelease/LCGYumInstaller.py
+++ b/LbRelease/python/LbRelease/LCGYumInstaller.py
@@ -54,14 +54,19 @@ class LCGYumInstaller(object):
else:
return self._installArea.installPackage(rpm, nodeps=nodeps)
-
def findRpm(self, rpm):
'''
- Fin a specific RPM instance
+ Find a specific RPM instance
'''
package = self._installArea.lbYumClient.findLatestMatchingName(rpm)
return package
+ def list(self, nameregexp):
+ '''
+ List packages macthing a given name
+ '''
+ return self._installArea.lbYumClient.listPackages(nameregexp)
+
def checkInstallArea(self, packageList):
'''
Check which files are packages, which are NOT
diff --git a/LbRelease/python/LbRelease/LbYum/LHCbConfig.py b/LbRelease/python/LbRelease/LbYum/LHCbConfig.py
index 743b41df2153e251ddf1290b9f241bc619972e94..34772fd417cc3ec508b9cc242b1139a8b59b0f22 100644
--- a/LbRelease/python/LbRelease/LbYum/LHCbConfig.py
+++ b/LbRelease/python/LbRelease/LbYum/LHCbConfig.py
@@ -25,7 +25,6 @@ class Config:
if not os.path.exists(yumrepolhcb):
yplf = open(yumrepolhcb, 'w')
- yplf.write(installArea._getYumRepo("lhcbold", rpmsurl))
yplf.write(installArea._getYumRepo("lhcb", lhcbsurl))
yplf.close()
diff --git a/LbRelease/scripts/makeTarFromRPM b/LbRelease/scripts/makeTarFromRPM
new file mode 100755
index 0000000000000000000000000000000000000000..76ed1369745c92232950c6b3b63d62455d2ef641
--- /dev/null
+++ b/LbRelease/scripts/makeTarFromRPM
@@ -0,0 +1,292 @@
+#!/usr/bin/env python
+###############################################################################
+# (c) Copyright 2014 CERN #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+'''
+Script to generate the LCG tarball based on the RPMs provided by PH-SFT
+
+Created on Oct 18, 2016
+
+@author: Ben Couturier
+'''
+
+from LbUtils.Script import Script
+from LbConfiguration.Version import ParseSvnVersion
+from LbRelease import LCGYumInstaller
+from LbLegacy.install_project import calculateMD5
+
+import logging
+import re
+import sys
+import os
+
+__version__ = ParseSvnVersion("$Id$", "$URL$")
+
+class MakeTarScript(Script):
+ _version = __version__
+ _description = __doc__
+
+ def defineOpts(self):
+ parser = self.parser
+ parser.set_defaults(output_dir=None)
+ parser.add_option("-o", "--output-dir",
+ help = "define the output directory for the created tarballs" \
+ "[default: the LHCBTAR directory for the project]")
+ parser.set_defaults(exclude=[])
+ parser.add_option("-e", "--exclude",
+ action = "append",
+ help = "add exclude glob pattern to the file/dir paths")
+ parser.add_option("-s","--siteroot",
+ help = "temporary directory where the RPMs will be installed before repackaging" \
+ "[default: /tmp/siteroot",
+ default="/tmp/siteroot")
+ parser.set_defaults(md5=True)
+ parser.add_option("--md5",
+ action = "store_true",
+ help = "Create the md5 sum file for each tarball [default: %default]")
+ parser.add_option("--no-md5",
+ dest = "md5",
+ action = "store_false",
+ help = "Prevent the creation of the md5 sum file")
+ parser.add_option("--no-install",
+ dest = "noinstall",
+ action = "store_true",
+ default = False,
+ help = "For DEV: does packaging, do not redo install")
+ parser.add_option("--force",
+ dest = "force",
+ action = "store_true",
+ default = False,
+ help = "Ignore installation errors and continue packaging")
+ parser.add_option("--limit-rpm-deps",
+ dest = "limit_rpm_deps",
+ action = "store_true",
+ default = False,
+ help = "Restrict to the RPMs explicitely mentioned, not their dependencies"
+ "Except in the case of LCG_XX RPMs that only contain links."
+ "In that case the package they depend on is installed as well.")
+ parser.add_option("-y", "--siteroot-not-empty",
+ dest = "ackSiterootNotEmpty",
+ action = "store_true",
+ default = False,
+ help = "Acknowledge the fact that there are files in the installation dir already")
+
+
+
+ def main(self):
+ log = logging.getLogger()
+ self.log = log
+ args = self.args
+ opts = self.options
+ version = None
+
+ if len(args) < 4:
+ log.error("Not enough arguments, please specify PROJECT VERSION CMTCONFIG externals_filename")
+ sys.exit(1)
+ else:
+ project = args[0]
+ version = args[1]
+ cmtconfig = args[2]
+ filename = args[3]
+
+ self.project = project
+ self.version = version
+ self.filename = filename
+ self.cmtconfig = cmtconfig
+
+ # Setting the CMTCONFIG to the requested one
+ os.environ['CMTCONFIG'] = self.cmtconfig
+
+ # preparing the actual RPM install area
+ log.warning("Repackaging %s %s for CMTCONFIG %s" % (self.project, self.version, self.cmtconfig))
+ installer = LCGYumInstaller.LCGYumInstaller(opts.siteroot)
+ self._installer = installer
+
+ # Check whether there is an install already
+ if not installer.isAreaEmpty() and not opts.ackSiterootNotEmpty:
+ log.error("The install area %s already contains files, please remove them of force installation with -y" % installer._siteroot)
+ sys.exit(1)
+
+ # Retrieving the list of externals from the mentioned project
+ (lcgVer, externalsList) = self.getExternalsList(filename)
+
+ # Getting the list of packages for LCG
+ log.warning("Loading list of LCG packages from YUM")
+ alllcg = list(installer.list("LCG_%s_" % lcgVer))
+
+ log.warning("Matching list of packages with LCG list")
+ packDict = {}
+ for ext in externalsList:
+ packDict[ext] = None
+ extName = "LCG_%s_%s.*_%s.*" % (lcgVer, ext, cmtconfig.replace("-", "_"))
+ for p in alllcg:
+ if re.match(extName, p.rpmName()):
+ packDict[ext] = p
+ log.debug("%s -> %s" % ( ext, p.rpmName()))
+
+
+ # performing the installation if requested
+ if opts.noinstall == False:
+ self.install(installer, opts.limit_rpm_deps,
+ lcgVer, packDict)
+
+ # Now preparing the tar file
+ output_dir = None
+ if opts.output_dir == None:
+ output_dir = os.environ['LHCBTAR']
+ else:
+ output_dir = os.path.abspath(opts.output_dir)
+
+ log.warning("Will use DIST directory: %s" % output_dir)
+ htmldir = os.path.join(output_dir, "html")
+ sourcedir = os.path.join(output_dir, "source")
+ if not os.path.exists(htmldir):
+ os.makedirs(htmldir)
+ if not os.path.exists(sourcedir):
+ os.makedirs(sourcedir)
+
+ baseFilename = self.getOutputFilename(project, version)
+
+ filename = "%s_%s" % (baseFilename, self.cmtconfig)
+ log.warning("Output Base filename: %s" % filename)
+
+ htmlFilename = os.path.join(output_dir, "html", filename + ".html")
+ tarFilename = os.path.join(output_dir, "source", filename + ".tar.gz")
+
+ # Preparing the tar file
+ # See if we need to restrict the packages to install
+ packageList = None
+ installer.prepareLCGTar(tarFilename, packageList)
+ self.createMD5File(tarFilename, "tar.gz")
+ self.createHTMLFile(htmlFilename, baseFilename, self.cmtconfig)
+ self.createMD5File(htmlFilename, "html")
+
+ def createMD5File(self, tarfile, origExtension):
+ '''
+ Create the md5sum file of the install area
+ '''
+ md5file = tarfile.replace("." + origExtension, ".md5")
+ md5sum = calculateMD5(tarfile)
+ mdf = open(md5file,"w")
+ mdf.write("%s %s" % (md5sum,os.path.basename(tarfile)))
+ mdf.close()
+
+ def createHTMLFile(self, htmlfile, baseName, cmtconfig):
+ '''
+ Create the HTML file for the tar
+ '''
+ fileProj = baseName.split("_")[0]
+ fileVer = baseName[baseName.find("_")+1]
+ namecfg = baseName + "_" + cmtconfig
+ fd = open(htmlfile, "w")
+ fd.write("<H3>Project %s version %s (%s binary files) </H3>\n" % (fileProj, fileVer, cmtconfig))
+ fd.write("<A NAME=%s HREF=source/%s>%s</A>\n" % (namecfg, namecfg + ".tar.gz", namecfg))
+ fd.close()
+
+ def getExternalsList(self, filename):
+ '''
+ Gets the list of all externals needed (as from LCG 86)
+ '''
+ self.log.warning("Loading list of externals from file")
+ # Cache the externals dictionary for the version...
+ # Loading the metadata
+ import json
+ with open(filename) as f:
+ data = json.load(f)
+
+ # Looking for LCG version
+ # We need a file like so:
+ # {
+ # "heptools": {
+ # "version": 84
+ # "packages":[
+ # "AIDA",
+ # "Boost",
+ # "CLHEP",
+ # "COOL"
+ # ]
+ # }
+ # }
+
+ heptools = data["heptools"]
+ self.lcgver = heptools["version"]
+ self.externalsList = heptools["packages"]
+ return (self.lcgver, self.externalsList)
+
+ def getOutputFilename(self, project, version):
+ return self.project.upper() + "_" + self.version
+
+ def install(self, installer, limit_rpm_deps, lcgVer, externalsDict):
+ '''
+ Perform the installation of the needed packages in the install area
+ '''
+ # Now prepare the list of RPMs based on the externals found
+ allrpms = []
+ missing = []
+ for k, linkPackage in externalsDict.iteritems():
+ # Now looking for the RPMs
+ try:
+ if linkPackage != None:
+ self.log.warning("Adding package RPM: %s" % linkPackage.name)
+ allrpms.append(linkPackage.name)
+
+ if linkPackage.name.startswith("LCG_%s" % lcgVer):
+ req = [ r for r in linkPackage.requires if k in r.name ]
+ if len(req) > 0:
+ self.log.warning("Link package - Looking for dependency: %s" % req[0].name)
+ mainPackage = installer.findRpm(req[0].name)
+ if mainPackage != None:
+ self.log.warning("Adding RPM: %s" % mainPackage.name)
+ allrpms.append(mainPackage.name)
+ else:
+ self.log.warning("Missing RPM: %s" % req[0].name)
+ missing.append((req[0].name, req[0].name, ""))
+ else:
+ self.log.warning("Missing RPM: %s" % k)
+ missing.append(k)
+
+ except Exception, e:
+ self.log.warning("Exception looking for RPMs %s " % linkPackage.rpmName())
+ import traceback
+ traceback.print_exc()
+ raise e
+
+ # Now performing the install
+ self.log.warning("Done looking for RPMs")
+ for k in missing:
+ self.log.warning("> Missing RPM for: " + str(k))
+
+ #if len(missing) > 0:
+ # sys.exit(1)
+
+ self.log.warning("Now performing the actual install")
+ self.log.warning("RPM list:" + " ".join(allrpms))
+ packageErrors = []
+ for rpm in allrpms:
+ try:
+ installer.install(rpm, nodeps=limit_rpm_deps)
+ except:
+ packageErrors.append(rpm)
+
+ if len(packageErrors) > 0:
+ self.log.error("Errors installing the following packages:")
+ for p in packageErrors:
+ self.log.error(p)
+ if not self.options.force:
+ self.log.error("Some packages are missing -stopping installation")
+ self.log.error("Use --force option to continue nonetheless")
+ sys.exit(1)
+ else:
+ self.log.warning("All packaged installed successfully")
+
+if __name__ == '__main__':
+ s = MakeTarScript(usage="%prog [options] project version cmtconfig")
+ sys.exit(s.run())
+