Merge branch 'manage_oldtar' into 'master'

Manage OldTar ball to keep compatibility



See merge request !93

LbRelease/python/LbRelease/OldTarRelease.py

+#!/usr/bin/env python
+# $Id: $
+
+import os, sys, re
+from subprocess import Popen, PIPE
+import shutil
+
+from LbUtils.Script import Script
+from LbConfiguration.Version import ParseSvnVersion
+
+# # @class ReleaseSlot
+# Main script class for to release RPMs to the repository.
+class ReleaseSlot( Script ):
+    _version = ParseSvnVersion( "$Id: $", "$URL: $" )
+    def __init__( self ):
+        Script.__init__( self, usage = "\n\t%prog [options] releasedir",
+                              description = "Script to copy OldTars to the LHCb TAR repository and reindex the DB" )
+
+    def defineOpts( self ):
+        ''' User options '''
+        self.parser.add_option( "-i", "--interactive", action = "store_true", default = False,
+                               help = "Prompt before copying the files" )
+        self.parser.add_option( "-o", "--oldtar-dir", action = "store", default = "/afs/cern.ch/lhcb/distribution",
+                               help = "Prompt before copying the files" )
+        self.parser.add_option( "--oldtar-regex", action = "store", default = None,
+                               help = "Regexp for the OLD tarball names to copy" )
+        self.parser.add_option( "-c", "--copy", action = "store_true", default = False,
+                               help = "Really copy the files, in dry-run mode otherwise" )
+
+    def releaseTars( self, builddir, repodir, copymode, rpmre ):
+        ''' Release the OLD tarballs in builddir to the OLD tarballs repo '''
+
+        builddir = os.path.abspath( builddir )
+        repodir = os.path.abspath( repodir )
+
+        self.log.warning( "Build dir: %s" % builddir )
+        self.log.warning( "Repo  dir: %s" % repodir )
+
+        if not os.path.exists( builddir ):
+            raise Exception( "The build directory %s does not exist" % builddir )
+
+        if not os.path.exists( repodir ):
+            raise Exception( "The OLD tarball repository %s does not exist" % repodir )
+
+        newoldtarfile = []
+
+        # listing of  directories
+        htmldir = os.path.join( builddir, 'html' )
+        if not os.path.exists( htmldir ):
+           self.log.error( "HTML directory do not exist : build probably not yet finished" )
+           return -1
+        tardir = [f for f in os.listdir( builddir ) if f != 'html' ]
+        print tardir
+        for d in tardir:
+          oldtardir = os.path.join( builddir, d )
+          oldtarrepodir = os.path.join( repodir, d )
+          for f in os.listdir( oldtardir ):
+            oldtardirfile = os.path.join( oldtardir, f )
+            repodirfile = os.path.join( oldtarrepodir, f )
+            if os.path.exists( repodirfile ):
+               self.log.warning( "OLD tar file EXISTS : %s already in repository" % f )
+            else:
+               self.log.warning( "OLD tar NEW : %s will be copied to repository" % f )
+               if copymode:
+                  shutil.copy( oldtardirfile, repodirfile )
+                  newoldtarfile.append( f )
+
+        # Returning
+        return newoldtarfile
+
+    def updateRepoHTML( self, builddir, repodir ):
+        ''' Recreate/Update the YUM repository DB  '''
+
+        htmlrepodir = os.path.join( repodir, 'html' )
+        htmldir = os.path.join( builddir, 'html' )
+        if not os.path.exists( htmlrepodir ):
+            raise Exception( "The OLD TAR repository %s does not exist" % htmlrepodir )
+
+        self.log.warning( "Updating HTML repository %s" % htmlrepodir )
+        for f in os.listdir( htmldir ):
+           shutil.copy( os.path.join( htmldir, f ), htmlrepodir )
+
+
+    def main( self ):
+        ''' Main method for the script '''
+        if len( self.args ) != 1:
+            self.parser.error( 'Please specify the directory with the OLD tarballs' )
+
+        if not self.options.copy:
+            self.log.warning( "In dry-run mode. use --copy to perform the actual copy" )
+        else:
+            self.log.warning( "Copying OLD tarballs to the YUM repository" )
+
+        copiedtars = self.releaseTars( self.args[0], self.options.oldtar_dir, self.options.copy, self.options.rpm_regex )
+        if copiedtars != -1 and len( copiedtars ) > 0 and self.options.copy:
+            self.updateRepoHTML( self.args[0], self.options.oldtar_dir )

LbRelease/scripts/lb-deployment-afs-install

 #!/bin/bash
+###############################################################################
+# (c) Copyright 2016 CERN                                                     #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
 #
 # Helper script to install RPMs to AFS during the deployment of a project.
 #
@@ -14,15 +24,10 @@ shift 2
 rpms=$(for platform in $* index ; do echo ${project^^}_${version}_$(echo ${platform} | tr - _) ; done)
 
 
-#echo "cleaning up 'lcg' scratch space, be patient"
-rm -rf /afs/cern.ch/lhcb/software/rpmrel/lcg/*
-#echo "cleaning up 'tmp', be patient"
-rm -rf /afs/cern.ch/lhcb/software/rpmrel/tmp/*
-
 echo installing RPMs
-/afs/cern.ch/lhcb/software/rpmrel/afslbpkr install $rpms
+listrpm=`/afs/cern.ch/lhcb/software/lbinstall/afslbinstall query ${project^^}_${version}`
 
-#echo "cleaning up 'lcg' scratch space, be patient"
-rm -rf /afs/cern.ch/lhcb/software/rpmrel/lcg/*
-#echo "cleaning up 'tmp', be patient"
-rm -rf /afs/cern.ch/lhcb/software/rpmrel/tmp/*
+for rpmfile in $listrpm
+do
+  echo "run /afs/cern.ch/lhcb/software/lbinstall/afslbinstall install $rpmfile"
+done

LbRelease/scripts/lb-deployment-checklist

 #!/usr/bin/env python
+###############################################################################
+# (c) Copyright 2016 CERN                                                     #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
 # -*- coding: utf-8 -*-
 '''
 Script to check the various steps of the deployment of a project.
@@ -348,12 +358,21 @@ def web_updated(project, version):
 @check
 def tarballs(project, version):
     '''
-    You need to generate the old-style distribution tarballs:
+    You need to add the OLD TARs to the TAR repository with
 
-      lb-deployment-oldtarballs <project> <version>
+      <ssh_archive>lb-release-oldtar /data/archive/artifacts/release/lhcb-release/${build_id}/olddist
 
-    See https://twiki.cern.ch/twiki/bin/view/LHCb/ProjectRelease#OldTarballs
+    and
+
+      <ssh_archive>lb-release-oldtar --copy /data/archive/artifacts/release/lhcb-release/${build_id}/olddist
+
+    if the check was successful.
+
+    where build_id should have been specified in the release request.
+
+    See https://twiki.cern.ch/twiki/bin/view/LHCb/ProjectRelease#Data_packages
     '''
+
     alltars = os.listdir(dist_dir(project))
     prefix = '{0}_{0}_{1}'.format(project.upper(), version)
     rex = re.compile(prefix + r'(_.*)?\.tar\.gz')
@@ -371,7 +390,6 @@ def tarballs(project, version):
     missing = md5sums - set(alltars)
     assert not missing, 'missing some md5 files'
 
-
 @check_pkg
 def pkg_tarballs(package, version):
     '''

LbRelease/scripts/lb-release-oldtar

+#!/usr/bin/env python
+import sys
+from LbRelease.OldTarRelease import ReleaseSlot
+sys.exit(ReleaseSlot().run())