From bd2dfed8ebc6b04d73f23aa17fd81dd4cf8ada10 Mon Sep 17 00:00:00 2001
From: Ivan Razumov <ivan.razumov@cern.ch>
Date: Thu, 2 Jun 2016 15:01:36 +0200
Subject: [PATCH] Copy create_lcg_view script from master
---
cmake/scripts/create_lcg_view.py | 656 ++++++++++++++++++++-----------
1 file changed, 428 insertions(+), 228 deletions(-)
diff --git a/cmake/scripts/create_lcg_view.py b/cmake/scripts/create_lcg_view.py
index 7408fe05c6..11f6e4f8de 100755
--- a/cmake/scripts/create_lcg_view.py
+++ b/cmake/scripts/create_lcg_view.py
@@ -9,156 +9,402 @@
# -r LCGREL, --release LCGREL LCG release number (default: 80)
# -l LCGPLAT, --platform LCGPLAT Platform to use (default: x86_64-slc6-gcc49-opt)
# -d, --delete delete old view before creating a new one
-# -v, --version show program's version number and exit
+# -s, --package-selection Create view only from packages specified in file. For experts only.
# -B, --enable-blacklists Enable built-in blacklists of files and packages. For experts only.
-#
-
+# -D, --dry-run Don't make changes to the file system
+# -v, --verbose Increase logging verbosity
+# -q, --quiet Decrease logging verbosity
+# --loglevel LEVEL Change logging level
+# --version show program's version number and exit
+
import argparse
+import glob
+import logging
import os
-import sys
-import shutil
import re
+import shutil
+import sys
import time
+from collections import defaultdict, OrderedDict
+
+
+class LCGViewMaker(object):
+ def __init__(self, lcgpath, lcgrel, lcgplat, pkgfile, view_destination, blacklist, greylist, pkg_blacklist,
+ dry_run):
+ super(LCGViewMaker, self).__init__()
+ self.lcg_root = lcgpath
+ self.lcg_release = lcgrel
+ self.lcg_platform = lcgplat
+ self.lcg_packages = []
+ self.view_root = view_destination
+ self.externals = defaultdict(OrderedDict)
+ self.blacklist = blacklist
+ self.greylist = greylist
+ self.pkg_blacklist = pkg_blacklist
+ self.topdir_whitelist = ['aclocal', 'cmake', 'emacs', 'fonts', 'include', 'macros', 'test', 'tests',
+ 'bin', 'config', 'etc', 'icons', 'lib', 'lib64', 'man', 'tutorials', 'share', 'src']
+ self.concatenatelist = ['easy-install.pth']
+ self.dry_run = dry_run
+
+ self.packages_to_install = {}
+
+ if pkgfile:
+ with open(pkgfile, "r") as pkgfd:
+ for line in pkgfd.readlines():
+ line = line.strip()
+ if not line or line.startswith('#'):
+ continue
+
+ data = [x.strip() for x in line.split(" ")]
+ if len(data) == 1:
+ data.append("")
+
+ self.lcg_packages.append(data)
+
+ def get_default_version(self, package):
+ if package not in self.externals:
+ logging.error("Package %s not known when choosing default version!", package)
+ return None
+
+ all_versions = self.externals[package].keys()
+ return sorted(all_versions)[-1]
+
+ def add_externals_from_release_file(self, lcg_file):
+ txt_file = open(lcg_file)
+ for line in txt_file.readlines():
+ try:
+ name, pkghash, version, home, deps = [x.strip() for x in line.split(';')]
+ except ValueError:
+ continue
+ else:
+ r = {'NAME': name, 'HASH': pkghash, 'HOME': home,
+ 'VERSION': version, 'DEPS': [x.split('-') for x in deps.split(',')]}
+ self.externals[name][version] = r
+
+ def splitall(self, path):
+ allparts = []
+ while 1:
+ parts = os.path.split(path)
+ if parts[0] == path: # sentinel for absolute paths
+ allparts.insert(0, parts[0])
+ break
+ elif parts[1] == path: # sentinel for relative paths
+ allparts.insert(0, parts[1])
+ break
+ else:
+ path = parts[0]
+ allparts.insert(0, parts[1])
+ return allparts
+
+ def add_externals_from_directory(self, directory, platform):
+ dirs = glob.glob(os.path.join(directory, '*/*', platform))
+ dirs.extend(glob.glob(os.path.join(directory, '*/*/*', platform)))
+ for d in dirs:
+ p = self.splitall(d)
+ name = p[-3]
+ version = p[-2]
+ r = {'NAME': name, 'VERSION': version, 'HOME': d.replace(directory, '.'), 'DEPS': []}
+ self.externals[name][version] = r
+
+ def write_setup(self):
+ if self.dry_run:
+ return
+
+ arch, osvers, compvers, buildtype = self.lcg_platform.split('-')
+ patt = re.compile('([a-z]+)([0-9]+)')
+ mobj = patt.match(compvers)
+ compiler = mobj.group(1)
+ version = '.'.join(list(mobj.group(2)))
+ if os.path.exists(os.path.normpath(os.path.join(self.lcg_root, '..', 'contrib', compiler))):
+ compiler = os.path.normpath(
+ os.path.join(self.lcg_root, '..', 'contrib', compiler, version, '-'.join((arch, osvers))))
+ elif os.path.exists(os.path.normpath(os.path.join(self.lcg_root, '../../../..', 'contrib', compiler))):
+ compiler = os.path.normpath(
+ os.path.join(self.lcg_root, '../../../..', 'contrib', compiler, version, '-'.join((arch, osvers))))
+ setup_sh = """#---Source this script to setup the complete environment for this LCG view
+ # Generated: @date@
+ #---Get the location this script (thisdir)
+ thisdir=$(dirname ${BASH_ARGV[0]})
+ # First the compiler
+ source @compilerlocation@/setup.sh
+ CC=`which gcc`; export CC
+ CXX=`which g++`; export CXX
+ FC=`which gfortran`; export FC
+ # then the rest...
+ if [ -z "${PATH}" ]; then
+ PATH=${thisdir}/bin; export PATH
+ else
+ PATH=${thisdir}/bin:$PATH; export PATH
+ fi
+ if [ -z "${LD_LIBRARY_PATH}" ]; then
+ LD_LIBRARY_PATH=${thisdir}/lib64:${thisdir}/lib; export LD_LIBRARY_PATH
+ else
+ LD_LIBRARY_PATH=${thisdir}/lib64:${thisdir}/lib:$LD_LIBRARY_PATH; export LD_LIBRARY_PATH
+ fi
+ if [ -z "${PYTHONPATH}" ]; then
+ PYTHONPATH=${thisdir}/lib:${thisdir}/lib/python2.7/site-packages; export PYTHONPATH
+ else
+ PYTHONPATH=${thisdir}/lib:${thisdir}/lib/python2.7/site-packages:$PYTHONPATH; export PYTHONPATH
+ fi
+ if [ -z "${MANPATH}" ]; then
+ MANPATH=${thisdir}/man; export MANPATH
+ else
+ MANPATH=${thisdir}/man:$MANPATH; export MANPATH
+ fi
+ if [ -z "${CMAKE_PREFIX_PATH}" ]; then
+ CMAKE_PREFIX_PATH=${thisdir}; export CMAKE_PREFIX_PATH
+ else
+ CMAKE_PREFIX_PATH=${thisdir}:$CMAKE_PREFIX_PATH; export CMAKE_PREFIX_PATH
+ fi
+ #---then ROOT
+ if [ -x $thisdir/bin/root ]; then
+ ROOTSYS=$(dirname $(dirname $(readlink $thisdir/bin/root))); export ROOTSYS
+ if [ -z "${ROOT_INCLUDE_PATH}" ]; then
+ ROOT_INCLUDE_PATH=${thisdir}/include; export ROOT_INCLUDE_PATH
+ else
+ ROOT_INCLUDE_PATH=${thisdir}/include:$ROOT_INCLUDE_PATH; export ROOT_INCLUDE_PATH
+ fi
+ fi
+ #---then SPARK
+ if [ -x $thisdir/bin/pyspark ]; then
+ SPARK_HOME=$(dirname $(dirname $(readlink $thisdir/bin/pyspark))); export SPARK_HOME
+ fi
+ """
+
+ setup_csh = """#---Source this script to setup the complete environment for this LCG view
+ # Generated: @date@
+ #---Get the location this script (thisdir)
+ set ARGS=($_)
+ set LSOF=`env PATH=/usr/sbin:${PATH} which lsof`
+ set thisfile="`${LSOF} -w +p $$ | grep -oE '/.*setup.csh' `"
+ if ( "$thisfile" != "" && -e ${thisfile} ) then
+ set thisdir="`dirname ${thisfile}`"
+ else if ("$ARGS" != "") then
+ set thisdir="`dirname ${ARGS[2]}`"
+ endif
+ #---First the compiler
+ source @compilerlocation@/setup.csh
+ setenv CC `which gcc`
+ setenv CXX `which g++`
+ setenv FC `which gfortran`
+ #---then the rest...
+ if ($?PATH) then
+ setenv PATH ${thisdir}/bin:${PATH}
+ else
+ setenv PATH ${thisdir}/bin
+ endif
+ if ($?LD_LIBRARY_PATH) then
+ setenv LD_LIBRARY_PATH ${thisdir}/lib64:${thisdir}/lib:${LD_LIBRARY_PATH}
+ else
+ setenv LD_LIBRARY_PATH ${thisdir}/lib64:${thisdir}/lib
+ endif
+ if ($?PYTHONPATH) then
+ setenv PYTHONPATH ${thisdir}/lib:${thisdir}/lib/python2.7/site-packages:${PYTHONPATH}
+ else
+ setenv PYTHONPATH ${thisdir}/lib:${thisdir}/lib/python2.7/site-packages
+ endif
+ if ($?CMAKE_PREFIX_PATH) then
+ setenv CMAKE_PREFIX_PATH ${thisdir}:${CMAKE_PREFIX_PATH}
+ else
+ setenv CMAKE_PREFIX_PATH $thisdir
+ endif
+ #---then ROOT
+ if ( -e $thisdir/bin/root ) then
+ set rootdir=`readlink $thisdir/bin/root`
+ set rootdir=`dirname $rootdir`
+ set rootdir=`dirname $rootdir`
+ setenv ROOTSYS $rootdir
+ if ($?ROOT_INCLUDE_PATH) then
+ setenv ROOT_INCLUDE_PATH ${thisdir}/include:${ROOT_INCLUDE_PATH}
+ else
+ setenv ROOT_INCLUDE_PATH ${thisdir}/include
+ endif
+ endif
+ #---then SPARK
+ if ( -e $thisdir/bin/pyspark ) then
+ set spark_home=`readlink $thisdir/bin/pyspark`
+ set spark_home=`dirname $spark_home`
+ set spark_home=`dirname $spark_home`
+ setenv SPARK_HOME $spark_home
+ endif
+ """
+ f = open(os.path.join(self.view_root, 'setup.sh'), 'w')
+ f.write(setup_sh.replace('@date@', time.strftime("%c")).replace('@compilerlocation@', compiler))
+ f.close()
+ f = open(os.path.join(self.view_root, 'setup.csh'), 'w')
+ f.write(setup_csh.replace('@date@', time.strftime("%c")).replace('@compilerlocation@', compiler))
+ f.close()
+
+ def install_pkg(self, pkg_name, pkg_version):
+ logging.debug('Installing {0} version {1}'.format(pkg_name, pkg_version))
+ if self.dry_run:
+ return
+
+ if self.lcg_release :
+ release_root = os.path.join(self.lcg_root, 'LCG_%s' % self.lcg_release)
+ else :
+ release_root = self.lcg_root
+
+ pkg_root = os.path.realpath(os.path.join(release_root, self.externals[pkg_name][pkg_version]['HOME']))
+ logging.info('Package {0} ver. {1}: root = {2}'.format(pkg_name, pkg_version, pkg_root))
+ for (dir_path, dirnames, filenames) in os.walk(pkg_root, followlinks=False):
+ if 'doc' in self.splitall(dir_path):
+ continue
+
+ if 'logs' in self.splitall(dir_path):
+ continue
+
+ dirpath = dir_path.replace(pkg_root, '.')
+ dirpath_s = self.splitall(dirpath)
-def get_externals(lcg_file):
- externals = {}
- txt_file = open(lcg_file)
- for line in txt_file.readlines():
- try:
- name, pkghash, version, home, deps = [x.strip() for x in line.split(';')]
- except ValueError:
- continue
+ # Elinimate any top level file or directory not in the topdir_whitelist
+ if len(dirpath_s) == 1 or dirpath_s[1] not in self.topdir_whitelist:
+ continue
+
+ view_dir = os.path.realpath(os.path.join(self.view_root, dirpath))
+
+ if not os.path.exists(view_dir):
+ # print 'Create directory', os.path.realpath(os.path.join(view_root, dirpath))
+ try:
+ os.makedirs(view_dir)
+ except OSError as e:
+ if e.errno == 20:
+ logging.warning("Target already exisits and is file: {0}".format(view_dir))
+ logging.info("Added from: {0}".format(os.path.realpath(view_dir)))
+ logging.info("Conflicts with: {0}".format(os.path.realpath(dir_path)))
+ else:
+ raise e
+
+ for f in filenames:
+ if f in self.blacklist or f.startswith('.') or f.endswith('-env.sh') or f.endswith('~'):
+ continue
+
+ source = os.path.join(dir_path, f)
+ target = os.path.join(view_dir, f)
+
+ source_rel = os.path.realpath(source).replace(self.lcg_root + os.path.sep, '')
+ target_rel = os.path.realpath(target).replace(self.lcg_root + os.path.sep, '')
+
+ if f in self.concatenatelist:
+ open(target, 'a').write(open(source, 'r').read())
+ continue
+
+ if not os.path.exists(target):
+ # print "Create symlink: {0} -> {1}".format(source, target)
+ try:
+ os.symlink(source, target)
+ except OSError as e:
+ if e.errno == 20:
+ logging.warning("Target already exisits and is file: {0}".format(f))
+ logging.info("Added from: {0}".format(os.path.realpath(target_rel)))
+ logging.info("Conflicts with: {0}".format(os.path.realpath(source_rel)))
+ else:
+ raise e
+ else:
+ if f not in self.greylist:
+ logging.warning("File already exisits: {0}".format(target))
+ logging.info("Added from: {0}".format(target_rel))
+ logging.info("Conflicts with: {0}".format(source_rel))
+ # return 1
+
+ def prepare_package(self, pkgname, pkgversion):
+ logging.debug("Add package {0} version {1}".format(pkgname, pkgversion))
+ if pkgname in self.packages_to_install:
+ logging.debug("Package {0} already added".format(pkgname))
+ return
+
+ if pkgname in self.externals:
+ if pkgversion in self.externals[pkgname] or pkgversion == '':
+ if pkgversion == '':
+ pkgversion = self.get_default_version(pkgname)
+ logging.info(
+ "Package version for {0} not specified, substituting {1}".format(pkgname, pkgversion))
+
+ self.packages_to_install[pkgname] = pkgversion
+ logging.debug("Add package {0} version {1}".format(pkgname, pkgversion))
+ logging.debug(
+ "Add {0} dependencies: {1}".format(len(self.externals[pkgname][pkgversion]['DEPS']),
+ self.externals[pkgname][pkgversion]['DEPS']))
+ for dep in self.externals[pkgname][pkgversion]['DEPS']:
+ if len(dep) != 2:
+ continue
+
+ depname, dephash = dep
+ depver = ''
+ for depv, depd in self.externals[depname].iteritems():
+ if depd['HASH'] == dephash:
+ depver = depv
+ break
+
+ if depver != '':
+ logging.debug("Add dependency: {0} version {1}".format(depname, depver))
+ for i, pkg in enumerate(self.lcg_packages):
+ if pkg[0] == depname and pkg[1] != depver:
+ logging.info(
+ "Dependency override: package {0}, version {1} -> {2}".format(depname, pkg[1],
+ depver))
+ self.lcg_packages.pop(i)
+ self.lcg_packages.append([depname, depver])
+ if depname in self.packages_to_install:
+ self.packages_to_install[depname] = depver
+ break
+ else:
+ logging.critical(
+ "Package {0} with hash {1} not found in release {2} when processing package "
+ "{3} version {4}".format(depname, dephash, self.lcg_release, pkgname, pkgversion))
+ return 1
+ else:
+ logging.critical(
+ "Version {1} of package {0} not found in release {2}!".format(pkgname, pkgversion,
+ self.lcg_release))
+ logging.info("Possible versions: {0}".format(", ".join(self.externals[pkgname].keys())))
+ return 1
else:
- r = {'NAME': name, 'HASH': pkghash, 'HOME': home,
- 'VERSION': version}
- externals.update({name: r})
-
- return externals
-
-
-def splitall(path):
- allparts = []
- while 1:
- parts = os.path.split(path)
- if parts[0] == path: # sentinel for absolute paths
- allparts.insert(0, parts[0])
- break
- elif parts[1] == path: # sentinel for relative paths
- allparts.insert(0, parts[1])
- break
+ logging.critical("Package {0} not found in release {1}!".format(pkgname, self.lcg_release))
+ return 1
+
+ pass
+
+ def make_view(self):
+ # ---Check whether the LCG release actually exists, otherwise take all the packages in the root directory
+ logging.debug("make_view start")
+ release_root = os.path.join(self.lcg_root, 'LCG_%s' % self.lcg_release)
+ if os.path.exists(release_root):
+ lcg_file = os.path.join(release_root, 'LCG_externals_%s.txt' % self.lcg_platform)
+ mc_file = os.path.join(release_root, 'LCG_generators_%s.txt' % self.lcg_platform)
+ self.add_externals_from_release_file(lcg_file)
+ self.add_externals_from_release_file(mc_file)
else:
- path = parts[0]
- allparts.insert(0, parts[1])
- return allparts
-
-
-def write_setup(lcg_root, view_root, platform):
- arch, osvers, compvers, buildtype = platform.split('-')
- patt = re.compile('([a-z]+)([0-9]+)')
- mobj = patt.match(compvers)
- compiler = mobj.group(1)
- version = '.'.join(list(mobj.group(2)))
- if os.path.exists(os.path.normpath(os.path.join(lcg_root,'..','contrib',compiler))):
- compiler = os.path.normpath(os.path.join(lcg_root,'..','contrib',compiler,version,'-'.join((arch, osvers))))
- setup_sh = """#---Source this script to setup the complete environment for this LCG view
-# Generated: @date@
-#---Get the location this script (thisroot)
-thisdir=$(dirname ${BASH_ARGV[0]})
-# First the compiler
-source @compilerlocation@/setup.sh
-CC=`which gcc`; export CC
-CXX=`which g++`; export CXX
-FC=`which gfortran`; export FC
-# then the rest...
-if [ -z "${PATH}" ]; then
- PATH=${thisdir}/bin; export PATH
-else
- PATH=${thisdir}/bin:$PATH; export PATH
-fi
-if [ -z "${LD_LIBRARY_PATH}" ]; then
- LD_LIBRARY_PATH=${thisdir}/lib64:${thisdir}/lib; export LD_LIBRARY_PATH
-else
- LD_LIBRARY_PATH=${thisdir}/lib64:${thisdir}/lib:$LD_LIBRARY_PATH; export LD_LIBRARY_PATH
-fi
-if [ -z "${PYTHONPATH}" ]; then
- PYTHONPATH=${thisdir}/lib:${thisdir}/lib/python2.7/site-packages; export PYTHONPATH
-else
- PYTHONPATH=${thisdir}/lib:${thisdir}/lib/python2.7/site-packages:$PYTHONPATH; export PYTHONPATH
-fi
-if [ -z "${MANPATH}" ]; then
- MANPATH=${thisdir}/man; export MANPATH
-else
- MANPATH=${thisdir}/man:$MANPATH; export MANPATH
-fi
-if [ -z "${CMAKE_PREFIX_PATH}" ]; then
- CMAKE_PREFIX_PATH=${thisdir}; export CMAKE_PREFIX_PATH
-else
- CMAKE_PREFIX_PATH=${thisdir}:$CMAKE_PREFIX_PATH; export CMAKE_PREFIX_PATH
-fi
-#---then ROOT
-ROOTSYS=$(dirname $(dirname $(readlink $thisdir/bin/root))); export ROOTSYS
-if [ -z "${ROOT_INCLUDE_PATH}" ]; then
- ROOT_INCLUDE_PATH=${thisdir}/include; export ROOT_INCLUDE_PATH
-else
- ROOT_INCLUDE_PATH=${thisdir}/include:$ROOT_INCLUDE_PATH; export ROOT_INCLUDE_PATH
-fi
-"""
+ release_root = self.lcg_root
+ self.add_externals_from_directory(release_root, self.lcg_platform)
+ self.lcg_release = None
+
+ logging.debug("Loaded {0} externals".format(len(self.externals)))
+ self.packages_to_install = {}
+
+ if self.lcg_packages:
+ logging.debug("Build view from {0} packages".format(len(self.lcg_packages)))
+ for pkgname, pkgversion in self.lcg_packages:
+ self.prepare_package(pkgname, pkgversion)
+ else:
+ logging.debug("No package list specified")
+ for pkgname in self.externals:
+ self.prepare_package(pkgname, max(self.externals[pkgname]))
+
+ logging.debug("Final list contains {0} packages".format(len(self.packages_to_install)))
+ for pkgname in self.pkg_blacklist:
+ if pkgname in self.packages_to_install:
+ self.packages_to_install.pop(pkgname)
+
+ logging.debug("Filtered list contains {0} packages".format(len(self.packages_to_install)))
+
+ for pkgname, pkgversion in self.packages_to_install.iteritems():
+ self.install_pkg(pkgname, pkgversion)
+
+ # ---Finalize the view with additional operations------------------------------------------------------------
+ self.write_setup()
- setup_csh = """#---Source this script to setup the complete environment for this LCG view
-# Generated: @date@
- #---Get the location this script (thisdir)
-set ARGS=($_)
-set LSOF=`env PATH=/usr/sbin:${PATH} which lsof`
-set thisfile="`${LSOF} -w +p $$ | grep -oE '/.*setup.csh' `"
-if ( "$thisfile" != "" && -e ${thisfile} ) then
- set thisdir="`dirname ${thisfile}`"
-else if ("$ARGS" != "") then
- set thisdir="`dirname ${ARGS[2]}`"
-endif
-#---First the compiler
-source @compilerlocation@/setup.csh
-setenv CC `which gcc`
-setenv CXX `which g++`
-setenv FC `which gfortran`
-#---then the rest...
-if ($?PATH) then
- setenv PATH ${thisdir}/bin:${PATH}
-else
- setenv PATH ${thisdir}/bin
-endif
-if ($?LD_LIBRARY_PATH) then
- setenv LD_LIBRARY_PATH ${thisdir}/lib64:${thisdir}/lib:${LD_LIBRARY_PATH}
-else
- setenv LD_LIBRARY_PATH ${thisdir}/lib64:${thisdir}/lib
-endif
-if ($?PYTHONPATH) then
- setenv PYTHONPATH ${thisdir}/lib:${thisdir}/lib/python2.7/site-packages:${PYTHONPATH}
-else
- setenv PYTHONPATH ${thisdir}/lib:${thisdir}/lib/python2.7/site-packages
-endif
-if ($?CMAKE_PREFIX_PATH) then
- setenv CMAKE_PREFIX_PATH ${thisdir}:${CMAKE_PREFIX_PATH}
-else
- setenv CMAKE_PREFIX_PATH $thisdir
-endif
-#---then ROOT
-set rootdir=`readlink $thisdir/bin/root`
-set rootdir=`dirname $rootdir`
-set rootdir=`dirname $rootdir`
-setenv ROOTSYS $rootdir
-if ($?ROOT_INCLUDE_PATH) then
- setenv ROOT_INCLUDE_PATH ${thisdir}/include:${ROOT_INCLUDE_PATH}
-else
- setenv ROOT_INCLUDE_PATH ${thisdir}/include
-endif
-"""
- f = open(os.path.join(view_root,'setup.sh'), 'w')
- f.write(setup_sh.replace('@date@',time.strftime("%c")).replace('@compilerlocation@',compiler))
- f.close()
- f = open(os.path.join(view_root,'setup.csh'), 'w')
- f.write(setup_csh.replace('@date@',time.strftime("%c")).replace('@compilerlocation@',compiler))
- f.close()
def main():
helpstring = """{0} [options] <view_destination>
@@ -176,116 +422,70 @@ This package creates a "view" of LCG release in folder <view_destination>.
parser.add_argument('-l', '--lcgpath', help="top directory of LCG releases (default: /afs/cern.ch/sw/lcg/releases)",
action="store",
default='/afs/cern.ch/sw/lcg/releases', dest='lcgpath')
- parser.add_argument('-r', '--release', help="LCG release number (default: 80)", action="store", default=80, dest="lcgrel")
+ parser.add_argument('-r', '--release', help="LCG release number (default: 80)", action="store", default=80,
+ dest="lcgrel")
parser.add_argument('-p', '--platform', help="Platform to use (default: x86_64-slc6-gcc49-opt)", action="store",
default='x86_64-slc6-gcc49-opt', dest='lcgplat')
parser.add_argument('-d', '--delete', help="delete old view before creating a new one", action="store_true",
default=False, dest='delview')
- parser.add_argument('-B', '--enable-blacklists', help='Enable built-in blacklists of files and packages. For experts only.',
- action="store_true", default=False, dest='bl_enabled')
- parser.add_argument('-v', '--version', action='version', version='%(prog)s 0.3')
+ parser.add_argument('-B', '--enable-blacklists',
+ help='Enable built-in blacklists of files and packages. For experts only.',
+ action="store_true", dest='bl_enabled')
+ parser.add_argument('-s', '--package-selection',
+ help='Create view only from packages specified in file. For experts only.',
+ action='store', dest='pkgfile')
+ parser.add_argument('-D', '--dry-run',
+ help="Don't delete or link anything. For debugging only.", action='store_true', dest='dry_run')
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument('-v', '--verbose', action='count', dest='verbose_level', help='Increase logging verbosity',
+ default=0)
+ group.add_argument('-q', '--quiet', action='count', dest='quiet_level', help='Decrease logging verbosity',
+ default=0)
+ group.add_argument('--loglevel', choices=['ERROR', 'WARNING', 'INFO', 'DEBUG'], action='store',
+ dest='loglvl', default='INFO', help='Set logging level (default: INFO)')
args = parser.parse_args()
+ parser.add_argument('--version', action='version', version='%(prog)s 0.5')
+
+ loglvl = min(logging.ERROR,
+ max(logging.DEBUG,
+ getattr(logging, args.loglvl.upper()) - args.verbose_level * 10 + args.quiet_level * 10))
- lcg_root = args.lcgpath
- lcg_release = args.lcgrel
- lcg_platform = args.lcgplat
- view_root = args.view_destination[0]
+ logging.basicConfig(format=u'*** %(levelname)s: %(message)s', level=loglvl)
+ # print "Logging config: {0} + {1} - {2} = {3}".format(args.loglvl.upper(), args.verbose_level * 10,
+ # args.quiet_level * 10, logging.getLevelName(loglvl))
if args.bl_enabled:
blacklist = ['version.txt', '.filelist', 'README',
'LICENSE', 'decimal.h', 'atan2.h', 'project.cmt', 'INSTALL']
greylist = ['site.py', 'site.pyc', 'easy_install', 'easy_install-2.7', 'setuptools.pth']
- pkg_blacklist = ['neurobayes_expert', 'vdt', 'cmt', 'Qt5', 'xrootd_python', 'powheg-box', 'sherpa-mpich2', 'yamlcpp']
+ pkg_blacklist = ['neurobayes_expert', 'vdt', 'cmt', 'Qt5', 'xrootd_python', 'powheg-box', 'sherpa-mpich2',
+ 'yamlcpp']
else:
blacklist = ['version.txt']
- greylist = []
+ greylist = []
pkg_blacklist = []
- topdir_whitelist = ['aclocal', 'cmake', 'emacs', 'fonts', 'include', 'macros', 'test', 'tests',
- 'bin', 'config', 'etc', 'icons', 'lib', 'lib64', 'man', 'tutorials', 'share']
+ topdir_whitelist = ['aclocal', 'cmake', 'emacs', 'fonts', 'include', 'macros', 'test', 'tests',
+ 'bin', 'config', 'etc', 'icons', 'lib', 'lib64', 'man', 'tutorials', 'share', 'src']
concatenatelist = ['easy-install.pth']
- if args.delview and os.path.exists(view_root):
- shutil.rmtree(view_root, True)
-
- if not os.path.exists(view_root):
- os.makedirs(view_root)
-
- release_root = os.path.join(lcg_root, 'LCG_%s' % lcg_release)
- lcg_file = os.path.join(release_root, 'LCG_externals_%s.txt' % lcg_platform)
- mc_file = os.path.join(release_root, 'LCG_generators_%s.txt' % lcg_platform)
-
- externals = get_externals(lcg_file)
- externals.update(get_externals(mc_file))
- for pkg in externals:
- if pkg in pkg_blacklist:
- continue
-
- pkg_root = os.path.realpath(os.path.join(release_root, externals[pkg]['HOME']))
- print 'Package {0}: root = {1}'.format(pkg, pkg_root)
-
- for (dir_path, dirnames, filenames) in os.walk(pkg_root, followlinks=False):
- if 'doc' in splitall(dir_path):
- continue
-
- if 'logs' in splitall(dir_path):
- continue
-
- dirpath = dir_path.replace(pkg_root, '.')
- dirpath_s = splitall(dirpath)
-
- # Elinimate any top level file or directory not in the topdir_whitelist
- if len(dirpath_s) == 1 or dirpath_s[1] not in topdir_whitelist :
- continue
-
- view_dir = os.path.realpath(os.path.join(view_root, dirpath))
+ if args.delview and os.path.exists(args.view_destination[0]):
+ shutil.rmtree(args.view_destination[0], True)
- if not os.path.exists(view_dir):
- # print 'Create directory', os.path.realpath(os.path.join(view_root, dirpath))
- try:
- os.makedirs(view_dir)
- except OSError as e:
- if e.errno == 20:
- print "*** WARNING: Target already exisits and is file: {0} ***".format(view_dir)
- print "*** NOTICE: Added from: {0} ***".format(os.path.realpath(view_dir))
- print "*** NOTICE: Conflicts with: {0} ***".format(os.path.realpath(dir_path))
- else:
- raise e
+ if not os.path.exists(args.view_destination[0]):
+ os.makedirs(args.view_destination[0])
- # continue
- for f in filenames:
- if f in blacklist or f.startswith('.') or f.endswith('-env.sh') or f.endswith('~'):
- continue
+ if args.pkgfile and not os.path.exists(args.pkgfile):
+ logging.critical("Package selection file {0} does not exist!".format(args.pkgfile))
+ return 1
- source = os.path.join(dir_path, f)
- target = os.path.join(view_dir, f)
+ v = LCGViewMaker(lcgpath=args.lcgpath, lcgrel=args.lcgrel, lcgplat=args.lcgplat, pkgfile=args.pkgfile,
+ view_destination=args.view_destination[0], blacklist=blacklist, greylist=greylist,
+ pkg_blacklist=pkg_blacklist, dry_run=args.dry_run)
- source_rel = os.path.realpath(source).replace(lcg_root + os.path.sep, '')
- target_rel = os.path.realpath(target).replace(lcg_root + os.path.sep, '')
-
- if f in concatenatelist:
- open(target,'a').write(open(source,'r').read())
- continue
+ return v.make_view()
- if not os.path.exists(target):
- # print "Create symlink: {0} -> {1}".format(source, target)
- try:
- os.symlink(source, target)
- except OSError as e:
- if e.errno == 20:
- print "*** WARNING: Target already exisits and is file: {0} ***".format(f)
- print "*** NOTICE: Added from: {0} ***".format(target_rel)
- print "*** NOTICE: Conflicts with: {0} ***".format(source_rel)
- else:
- raise e
- else:
- if f not in greylist:
- print "*** WARNING: File already exisits: {0} ***".format(target)
- print "*** NOTICE: Added from: {0} ***".format(target_rel)
- print "*** NOTICE: Conflicts with: {0} ***".format(source_rel)
- #return 1
- write_setup(lcg_root, view_root, lcg_platform)
- return 0
+ # return 0
if __name__ == '__main__':
--
GitLab