From c4b1b6b94e16cc36a9b3df3776c90ebedf42ad8c Mon Sep 17 00:00:00 2001
From: Ivan Razumov <ivan.razumov@cern.ch>
Date: Thu, 2 Jun 2016 15:01:14 +0200
Subject: [PATCH] Copy create_lcg_view script from master
---
cmake/scripts/create_lcg_view.py | 492 +++++++++++++++++++++++++++++++
1 file changed, 492 insertions(+)
create mode 100755 cmake/scripts/create_lcg_view.py
diff --git a/cmake/scripts/create_lcg_view.py b/cmake/scripts/create_lcg_view.py
new file mode 100755
index 0000000000..11f6e4f8de
--- /dev/null
+++ b/cmake/scripts/create_lcg_view.py
@@ -0,0 +1,492 @@
+#!/usr/bin/env python
+
+# Author: Ivan Razumov
+# Usage: lcgview.py [options] <view_root>
+# This package creates a "view" of LCG release in folder <view_root>.
+# optional arguments:
+# -h, --help show this help message and exit
+# -p LCGPATH, --lcgpath LCGPATH top directory of LCG releases (default: /afs/cern.ch/sw/lcg/releases)
+# -r LCGREL, --release LCGREL LCG release number (default: 80)
+# -l LCGPLAT, --platform LCGPLAT Platform to use (default: x86_64-slc6-gcc49-opt)
+# -d, --delete delete old view before creating a new one
+# -s, --package-selection Create view only from packages specified in file. For experts only.
+# -B, --enable-blacklists Enable built-in blacklists of files and packages. For experts only.
+# -D, --dry-run Don't make changes to the file system
+# -v, --verbose Increase logging verbosity
+# -q, --quiet Decrease logging verbosity
+# --loglevel LEVEL Change logging level
+# --version show program's version number and exit
+
+import argparse
+import glob
+import logging
+import os
+import re
+import shutil
+import sys
+import time
+from collections import defaultdict, OrderedDict
+
+
+class LCGViewMaker(object):
+ def __init__(self, lcgpath, lcgrel, lcgplat, pkgfile, view_destination, blacklist, greylist, pkg_blacklist,
+ dry_run):
+ super(LCGViewMaker, self).__init__()
+ self.lcg_root = lcgpath
+ self.lcg_release = lcgrel
+ self.lcg_platform = lcgplat
+ self.lcg_packages = []
+ self.view_root = view_destination
+ self.externals = defaultdict(OrderedDict)
+ self.blacklist = blacklist
+ self.greylist = greylist
+ self.pkg_blacklist = pkg_blacklist
+ self.topdir_whitelist = ['aclocal', 'cmake', 'emacs', 'fonts', 'include', 'macros', 'test', 'tests',
+ 'bin', 'config', 'etc', 'icons', 'lib', 'lib64', 'man', 'tutorials', 'share', 'src']
+ self.concatenatelist = ['easy-install.pth']
+ self.dry_run = dry_run
+
+ self.packages_to_install = {}
+
+ if pkgfile:
+ with open(pkgfile, "r") as pkgfd:
+ for line in pkgfd.readlines():
+ line = line.strip()
+ if not line or line.startswith('#'):
+ continue
+
+ data = [x.strip() for x in line.split(" ")]
+ if len(data) == 1:
+ data.append("")
+
+ self.lcg_packages.append(data)
+
+ def get_default_version(self, package):
+ if package not in self.externals:
+ logging.error("Package %s not known when choosing default version!", package)
+ return None
+
+ all_versions = self.externals[package].keys()
+ return sorted(all_versions)[-1]
+
+ def add_externals_from_release_file(self, lcg_file):
+ txt_file = open(lcg_file)
+ for line in txt_file.readlines():
+ try:
+ name, pkghash, version, home, deps = [x.strip() for x in line.split(';')]
+ except ValueError:
+ continue
+ else:
+ r = {'NAME': name, 'HASH': pkghash, 'HOME': home,
+ 'VERSION': version, 'DEPS': [x.split('-') for x in deps.split(',')]}
+ self.externals[name][version] = r
+
+ def splitall(self, path):
+ allparts = []
+ while 1:
+ parts = os.path.split(path)
+ if parts[0] == path: # sentinel for absolute paths
+ allparts.insert(0, parts[0])
+ break
+ elif parts[1] == path: # sentinel for relative paths
+ allparts.insert(0, parts[1])
+ break
+ else:
+ path = parts[0]
+ allparts.insert(0, parts[1])
+ return allparts
+
+ def add_externals_from_directory(self, directory, platform):
+ dirs = glob.glob(os.path.join(directory, '*/*', platform))
+ dirs.extend(glob.glob(os.path.join(directory, '*/*/*', platform)))
+ for d in dirs:
+ p = self.splitall(d)
+ name = p[-3]
+ version = p[-2]
+ r = {'NAME': name, 'VERSION': version, 'HOME': d.replace(directory, '.'), 'DEPS': []}
+ self.externals[name][version] = r
+
+ def write_setup(self):
+ if self.dry_run:
+ return
+
+ arch, osvers, compvers, buildtype = self.lcg_platform.split('-')
+ patt = re.compile('([a-z]+)([0-9]+)')
+ mobj = patt.match(compvers)
+ compiler = mobj.group(1)
+ version = '.'.join(list(mobj.group(2)))
+ if os.path.exists(os.path.normpath(os.path.join(self.lcg_root, '..', 'contrib', compiler))):
+ compiler = os.path.normpath(
+ os.path.join(self.lcg_root, '..', 'contrib', compiler, version, '-'.join((arch, osvers))))
+ elif os.path.exists(os.path.normpath(os.path.join(self.lcg_root, '../../../..', 'contrib', compiler))):
+ compiler = os.path.normpath(
+ os.path.join(self.lcg_root, '../../../..', 'contrib', compiler, version, '-'.join((arch, osvers))))
+ setup_sh = """#---Source this script to setup the complete environment for this LCG view
+ # Generated: @date@
+ #---Get the location this script (thisdir)
+ thisdir=$(dirname ${BASH_ARGV[0]})
+ # First the compiler
+ source @compilerlocation@/setup.sh
+ CC=`which gcc`; export CC
+ CXX=`which g++`; export CXX
+ FC=`which gfortran`; export FC
+ # then the rest...
+ if [ -z "${PATH}" ]; then
+ PATH=${thisdir}/bin; export PATH
+ else
+ PATH=${thisdir}/bin:$PATH; export PATH
+ fi
+ if [ -z "${LD_LIBRARY_PATH}" ]; then
+ LD_LIBRARY_PATH=${thisdir}/lib64:${thisdir}/lib; export LD_LIBRARY_PATH
+ else
+ LD_LIBRARY_PATH=${thisdir}/lib64:${thisdir}/lib:$LD_LIBRARY_PATH; export LD_LIBRARY_PATH
+ fi
+ if [ -z "${PYTHONPATH}" ]; then
+ PYTHONPATH=${thisdir}/lib:${thisdir}/lib/python2.7/site-packages; export PYTHONPATH
+ else
+ PYTHONPATH=${thisdir}/lib:${thisdir}/lib/python2.7/site-packages:$PYTHONPATH; export PYTHONPATH
+ fi
+ if [ -z "${MANPATH}" ]; then
+ MANPATH=${thisdir}/man; export MANPATH
+ else
+ MANPATH=${thisdir}/man:$MANPATH; export MANPATH
+ fi
+ if [ -z "${CMAKE_PREFIX_PATH}" ]; then
+ CMAKE_PREFIX_PATH=${thisdir}; export CMAKE_PREFIX_PATH
+ else
+ CMAKE_PREFIX_PATH=${thisdir}:$CMAKE_PREFIX_PATH; export CMAKE_PREFIX_PATH
+ fi
+ #---then ROOT
+ if [ -x $thisdir/bin/root ]; then
+ ROOTSYS=$(dirname $(dirname $(readlink $thisdir/bin/root))); export ROOTSYS
+ if [ -z "${ROOT_INCLUDE_PATH}" ]; then
+ ROOT_INCLUDE_PATH=${thisdir}/include; export ROOT_INCLUDE_PATH
+ else
+ ROOT_INCLUDE_PATH=${thisdir}/include:$ROOT_INCLUDE_PATH; export ROOT_INCLUDE_PATH
+ fi
+ fi
+ #---then SPARK
+ if [ -x $thisdir/bin/pyspark ]; then
+ SPARK_HOME=$(dirname $(dirname $(readlink $thisdir/bin/pyspark))); export SPARK_HOME
+ fi
+ """
+
+ setup_csh = """#---Source this script to setup the complete environment for this LCG view
+ # Generated: @date@
+ #---Get the location this script (thisdir)
+ set ARGS=($_)
+ set LSOF=`env PATH=/usr/sbin:${PATH} which lsof`
+ set thisfile="`${LSOF} -w +p $$ | grep -oE '/.*setup.csh' `"
+ if ( "$thisfile" != "" && -e ${thisfile} ) then
+ set thisdir="`dirname ${thisfile}`"
+ else if ("$ARGS" != "") then
+ set thisdir="`dirname ${ARGS[2]}`"
+ endif
+ #---First the compiler
+ source @compilerlocation@/setup.csh
+ setenv CC `which gcc`
+ setenv CXX `which g++`
+ setenv FC `which gfortran`
+ #---then the rest...
+ if ($?PATH) then
+ setenv PATH ${thisdir}/bin:${PATH}
+ else
+ setenv PATH ${thisdir}/bin
+ endif
+ if ($?LD_LIBRARY_PATH) then
+ setenv LD_LIBRARY_PATH ${thisdir}/lib64:${thisdir}/lib:${LD_LIBRARY_PATH}
+ else
+ setenv LD_LIBRARY_PATH ${thisdir}/lib64:${thisdir}/lib
+ endif
+ if ($?PYTHONPATH) then
+ setenv PYTHONPATH ${thisdir}/lib:${thisdir}/lib/python2.7/site-packages:${PYTHONPATH}
+ else
+ setenv PYTHONPATH ${thisdir}/lib:${thisdir}/lib/python2.7/site-packages
+ endif
+ if ($?CMAKE_PREFIX_PATH) then
+ setenv CMAKE_PREFIX_PATH ${thisdir}:${CMAKE_PREFIX_PATH}
+ else
+ setenv CMAKE_PREFIX_PATH $thisdir
+ endif
+ #---then ROOT
+ if ( -e $thisdir/bin/root ) then
+ set rootdir=`readlink $thisdir/bin/root`
+ set rootdir=`dirname $rootdir`
+ set rootdir=`dirname $rootdir`
+ setenv ROOTSYS $rootdir
+ if ($?ROOT_INCLUDE_PATH) then
+ setenv ROOT_INCLUDE_PATH ${thisdir}/include:${ROOT_INCLUDE_PATH}
+ else
+ setenv ROOT_INCLUDE_PATH ${thisdir}/include
+ endif
+ endif
+ #---then SPARK
+ if ( -e $thisdir/bin/pyspark ) then
+ set spark_home=`readlink $thisdir/bin/pyspark`
+ set spark_home=`dirname $spark_home`
+ set spark_home=`dirname $spark_home`
+ setenv SPARK_HOME $spark_home
+ endif
+ """
+ f = open(os.path.join(self.view_root, 'setup.sh'), 'w')
+ f.write(setup_sh.replace('@date@', time.strftime("%c")).replace('@compilerlocation@', compiler))
+ f.close()
+ f = open(os.path.join(self.view_root, 'setup.csh'), 'w')
+ f.write(setup_csh.replace('@date@', time.strftime("%c")).replace('@compilerlocation@', compiler))
+ f.close()
+
+ def install_pkg(self, pkg_name, pkg_version):
+ logging.debug('Installing {0} version {1}'.format(pkg_name, pkg_version))
+ if self.dry_run:
+ return
+
+ if self.lcg_release :
+ release_root = os.path.join(self.lcg_root, 'LCG_%s' % self.lcg_release)
+ else :
+ release_root = self.lcg_root
+
+ pkg_root = os.path.realpath(os.path.join(release_root, self.externals[pkg_name][pkg_version]['HOME']))
+ logging.info('Package {0} ver. {1}: root = {2}'.format(pkg_name, pkg_version, pkg_root))
+
+ for (dir_path, dirnames, filenames) in os.walk(pkg_root, followlinks=False):
+ if 'doc' in self.splitall(dir_path):
+ continue
+
+ if 'logs' in self.splitall(dir_path):
+ continue
+
+ dirpath = dir_path.replace(pkg_root, '.')
+ dirpath_s = self.splitall(dirpath)
+
+ # Elinimate any top level file or directory not in the topdir_whitelist
+ if len(dirpath_s) == 1 or dirpath_s[1] not in self.topdir_whitelist:
+ continue
+
+ view_dir = os.path.realpath(os.path.join(self.view_root, dirpath))
+
+ if not os.path.exists(view_dir):
+ # print 'Create directory', os.path.realpath(os.path.join(view_root, dirpath))
+ try:
+ os.makedirs(view_dir)
+ except OSError as e:
+ if e.errno == 20:
+ logging.warning("Target already exisits and is file: {0}".format(view_dir))
+ logging.info("Added from: {0}".format(os.path.realpath(view_dir)))
+ logging.info("Conflicts with: {0}".format(os.path.realpath(dir_path)))
+ else:
+ raise e
+
+ for f in filenames:
+ if f in self.blacklist or f.startswith('.') or f.endswith('-env.sh') or f.endswith('~'):
+ continue
+
+ source = os.path.join(dir_path, f)
+ target = os.path.join(view_dir, f)
+
+ source_rel = os.path.realpath(source).replace(self.lcg_root + os.path.sep, '')
+ target_rel = os.path.realpath(target).replace(self.lcg_root + os.path.sep, '')
+
+ if f in self.concatenatelist:
+ open(target, 'a').write(open(source, 'r').read())
+ continue
+
+ if not os.path.exists(target):
+ # print "Create symlink: {0} -> {1}".format(source, target)
+ try:
+ os.symlink(source, target)
+ except OSError as e:
+ if e.errno == 20:
+ logging.warning("Target already exisits and is file: {0}".format(f))
+ logging.info("Added from: {0}".format(os.path.realpath(target_rel)))
+ logging.info("Conflicts with: {0}".format(os.path.realpath(source_rel)))
+ else:
+ raise e
+ else:
+ if f not in self.greylist:
+ logging.warning("File already exisits: {0}".format(target))
+ logging.info("Added from: {0}".format(target_rel))
+ logging.info("Conflicts with: {0}".format(source_rel))
+ # return 1
+
+ def prepare_package(self, pkgname, pkgversion):
+ logging.debug("Add package {0} version {1}".format(pkgname, pkgversion))
+ if pkgname in self.packages_to_install:
+ logging.debug("Package {0} already added".format(pkgname))
+ return
+
+ if pkgname in self.externals:
+ if pkgversion in self.externals[pkgname] or pkgversion == '':
+ if pkgversion == '':
+ pkgversion = self.get_default_version(pkgname)
+ logging.info(
+ "Package version for {0} not specified, substituting {1}".format(pkgname, pkgversion))
+
+ self.packages_to_install[pkgname] = pkgversion
+ logging.debug("Add package {0} version {1}".format(pkgname, pkgversion))
+ logging.debug(
+ "Add {0} dependencies: {1}".format(len(self.externals[pkgname][pkgversion]['DEPS']),
+ self.externals[pkgname][pkgversion]['DEPS']))
+ for dep in self.externals[pkgname][pkgversion]['DEPS']:
+ if len(dep) != 2:
+ continue
+
+ depname, dephash = dep
+ depver = ''
+ for depv, depd in self.externals[depname].iteritems():
+ if depd['HASH'] == dephash:
+ depver = depv
+ break
+
+ if depver != '':
+ logging.debug("Add dependency: {0} version {1}".format(depname, depver))
+ for i, pkg in enumerate(self.lcg_packages):
+ if pkg[0] == depname and pkg[1] != depver:
+ logging.info(
+ "Dependency override: package {0}, version {1} -> {2}".format(depname, pkg[1],
+ depver))
+ self.lcg_packages.pop(i)
+ self.lcg_packages.append([depname, depver])
+ if depname in self.packages_to_install:
+ self.packages_to_install[depname] = depver
+ break
+ else:
+ logging.critical(
+ "Package {0} with hash {1} not found in release {2} when processing package "
+ "{3} version {4}".format(depname, dephash, self.lcg_release, pkgname, pkgversion))
+ return 1
+ else:
+ logging.critical(
+ "Version {1} of package {0} not found in release {2}!".format(pkgname, pkgversion,
+ self.lcg_release))
+ logging.info("Possible versions: {0}".format(", ".join(self.externals[pkgname].keys())))
+ return 1
+ else:
+ logging.critical("Package {0} not found in release {1}!".format(pkgname, self.lcg_release))
+ return 1
+
+ pass
+
+ def make_view(self):
+ # ---Check whether the LCG release actually exists, otherwise take all the packages in the root directory
+ logging.debug("make_view start")
+ release_root = os.path.join(self.lcg_root, 'LCG_%s' % self.lcg_release)
+ if os.path.exists(release_root):
+ lcg_file = os.path.join(release_root, 'LCG_externals_%s.txt' % self.lcg_platform)
+ mc_file = os.path.join(release_root, 'LCG_generators_%s.txt' % self.lcg_platform)
+ self.add_externals_from_release_file(lcg_file)
+ self.add_externals_from_release_file(mc_file)
+ else:
+ release_root = self.lcg_root
+ self.add_externals_from_directory(release_root, self.lcg_platform)
+ self.lcg_release = None
+
+ logging.debug("Loaded {0} externals".format(len(self.externals)))
+ self.packages_to_install = {}
+
+ if self.lcg_packages:
+ logging.debug("Build view from {0} packages".format(len(self.lcg_packages)))
+ for pkgname, pkgversion in self.lcg_packages:
+ self.prepare_package(pkgname, pkgversion)
+ else:
+ logging.debug("No package list specified")
+ for pkgname in self.externals:
+ self.prepare_package(pkgname, max(self.externals[pkgname]))
+
+ logging.debug("Final list contains {0} packages".format(len(self.packages_to_install)))
+ for pkgname in self.pkg_blacklist:
+ if pkgname in self.packages_to_install:
+ self.packages_to_install.pop(pkgname)
+
+ logging.debug("Filtered list contains {0} packages".format(len(self.packages_to_install)))
+
+ for pkgname, pkgversion in self.packages_to_install.iteritems():
+ self.install_pkg(pkgname, pkgversion)
+
+ # ---Finalize the view with additional operations------------------------------------------------------------
+ self.write_setup()
+
+
+def main():
+ helpstring = """{0} [options] <view_destination>
+
+This package creates a "view" of LCG release in folder <view_destination>.
+"""
+
+ # lcg_root = '/afs/cern.ch/sw/lcg/releases'
+ # lcg_release = 79
+ # lcg_platform = 'x86_64-slc6-gcc49-opt'
+ # view_root = '/tmp/view_{0}{1}'.format(lcg_release, lcg_platform)
+
+ parser = argparse.ArgumentParser(usage=helpstring.format(sys.argv[0]))
+ parser.add_argument('view_destination', metavar='view_destination', nargs='+', help=argparse.SUPPRESS)
+ parser.add_argument('-l', '--lcgpath', help="top directory of LCG releases (default: /afs/cern.ch/sw/lcg/releases)",
+ action="store",
+ default='/afs/cern.ch/sw/lcg/releases', dest='lcgpath')
+ parser.add_argument('-r', '--release', help="LCG release number (default: 80)", action="store", default=80,
+ dest="lcgrel")
+ parser.add_argument('-p', '--platform', help="Platform to use (default: x86_64-slc6-gcc49-opt)", action="store",
+ default='x86_64-slc6-gcc49-opt', dest='lcgplat')
+ parser.add_argument('-d', '--delete', help="delete old view before creating a new one", action="store_true",
+ default=False, dest='delview')
+ parser.add_argument('-B', '--enable-blacklists',
+ help='Enable built-in blacklists of files and packages. For experts only.',
+ action="store_true", dest='bl_enabled')
+ parser.add_argument('-s', '--package-selection',
+ help='Create view only from packages specified in file. For experts only.',
+ action='store', dest='pkgfile')
+ parser.add_argument('-D', '--dry-run',
+ help="Don't delete or link anything. For debugging only.", action='store_true', dest='dry_run')
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument('-v', '--verbose', action='count', dest='verbose_level', help='Increase logging verbosity',
+ default=0)
+ group.add_argument('-q', '--quiet', action='count', dest='quiet_level', help='Decrease logging verbosity',
+ default=0)
+ group.add_argument('--loglevel', choices=['ERROR', 'WARNING', 'INFO', 'DEBUG'], action='store',
+ dest='loglvl', default='INFO', help='Set logging level (default: INFO)')
+ args = parser.parse_args()
+ parser.add_argument('--version', action='version', version='%(prog)s 0.5')
+
+ loglvl = min(logging.ERROR,
+ max(logging.DEBUG,
+ getattr(logging, args.loglvl.upper()) - args.verbose_level * 10 + args.quiet_level * 10))
+
+ logging.basicConfig(format=u'*** %(levelname)s: %(message)s', level=loglvl)
+ # print "Logging config: {0} + {1} - {2} = {3}".format(args.loglvl.upper(), args.verbose_level * 10,
+ # args.quiet_level * 10, logging.getLevelName(loglvl))
+
+ if args.bl_enabled:
+ blacklist = ['version.txt', '.filelist', 'README',
+ 'LICENSE', 'decimal.h', 'atan2.h', 'project.cmt', 'INSTALL']
+ greylist = ['site.py', 'site.pyc', 'easy_install', 'easy_install-2.7', 'setuptools.pth']
+ pkg_blacklist = ['neurobayes_expert', 'vdt', 'cmt', 'Qt5', 'xrootd_python', 'powheg-box', 'sherpa-mpich2',
+ 'yamlcpp']
+ else:
+ blacklist = ['version.txt']
+ greylist = []
+ pkg_blacklist = []
+
+ topdir_whitelist = ['aclocal', 'cmake', 'emacs', 'fonts', 'include', 'macros', 'test', 'tests',
+ 'bin', 'config', 'etc', 'icons', 'lib', 'lib64', 'man', 'tutorials', 'share', 'src']
+ concatenatelist = ['easy-install.pth']
+
+ if args.delview and os.path.exists(args.view_destination[0]):
+ shutil.rmtree(args.view_destination[0], True)
+
+ if not os.path.exists(args.view_destination[0]):
+ os.makedirs(args.view_destination[0])
+
+ if args.pkgfile and not os.path.exists(args.pkgfile):
+ logging.critical("Package selection file {0} does not exist!".format(args.pkgfile))
+ return 1
+
+ v = LCGViewMaker(lcgpath=args.lcgpath, lcgrel=args.lcgrel, lcgplat=args.lcgplat, pkgfile=args.pkgfile,
+ view_destination=args.view_destination[0], blacklist=blacklist, greylist=greylist,
+ pkg_blacklist=pkg_blacklist, dry_run=args.dry_run)
+
+ return v.make_view()
+
+ # return 0
+
+
+if __name__ == '__main__':
+ exit(main())
--
GitLab