Skip to content
Snippets Groups Projects
Commit bd2dfed8 authored by Ivan Razumov's avatar Ivan Razumov
Browse files

Copy create_lcg_view script from master

parent 1128f8a4
Branches LCG_83
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
cmake/scripts/create_lcg_view.py
+ 428
− 228
View file @ bd2dfed8
......@@ -9,156 +9,402 @@
# -r LCGREL, --release LCGREL LCG release number (default: 80)
# -l LCGPLAT, --platform LCGPLAT Platform to use (default: x86_64-slc6-gcc49-opt)
# -d, --delete delete old view before creating a new one
# -v, --version show program's version number and exit
# -s, --package-selection Create view only from packages specified in file. For experts only.
# -B, --enable-blacklists Enable built-in blacklists of files and packages. For experts only.
#
# -D, --dry-run Don't make changes to the file system
# -v, --verbose Increase logging verbosity
# -q, --quiet Decrease logging verbosity
# --loglevel LEVEL Change logging level
# --version show program's version number and exit
import argparse
import glob
import logging
import os
import sys
import shutil
import re
import shutil
import sys
import time
from collections import defaultdict, OrderedDict
class LCGViewMaker(object):
def __init__(self, lcgpath, lcgrel, lcgplat, pkgfile, view_destination, blacklist, greylist, pkg_blacklist,
dry_run):
super(LCGViewMaker, self).__init__()
self.lcg_root = lcgpath
self.lcg_release = lcgrel
self.lcg_platform = lcgplat
self.lcg_packages = []
self.view_root = view_destination
self.externals = defaultdict(OrderedDict)
self.blacklist = blacklist
self.greylist = greylist
self.pkg_blacklist = pkg_blacklist
self.topdir_whitelist = ['aclocal', 'cmake', 'emacs', 'fonts', 'include', 'macros', 'test', 'tests',
'bin', 'config', 'etc', 'icons', 'lib', 'lib64', 'man', 'tutorials', 'share', 'src']
self.concatenatelist = ['easy-install.pth']
self.dry_run = dry_run
self.packages_to_install = {}
if pkgfile:
with open(pkgfile, "r") as pkgfd:
for line in pkgfd.readlines():
line = line.strip()
if not line or line.startswith('#'):
continue
data = [x.strip() for x in line.split(" ")]
if len(data) == 1:
data.append("")
self.lcg_packages.append(data)
def get_default_version(self, package):
if package not in self.externals:
logging.error("Package %s not known when choosing default version!", package)
return None
all_versions = self.externals[package].keys()
return sorted(all_versions)[-1]
def add_externals_from_release_file(self, lcg_file):
txt_file = open(lcg_file)
for line in txt_file.readlines():
try:
name, pkghash, version, home, deps = [x.strip() for x in line.split(';')]
except ValueError:
continue
else:
r = {'NAME': name, 'HASH': pkghash, 'HOME': home,
'VERSION': version, 'DEPS': [x.split('-') for x in deps.split(',')]}
self.externals[name][version] = r
def splitall(self, path):
allparts = []
while 1:
parts = os.path.split(path)
if parts[0] == path: # sentinel for absolute paths
allparts.insert(0, parts[0])
break
elif parts[1] == path: # sentinel for relative paths
allparts.insert(0, parts[1])
break
else:
path = parts[0]
allparts.insert(0, parts[1])
return allparts
def add_externals_from_directory(self, directory, platform):
dirs = glob.glob(os.path.join(directory, '*/*', platform))
dirs.extend(glob.glob(os.path.join(directory, '*/*/*', platform)))
for d in dirs:
p = self.splitall(d)
name = p[-3]
version = p[-2]
r = {'NAME': name, 'VERSION': version, 'HOME': d.replace(directory, '.'), 'DEPS': []}
self.externals[name][version] = r
def write_setup(self):
if self.dry_run:
return
arch, osvers, compvers, buildtype = self.lcg_platform.split('-')
patt = re.compile('([a-z]+)([0-9]+)')
mobj = patt.match(compvers)
compiler = mobj.group(1)
version = '.'.join(list(mobj.group(2)))
if os.path.exists(os.path.normpath(os.path.join(self.lcg_root, '..', 'contrib', compiler))):
compiler = os.path.normpath(
os.path.join(self.lcg_root, '..', 'contrib', compiler, version, '-'.join((arch, osvers))))
elif os.path.exists(os.path.normpath(os.path.join(self.lcg_root, '../../../..', 'contrib', compiler))):
compiler = os.path.normpath(
os.path.join(self.lcg_root, '../../../..', 'contrib', compiler, version, '-'.join((arch, osvers))))
setup_sh = """#---Source this script to setup the complete environment for this LCG view
# Generated: @date@
#---Get the location this script (thisdir)
thisdir=$(dirname ${BASH_ARGV[0]})
# First the compiler
source @compilerlocation@/setup.sh
CC=`which gcc`; export CC
CXX=`which g++`; export CXX
FC=`which gfortran`; export FC
# then the rest...
if [ -z "${PATH}" ]; then
PATH=${thisdir}/bin; export PATH
else
PATH=${thisdir}/bin:$PATH; export PATH
fi
if [ -z "${LD_LIBRARY_PATH}" ]; then
LD_LIBRARY_PATH=${thisdir}/lib64:${thisdir}/lib; export LD_LIBRARY_PATH
else
LD_LIBRARY_PATH=${thisdir}/lib64:${thisdir}/lib:$LD_LIBRARY_PATH; export LD_LIBRARY_PATH
fi
if [ -z "${PYTHONPATH}" ]; then
PYTHONPATH=${thisdir}/lib:${thisdir}/lib/python2.7/site-packages; export PYTHONPATH
else
PYTHONPATH=${thisdir}/lib:${thisdir}/lib/python2.7/site-packages:$PYTHONPATH; export PYTHONPATH
fi
if [ -z "${MANPATH}" ]; then
MANPATH=${thisdir}/man; export MANPATH
else
MANPATH=${thisdir}/man:$MANPATH; export MANPATH
fi
if [ -z "${CMAKE_PREFIX_PATH}" ]; then
CMAKE_PREFIX_PATH=${thisdir}; export CMAKE_PREFIX_PATH
else
CMAKE_PREFIX_PATH=${thisdir}:$CMAKE_PREFIX_PATH; export CMAKE_PREFIX_PATH
fi
#---then ROOT
if [ -x $thisdir/bin/root ]; then
ROOTSYS=$(dirname $(dirname $(readlink $thisdir/bin/root))); export ROOTSYS
if [ -z "${ROOT_INCLUDE_PATH}" ]; then
ROOT_INCLUDE_PATH=${thisdir}/include; export ROOT_INCLUDE_PATH
else
ROOT_INCLUDE_PATH=${thisdir}/include:$ROOT_INCLUDE_PATH; export ROOT_INCLUDE_PATH
fi
fi
#---then SPARK
if [ -x $thisdir/bin/pyspark ]; then
SPARK_HOME=$(dirname $(dirname $(readlink $thisdir/bin/pyspark))); export SPARK_HOME
fi
"""
setup_csh = """#---Source this script to setup the complete environment for this LCG view
# Generated: @date@
#---Get the location this script (thisdir)
set ARGS=($_)
set LSOF=`env PATH=/usr/sbin:${PATH} which lsof`
set thisfile="`${LSOF} -w +p $$ | grep -oE '/.*setup.csh' `"
if ( "$thisfile" != "" && -e ${thisfile} ) then
set thisdir="`dirname ${thisfile}`"
else if ("$ARGS" != "") then
set thisdir="`dirname ${ARGS[2]}`"
endif
#---First the compiler
source @compilerlocation@/setup.csh
setenv CC `which gcc`
setenv CXX `which g++`
setenv FC `which gfortran`
#---then the rest...
if ($?PATH) then
setenv PATH ${thisdir}/bin:${PATH}
else
setenv PATH ${thisdir}/bin
endif
if ($?LD_LIBRARY_PATH) then
setenv LD_LIBRARY_PATH ${thisdir}/lib64:${thisdir}/lib:${LD_LIBRARY_PATH}
else
setenv LD_LIBRARY_PATH ${thisdir}/lib64:${thisdir}/lib
endif
if ($?PYTHONPATH) then
setenv PYTHONPATH ${thisdir}/lib:${thisdir}/lib/python2.7/site-packages:${PYTHONPATH}
else
setenv PYTHONPATH ${thisdir}/lib:${thisdir}/lib/python2.7/site-packages
endif
if ($?CMAKE_PREFIX_PATH) then
setenv CMAKE_PREFIX_PATH ${thisdir}:${CMAKE_PREFIX_PATH}
else
setenv CMAKE_PREFIX_PATH $thisdir
endif
#---then ROOT
if ( -e $thisdir/bin/root ) then
set rootdir=`readlink $thisdir/bin/root`
set rootdir=`dirname $rootdir`
set rootdir=`dirname $rootdir`
setenv ROOTSYS $rootdir
if ($?ROOT_INCLUDE_PATH) then
setenv ROOT_INCLUDE_PATH ${thisdir}/include:${ROOT_INCLUDE_PATH}
else
setenv ROOT_INCLUDE_PATH ${thisdir}/include
endif
endif
#---then SPARK
if ( -e $thisdir/bin/pyspark ) then
set spark_home=`readlink $thisdir/bin/pyspark`
set spark_home=`dirname $spark_home`
set spark_home=`dirname $spark_home`
setenv SPARK_HOME $spark_home
endif
"""
f = open(os.path.join(self.view_root, 'setup.sh'), 'w')
f.write(setup_sh.replace('@date@', time.strftime("%c")).replace('@compilerlocation@', compiler))
f.close()
f = open(os.path.join(self.view_root, 'setup.csh'), 'w')
f.write(setup_csh.replace('@date@', time.strftime("%c")).replace('@compilerlocation@', compiler))
f.close()
def install_pkg(self, pkg_name, pkg_version):
logging.debug('Installing {0} version {1}'.format(pkg_name, pkg_version))
if self.dry_run:
return
if self.lcg_release :
release_root = os.path.join(self.lcg_root, 'LCG_%s' % self.lcg_release)
else :
release_root = self.lcg_root
pkg_root = os.path.realpath(os.path.join(release_root, self.externals[pkg_name][pkg_version]['HOME']))
logging.info('Package {0} ver. {1}: root = {2}'.format(pkg_name, pkg_version, pkg_root))
for (dir_path, dirnames, filenames) in os.walk(pkg_root, followlinks=False):
if 'doc' in self.splitall(dir_path):
continue
if 'logs' in self.splitall(dir_path):
continue
dirpath = dir_path.replace(pkg_root, '.')
dirpath_s = self.splitall(dirpath)
def get_externals(lcg_file):
externals = {}
txt_file = open(lcg_file)
for line in txt_file.readlines():
try:
name, pkghash, version, home, deps = [x.strip() for x in line.split(';')]
except ValueError:
continue
# Elinimate any top level file or directory not in the topdir_whitelist
if len(dirpath_s) == 1 or dirpath_s[1] not in self.topdir_whitelist:
continue
view_dir = os.path.realpath(os.path.join(self.view_root, dirpath))
if not os.path.exists(view_dir):
# print 'Create directory', os.path.realpath(os.path.join(view_root, dirpath))
try:
os.makedirs(view_dir)
except OSError as e:
if e.errno == 20:
logging.warning("Target already exisits and is file: {0}".format(view_dir))
logging.info("Added from: {0}".format(os.path.realpath(view_dir)))
logging.info("Conflicts with: {0}".format(os.path.realpath(dir_path)))
else:
raise e
for f in filenames:
if f in self.blacklist or f.startswith('.') or f.endswith('-env.sh') or f.endswith('~'):
continue
source = os.path.join(dir_path, f)
target = os.path.join(view_dir, f)
source_rel = os.path.realpath(source).replace(self.lcg_root + os.path.sep, '')
target_rel = os.path.realpath(target).replace(self.lcg_root + os.path.sep, '')
if f in self.concatenatelist:
open(target, 'a').write(open(source, 'r').read())
continue
if not os.path.exists(target):
# print "Create symlink: {0} -> {1}".format(source, target)
try:
os.symlink(source, target)
except OSError as e:
if e.errno == 20:
logging.warning("Target already exisits and is file: {0}".format(f))
logging.info("Added from: {0}".format(os.path.realpath(target_rel)))
logging.info("Conflicts with: {0}".format(os.path.realpath(source_rel)))
else:
raise e
else:
if f not in self.greylist:
logging.warning("File already exisits: {0}".format(target))
logging.info("Added from: {0}".format(target_rel))
logging.info("Conflicts with: {0}".format(source_rel))
# return 1
def prepare_package(self, pkgname, pkgversion):
logging.debug("Add package {0} version {1}".format(pkgname, pkgversion))
if pkgname in self.packages_to_install:
logging.debug("Package {0} already added".format(pkgname))
return
if pkgname in self.externals:
if pkgversion in self.externals[pkgname] or pkgversion == '':
if pkgversion == '':
pkgversion = self.get_default_version(pkgname)
logging.info(
"Package version for {0} not specified, substituting {1}".format(pkgname, pkgversion))
self.packages_to_install[pkgname] = pkgversion
logging.debug("Add package {0} version {1}".format(pkgname, pkgversion))
logging.debug(
"Add {0} dependencies: {1}".format(len(self.externals[pkgname][pkgversion]['DEPS']),
self.externals[pkgname][pkgversion]['DEPS']))
for dep in self.externals[pkgname][pkgversion]['DEPS']:
if len(dep) != 2:
continue
depname, dephash = dep
depver = ''
for depv, depd in self.externals[depname].iteritems():
if depd['HASH'] == dephash:
depver = depv
break
if depver != '':
logging.debug("Add dependency: {0} version {1}".format(depname, depver))
for i, pkg in enumerate(self.lcg_packages):
if pkg[0] == depname and pkg[1] != depver:
logging.info(
"Dependency override: package {0}, version {1} -> {2}".format(depname, pkg[1],
depver))
self.lcg_packages.pop(i)
self.lcg_packages.append([depname, depver])
if depname in self.packages_to_install:
self.packages_to_install[depname] = depver
break
else:
logging.critical(
"Package {0} with hash {1} not found in release {2} when processing package "
"{3} version {4}".format(depname, dephash, self.lcg_release, pkgname, pkgversion))
return 1
else:
logging.critical(
"Version {1} of package {0} not found in release {2}!".format(pkgname, pkgversion,
self.lcg_release))
logging.info("Possible versions: {0}".format(", ".join(self.externals[pkgname].keys())))
return 1
else:
r = {'NAME': name, 'HASH': pkghash, 'HOME': home,
'VERSION': version}
externals.update({name: r})
return externals
def splitall(path):
allparts = []
while 1:
parts = os.path.split(path)
if parts[0] == path: # sentinel for absolute paths
allparts.insert(0, parts[0])
break
elif parts[1] == path: # sentinel for relative paths
allparts.insert(0, parts[1])
break
logging.critical("Package {0} not found in release {1}!".format(pkgname, self.lcg_release))
return 1
pass
def make_view(self):
# ---Check whether the LCG release actually exists, otherwise take all the packages in the root directory
logging.debug("make_view start")
release_root = os.path.join(self.lcg_root, 'LCG_%s' % self.lcg_release)
if os.path.exists(release_root):
lcg_file = os.path.join(release_root, 'LCG_externals_%s.txt' % self.lcg_platform)
mc_file = os.path.join(release_root, 'LCG_generators_%s.txt' % self.lcg_platform)
self.add_externals_from_release_file(lcg_file)
self.add_externals_from_release_file(mc_file)
else:
path = parts[0]
allparts.insert(0, parts[1])
return allparts
def write_setup(lcg_root, view_root, platform):
arch, osvers, compvers, buildtype = platform.split('-')
patt = re.compile('([a-z]+)([0-9]+)')
mobj = patt.match(compvers)
compiler = mobj.group(1)
version = '.'.join(list(mobj.group(2)))
if os.path.exists(os.path.normpath(os.path.join(lcg_root,'..','contrib',compiler))):
compiler = os.path.normpath(os.path.join(lcg_root,'..','contrib',compiler,version,'-'.join((arch, osvers))))
setup_sh = """#---Source this script to setup the complete environment for this LCG view
# Generated: @date@
#---Get the location this script (thisroot)
thisdir=$(dirname ${BASH_ARGV[0]})
# First the compiler
source @compilerlocation@/setup.sh
CC=`which gcc`; export CC
CXX=`which g++`; export CXX
FC=`which gfortran`; export FC
# then the rest...
if [ -z "${PATH}" ]; then
PATH=${thisdir}/bin; export PATH
else
PATH=${thisdir}/bin:$PATH; export PATH
fi
if [ -z "${LD_LIBRARY_PATH}" ]; then
LD_LIBRARY_PATH=${thisdir}/lib64:${thisdir}/lib; export LD_LIBRARY_PATH
else
LD_LIBRARY_PATH=${thisdir}/lib64:${thisdir}/lib:$LD_LIBRARY_PATH; export LD_LIBRARY_PATH
fi
if [ -z "${PYTHONPATH}" ]; then
PYTHONPATH=${thisdir}/lib:${thisdir}/lib/python2.7/site-packages; export PYTHONPATH
else
PYTHONPATH=${thisdir}/lib:${thisdir}/lib/python2.7/site-packages:$PYTHONPATH; export PYTHONPATH
fi
if [ -z "${MANPATH}" ]; then
MANPATH=${thisdir}/man; export MANPATH
else
MANPATH=${thisdir}/man:$MANPATH; export MANPATH
fi
if [ -z "${CMAKE_PREFIX_PATH}" ]; then
CMAKE_PREFIX_PATH=${thisdir}; export CMAKE_PREFIX_PATH
else
CMAKE_PREFIX_PATH=${thisdir}:$CMAKE_PREFIX_PATH; export CMAKE_PREFIX_PATH
fi
#---then ROOT
ROOTSYS=$(dirname $(dirname $(readlink $thisdir/bin/root))); export ROOTSYS
if [ -z "${ROOT_INCLUDE_PATH}" ]; then
ROOT_INCLUDE_PATH=${thisdir}/include; export ROOT_INCLUDE_PATH
else
ROOT_INCLUDE_PATH=${thisdir}/include:$ROOT_INCLUDE_PATH; export ROOT_INCLUDE_PATH
fi
"""
release_root = self.lcg_root
self.add_externals_from_directory(release_root, self.lcg_platform)
self.lcg_release = None
logging.debug("Loaded {0} externals".format(len(self.externals)))
self.packages_to_install = {}
if self.lcg_packages:
logging.debug("Build view from {0} packages".format(len(self.lcg_packages)))
for pkgname, pkgversion in self.lcg_packages:
self.prepare_package(pkgname, pkgversion)
else:
logging.debug("No package list specified")
for pkgname in self.externals:
self.prepare_package(pkgname, max(self.externals[pkgname]))
logging.debug("Final list contains {0} packages".format(len(self.packages_to_install)))
for pkgname in self.pkg_blacklist:
if pkgname in self.packages_to_install:
self.packages_to_install.pop(pkgname)
logging.debug("Filtered list contains {0} packages".format(len(self.packages_to_install)))
for pkgname, pkgversion in self.packages_to_install.iteritems():
self.install_pkg(pkgname, pkgversion)
# ---Finalize the view with additional operations------------------------------------------------------------
self.write_setup()
setup_csh = """#---Source this script to setup the complete environment for this LCG view
# Generated: @date@
#---Get the location this script (thisdir)
set ARGS=($_)
set LSOF=`env PATH=/usr/sbin:${PATH} which lsof`
set thisfile="`${LSOF} -w +p $$ | grep -oE '/.*setup.csh' `"
if ( "$thisfile" != "" && -e ${thisfile} ) then
set thisdir="`dirname ${thisfile}`"
else if ("$ARGS" != "") then
set thisdir="`dirname ${ARGS[2]}`"
endif
#---First the compiler
source @compilerlocation@/setup.csh
setenv CC `which gcc`
setenv CXX `which g++`
setenv FC `which gfortran`
#---then the rest...
if ($?PATH) then
setenv PATH ${thisdir}/bin:${PATH}
else
setenv PATH ${thisdir}/bin
endif
if ($?LD_LIBRARY_PATH) then
setenv LD_LIBRARY_PATH ${thisdir}/lib64:${thisdir}/lib:${LD_LIBRARY_PATH}
else
setenv LD_LIBRARY_PATH ${thisdir}/lib64:${thisdir}/lib
endif
if ($?PYTHONPATH) then
setenv PYTHONPATH ${thisdir}/lib:${thisdir}/lib/python2.7/site-packages:${PYTHONPATH}
else
setenv PYTHONPATH ${thisdir}/lib:${thisdir}/lib/python2.7/site-packages
endif
if ($?CMAKE_PREFIX_PATH) then
setenv CMAKE_PREFIX_PATH ${thisdir}:${CMAKE_PREFIX_PATH}
else
setenv CMAKE_PREFIX_PATH $thisdir
endif
#---then ROOT
set rootdir=`readlink $thisdir/bin/root`
set rootdir=`dirname $rootdir`
set rootdir=`dirname $rootdir`
setenv ROOTSYS $rootdir
if ($?ROOT_INCLUDE_PATH) then
setenv ROOT_INCLUDE_PATH ${thisdir}/include:${ROOT_INCLUDE_PATH}
else
setenv ROOT_INCLUDE_PATH ${thisdir}/include
endif
"""
f = open(os.path.join(view_root,'setup.sh'), 'w')
f.write(setup_sh.replace('@date@',time.strftime("%c")).replace('@compilerlocation@',compiler))
f.close()
f = open(os.path.join(view_root,'setup.csh'), 'w')
f.write(setup_csh.replace('@date@',time.strftime("%c")).replace('@compilerlocation@',compiler))
f.close()
def main():
helpstring = """{0} [options] <view_destination>
......@@ -176,116 +422,70 @@ This package creates a "view" of LCG release in folder <view_destination>.
parser.add_argument('-l', '--lcgpath', help="top directory of LCG releases (default: /afs/cern.ch/sw/lcg/releases)",
action="store",
default='/afs/cern.ch/sw/lcg/releases', dest='lcgpath')
parser.add_argument('-r', '--release', help="LCG release number (default: 80)", action="store", default=80, dest="lcgrel")
parser.add_argument('-r', '--release', help="LCG release number (default: 80)", action="store", default=80,
dest="lcgrel")
parser.add_argument('-p', '--platform', help="Platform to use (default: x86_64-slc6-gcc49-opt)", action="store",
default='x86_64-slc6-gcc49-opt', dest='lcgplat')
parser.add_argument('-d', '--delete', help="delete old view before creating a new one", action="store_true",
default=False, dest='delview')
parser.add_argument('-B', '--enable-blacklists', help='Enable built-in blacklists of files and packages. For experts only.',
action="store_true", default=False, dest='bl_enabled')
parser.add_argument('-v', '--version', action='version', version='%(prog)s 0.3')
parser.add_argument('-B', '--enable-blacklists',
help='Enable built-in blacklists of files and packages. For experts only.',
action="store_true", dest='bl_enabled')
parser.add_argument('-s', '--package-selection',
help='Create view only from packages specified in file. For experts only.',
action='store', dest='pkgfile')
parser.add_argument('-D', '--dry-run',
help="Don't delete or link anything. For debugging only.", action='store_true', dest='dry_run')
group = parser.add_mutually_exclusive_group()
group.add_argument('-v', '--verbose', action='count', dest='verbose_level', help='Increase logging verbosity',
default=0)
group.add_argument('-q', '--quiet', action='count', dest='quiet_level', help='Decrease logging verbosity',
default=0)
group.add_argument('--loglevel', choices=['ERROR', 'WARNING', 'INFO', 'DEBUG'], action='store',
dest='loglvl', default='INFO', help='Set logging level (default: INFO)')
args = parser.parse_args()
parser.add_argument('--version', action='version', version='%(prog)s 0.5')
loglvl = min(logging.ERROR,
max(logging.DEBUG,
getattr(logging, args.loglvl.upper()) - args.verbose_level * 10 + args.quiet_level * 10))
lcg_root = args.lcgpath
lcg_release = args.lcgrel
lcg_platform = args.lcgplat
view_root = args.view_destination[0]
logging.basicConfig(format=u'*** %(levelname)s: %(message)s', level=loglvl)
# print "Logging config: {0} + {1} - {2} = {3}".format(args.loglvl.upper(), args.verbose_level * 10,
# args.quiet_level * 10, logging.getLevelName(loglvl))
if args.bl_enabled:
blacklist = ['version.txt', '.filelist', 'README',
'LICENSE', 'decimal.h', 'atan2.h', 'project.cmt', 'INSTALL']
greylist = ['site.py', 'site.pyc', 'easy_install', 'easy_install-2.7', 'setuptools.pth']
pkg_blacklist = ['neurobayes_expert', 'vdt', 'cmt', 'Qt5', 'xrootd_python', 'powheg-box', 'sherpa-mpich2', 'yamlcpp']
pkg_blacklist = ['neurobayes_expert', 'vdt', 'cmt', 'Qt5', 'xrootd_python', 'powheg-box', 'sherpa-mpich2',
'yamlcpp']
else:
blacklist = ['version.txt']
greylist = []
greylist = []
pkg_blacklist = []
topdir_whitelist = ['aclocal', 'cmake', 'emacs', 'fonts', 'include', 'macros', 'test', 'tests',
'bin', 'config', 'etc', 'icons', 'lib', 'lib64', 'man', 'tutorials', 'share']
topdir_whitelist = ['aclocal', 'cmake', 'emacs', 'fonts', 'include', 'macros', 'test', 'tests',
'bin', 'config', 'etc', 'icons', 'lib', 'lib64', 'man', 'tutorials', 'share', 'src']
concatenatelist = ['easy-install.pth']
if args.delview and os.path.exists(view_root):
shutil.rmtree(view_root, True)
if not os.path.exists(view_root):
os.makedirs(view_root)
release_root = os.path.join(lcg_root, 'LCG_%s' % lcg_release)
lcg_file = os.path.join(release_root, 'LCG_externals_%s.txt' % lcg_platform)
mc_file = os.path.join(release_root, 'LCG_generators_%s.txt' % lcg_platform)
externals = get_externals(lcg_file)
externals.update(get_externals(mc_file))
for pkg in externals:
if pkg in pkg_blacklist:
continue
pkg_root = os.path.realpath(os.path.join(release_root, externals[pkg]['HOME']))
print 'Package {0}: root = {1}'.format(pkg, pkg_root)
for (dir_path, dirnames, filenames) in os.walk(pkg_root, followlinks=False):
if 'doc' in splitall(dir_path):
continue
if 'logs' in splitall(dir_path):
continue
dirpath = dir_path.replace(pkg_root, '.')
dirpath_s = splitall(dirpath)
# Elinimate any top level file or directory not in the topdir_whitelist
if len(dirpath_s) == 1 or dirpath_s[1] not in topdir_whitelist :
continue
view_dir = os.path.realpath(os.path.join(view_root, dirpath))
if args.delview and os.path.exists(args.view_destination[0]):
shutil.rmtree(args.view_destination[0], True)
if not os.path.exists(view_dir):
# print 'Create directory', os.path.realpath(os.path.join(view_root, dirpath))
try:
os.makedirs(view_dir)
except OSError as e:
if e.errno == 20:
print "*** WARNING: Target already exisits and is file: {0} ***".format(view_dir)
print "*** NOTICE: Added from: {0} ***".format(os.path.realpath(view_dir))
print "*** NOTICE: Conflicts with: {0} ***".format(os.path.realpath(dir_path))
else:
raise e
if not os.path.exists(args.view_destination[0]):
os.makedirs(args.view_destination[0])
# continue
for f in filenames:
if f in blacklist or f.startswith('.') or f.endswith('-env.sh') or f.endswith('~'):
continue
if args.pkgfile and not os.path.exists(args.pkgfile):
logging.critical("Package selection file {0} does not exist!".format(args.pkgfile))
return 1
source = os.path.join(dir_path, f)
target = os.path.join(view_dir, f)
v = LCGViewMaker(lcgpath=args.lcgpath, lcgrel=args.lcgrel, lcgplat=args.lcgplat, pkgfile=args.pkgfile,
view_destination=args.view_destination[0], blacklist=blacklist, greylist=greylist,
pkg_blacklist=pkg_blacklist, dry_run=args.dry_run)
source_rel = os.path.realpath(source).replace(lcg_root + os.path.sep, '')
target_rel = os.path.realpath(target).replace(lcg_root + os.path.sep, '')
if f in concatenatelist:
open(target,'a').write(open(source,'r').read())
continue
return v.make_view()
if not os.path.exists(target):
# print "Create symlink: {0} -> {1}".format(source, target)
try:
os.symlink(source, target)
except OSError as e:
if e.errno == 20:
print "*** WARNING: Target already exisits and is file: {0} ***".format(f)
print "*** NOTICE: Added from: {0} ***".format(target_rel)
print "*** NOTICE: Conflicts with: {0} ***".format(source_rel)
else:
raise e
else:
if f not in greylist:
print "*** WARNING: File already exisits: {0} ***".format(target)
print "*** NOTICE: Added from: {0} ***".format(target_rel)
print "*** NOTICE: Conflicts with: {0} ***".format(source_rel)
#return 1
write_setup(lcg_root, view_root, lcg_platform)
return 0
# return 0
if __name__ == '__main__':
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment