Skip to content
Snippets Groups Projects
Commit 561db725 authored by Marco Clemencic's avatar Marco Clemencic
Browse files

Externalize shared python tools (mr !618) 

- change Gitlab-CI Docker image to latest LHCb one
- prefer host-binary-tag from PATH
  (see https://gitlab.cern.ch/lhcb-core/LbPlatformUtils/)
- use xenv from PATH or get it from the Git repository
  (see https://gitlab.cern.ch/gaudi/xenv)

Although having the scripts as part of Gaudi is desirable, it's also
very useful to be able to update/fix them without having to make a new
release of Gaudi.

Backport of !612
parents 6ebc5d99 504248e8
No related branches found
No related tags found
1 merge request!618Externalize shared python tools
Pipeline #
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 48 additions and 2055 deletions
.gitlab-ci.yml
+ 7
− 5
View file @ 561db725
......@@ -3,18 +3,23 @@ stages:
- test
- deploy
image: gitlab-registry.cern.ch/lhcb-core/lbdocker/centos7-build
variables:
NO_LBLOGIN: 1
before_script:
- export BINARY_TAG=x86_64-centos7-gcc7-opt
- export LCG_hostos=x86_64-centos7
- export PATH=/cvmfs/sft.cern.ch/lcg/contrib/CMake/3.8.1/Linux-x86_64/bin:/cvmfs/sft.cern.ch/lcg/contrib/ninja/1.7.1/x86_64-slc6:/cvmfs/sft.cern.ch/lcg/releases/Python/2.7.13-597a5/${BINARY_TAG}/bin:${PATH}
- export LCG_release_area=/cvmfs/sft.cern.ch/lcg/releases
- export PATH=/cvmfs/lhcb.cern.ch/lib/lhcb/LBSCRIPTS/dev/InstallArea/scripts:${PATH}
- export PYTHONPATH=/cvmfs/lhcb.cern.ch/lib/lhcb/LBSCRIPTS/dev/InstallArea/python:${PYTHONPATH}
- export CMAKE_PREFIX_PATH=/cvmfs/sft.cern.ch/lcg/releases:/cvmfs/projects.cern.ch/intelsw/psxe/linux/x86_64/2017/vtune_amplifier_xe
- export CCACHE_DIR=${PWD}/.ccache
- export CCACHE_CPP2=1
build:
image: lhcbdev/centos7-build:latest
tags:
- cvmfs
stage: build
......@@ -33,13 +38,12 @@ build:
expire_in: 1 week
coding-conventions:
image: lhcbdev/centos7-build:latest
tags:
- cvmfs
stage: test
script:
- export LCG_release_area=${LCG_release_area}:/cvmfs/lhcb.cern.ch/lib/lcg/external
- pip install autopep8
- sudo pip install autopep8
- make BUILDDIR=build apply-formatting
- git diff > apply-formatting.patch
- git diff --stat --exit-code
......@@ -50,7 +54,6 @@ coding-conventions:
expire_in: 1 day
doxygen:
image: lhcbdev/centos7-build:latest
tags:
- cvmfs
stage: test
......@@ -70,7 +73,6 @@ doxygen:
expire_in: 1 day
test:
image: lhcbdev/centos7-build:latest
tags:
- cvmfs
stage: test
......
CMakeLists.txt
+ 0
− 2
View file @ 561db725
......@@ -30,8 +30,6 @@ include(${CMAKE_SOURCE_DIR}/cmake/externals.cmake)
gaudi_project(Gaudi v29r2)
# These tests do not really fit in a subdirectory.
gaudi_add_test(cmake.EnvConfigTests
COMMAND nosetests --with-doctest ${CMAKE_SOURCE_DIR}/cmake/EnvConfig)
gaudi_add_test(cmake.CMakeModules
COMMAND nosetests ${CMAKE_SOURCE_DIR}/cmake/tests)
gaudi_add_test(cmake.QMTDeps
......
cmake/BinaryTagUtils.cmake
+ 8
− 8
View file @ 561db725
......@@ -151,7 +151,10 @@ function(check_compiler)
endif()
endfunction()
set(_GET_HOST_BINARY_TAG_SCRIPT "${CMAKE_CURRENT_LIST_DIR}/get_host_binary_tag.py" CACHE INTERNAL "")
find_program(HOST_BINARY_TAG_COMMAND
NAMES host-binary-tag get_host_binary_tag.py
HINTS "${CMAKE_CURRENT_LIST_DIR}")
mark_as_advanced(HOST_BINARY_TAG_COMMAND)
#.rst
# .. command:: get_host_binary_tag
#
......@@ -171,14 +174,11 @@ function(get_host_binary_tag variable)
else()
set(type opt)
endif()
if(NOT HOST_BINARY_TAG_COMMAND)
message(FATAL_ERROR "No host-binary-tag command, cannot get host binary tag")
endif()
if(NOT HOST_BINARY_TAG)
if(NOT PYTHON_EXECUTABLE)
find_package(PythonInterp QUIET)
if(NOT PYTHONINTERP_FOUND)
message(FATAL_ERROR "Python interpreter required to get host binary tag")
endif()
endif()
execute_process(COMMAND ${PYTHON_EXECUTABLE} "${_GET_HOST_BINARY_TAG_SCRIPT}"
execute_process(COMMAND "${HOST_BINARY_TAG_COMMAND}"
OUTPUT_VARIABLE HOST_BINARY_TAG
OUTPUT_STRIP_TRAILING_WHITESPACE)
set(HOST_BINARY_TAG ${HOST_BINARY_TAG} CACHE STRING "BINARY_TAG of the host")
......
cmake/EnvConfig/Control.py deleted 100644 → 0
+ 0
− 385
View file @ 6ebc5d99
'''
Created on Jun 27, 2011
@author: mplajner
'''
import xmlModule
import os
from time import gmtime, strftime
import Variable
import EnvConfig
import logging
class Environment(object):
'''object to hold settings of environment'''
def __init__(self, loadFromSystem=True, useAsWriter=False, searchPath=None):
'''Initial variables to be pushed and setup
append switch between append and prepend for initial variables.
loadFromSystem causes variable`s system value to be loaded on first encounter.
If useAsWriter == True than every change to variables is recorded to XML file.
reportLevel sets the level of messaging.
'''
self.log = logging.getLogger('Environment')
self.separator = ':'
# Prepeare the internal search path for xml files (used by 'include' elements)
if searchPath is None:
self.searchPath = []
else:
self.searchPath = list(searchPath)
def addToSearchPath(n, _1, _2):
'''
Add entries to the search path expanding variables inside.
'''
entries = Variable.List('_SEARCH_PATH')
entries.set(n, os.pathsep, environment=self.variables)
self.searchPath.extend(entries)
self.actions = {}
self.actions['include'] = lambda n, c, h: self.loadXML(
self._locate(n, c, h))
self.actions['append'] = lambda n, v, _: self.append(n, v)
self.actions['prepend'] = lambda n, v, _: self.prepend(n, v)
self.actions['set'] = lambda n, v, _: self.set(n, v)
self.actions['unset'] = lambda n, v, _: self.unset(n, v)
self.actions['default'] = lambda n, v, _: self.default(n, v)
self.actions['remove'] = lambda n, v, _: self.remove(n, v)
self.actions['remove-regexp'] = lambda n, v, _: self.remove_regexp(
n, v)
self.actions['declare'] = self.declare
self.actions['search_path'] = addToSearchPath
self.variables = {}
self.loadFromSystem = loadFromSystem
self.asWriter = useAsWriter
if useAsWriter:
self.writer = xmlModule.XMLFile()
self.startXMLinput()
self.loadedFiles = set()
# Prepare the stack for the directory of the loaded file(s)
self._fileDirStack = []
# Note: cannot use self.declare() because we do not want to write out
# the changes to ${.}
dot = Variable.Scalar('.', local=True)
dot.expandVars = False
dot.set('')
self.variables['.'] = dot
def _locate(self, filename, caller=None, hints=None):
'''
Find 'filename' in the internal search path.
'''
from os.path import isabs, isfile, join, dirname, normpath, abspath
if isabs(filename):
return filename
self.log.debug('looking for %s', filename)
if hints is None:
hints = []
elif type(hints) is str:
hints = hints.split(self.separator)
if caller:
calldir = dirname(caller)
localfile = join(calldir, filename)
self.log.debug('trying %s', localfile)
if isfile(localfile):
self.log.debug('OK (local file)')
return localfile
# allow for relative hints
hints = [join(calldir, hint) for hint in hints]
sp = EnvConfig.path + self.searchPath + hints
def candidates():
for d in sp:
f = normpath(join(d, filename))
self.log.debug('trying %s', f)
yield f
try:
f = (abspath(f) for f in candidates() if isfile(f)).next()
self.log.debug('OK')
return f
except StopIteration:
from errno import ENOENT
raise OSError(ENOENT, 'cannot find file in %r' % sp, filename)
def vars(self, strings=True):
'''returns dictionary of all variables optionally converted to string'''
if strings:
return dict([(n, v.value(True))
for n, v in self.variables.items()
if n != '.'])
else:
# clone the dictionary to remove the internal special var '.'
vars_ = dict(self.variables)
if '.' in vars_:
del vars_['.']
return vars_
def var(self, name):
'''Gets a single variable. If not available then tries to load from system.'''
if name in self.variables:
return self.variables[name]
else:
return os.environ[name]
def search(self, varName, expr, regExp=False):
'''Searches in a variable for a value.'''
return self.variables[varName].search(expr, regExp)
def _guessType(self, varname):
'''
Guess the type of the variable from its name: if the name contains
'PATH' or 'DIRS', then the variable is a list, otherwise it is a scalar.
'''
varname = varname.upper() # make the comparison case insensitive
if 'PATH' in varname or 'DIRS' in varname:
return 'list'
else:
return 'scalar'
def declare(self, name, vartype, local):
'''Creates an instance of new variable. It loads values from the OS if the variable is not local.'''
if self.asWriter:
self._writeVarToXML(name, 'declare', '', vartype, local)
if not isinstance(local, bool):
if str(local).lower() == 'true':
local = True
else:
local = False
if name in self.variables.keys():
if self.variables[name].local != local:
raise Variable.EnvError(name, 'redeclaration')
else:
if vartype.lower() == "list":
if not isinstance(self.variables[name], Variable.List):
raise Variable.EnvError(name, 'redeclaration')
else:
if not isinstance(self.variables[name], Variable.Scalar):
raise Variable.EnvError(name, 'redeclaration')
if vartype.lower() == "list":
a = Variable.List(name, local)
else:
a = Variable.Scalar(name, local)
if self.loadFromSystem and not local and name in os.environ:
a.expandVars = False # disable var expansion when importing from the environment
a.set(os.environ[name], os.pathsep, environment=self.variables)
a.expandVars = True
self.variables[name] = a
def append(self, name, value):
'''Appends to an existing variable.'''
if self.asWriter:
self._writeVarToXML(name, 'append', value)
else:
if name not in self.variables:
self.declare(name, self._guessType(name), False)
self.variables[name].append(value, self.separator, self.variables)
def prepend(self, name, value):
'''Prepends to an existing variable, or create a new one.'''
if self.asWriter:
self._writeVarToXML(name, 'prepend', value)
else:
if name not in self.variables:
self.declare(name, self._guessType(name), False)
self.variables[name].prepend(value, self.separator, self.variables)
def set(self, name, value):
'''Sets a single variable - overrides any previous value!'''
name = str(name)
if self.asWriter:
self._writeVarToXML(name, 'set', value)
else:
if name not in self.variables:
self.declare(name, self._guessType(name), False)
self.variables[name].set(value, self.separator, self.variables)
def default(self, name, value):
'''Sets a single variable only if it is not already set!'''
name = str(name)
if self.asWriter:
self._writeVarToXML(name, 'default', value)
else:
# Here it is different from the other actions because after a 'declare'
# we cannot tell if the variable was already set or not.
# FIXME: improve declare() to allow for a default.
if name not in self.variables:
if self._guessType(name) == 'list':
v = Variable.List(name, False)
else:
v = Variable.Scalar(name, False)
if self.loadFromSystem and name in os.environ:
v.set(os.environ[name], os.pathsep,
environment=self.variables)
else:
v.set(value, self.separator, environment=self.variables)
self.variables[name] = v
else:
v = self.variables[name]
if not v.val:
v.set(value, self.separator, environment=self.variables)
def unset(self, name, value=None): # pylint: disable=W0613
'''Unsets a single variable to an empty value - overrides any previous value!'''
if self.asWriter:
self._writeVarToXML(name, 'unset', '')
else:
if name in self.variables:
del self.variables[name]
def remove(self, name, value, regexp=False):
'''Remove value from variable.'''
if self.asWriter:
self._writeVarToXML(name, 'remove', value)
else:
if name not in self.variables:
self.declare(name, self._guessType(name), False)
self.variables[name].remove(value, self.separator, regexp)
def remove_regexp(self, name, value):
self.remove(name, value, True)
def searchFile(self, filename, varName):
'''Searches for appearance of variable in a file.'''
XMLFile = xmlModule.XMLFile()
variable = XMLFile.variable(filename, name=varName)
return variable
def loadXML(self, fileName=None, namespace='EnvSchema'):
'''Loads XML file for input variables.'''
XMLfile = xmlModule.XMLFile()
fileName = self._locate(fileName)
if fileName in self.loadedFiles:
self.log.info('ignore %s: already loaded', fileName)
return # ignore recursion
self.log.debug('loading %s', fileName)
self.loadedFiles.add(fileName)
dot = self.variables['.']
# push the previous value of ${.} onto the stack...
self._fileDirStack.append(dot.value())
# ... and update the variable
dot.set(os.path.dirname(fileName))
variables = XMLfile.variable(fileName, namespace=namespace)
for i, (action, args) in enumerate(variables):
if action not in self.actions:
self.log.error(('Node {0}: No action taken with var "{1}". '
'Probably wrong action argument: "{2}".')
.format(i, args[0], action))
else:
self.actions[action](*args) # pylint: disable=W0142
# restore the old value of ${.}
dot.set(self._fileDirStack.pop())
# ensure that a change of ${.} in the file is reverted when exiting it
self.variables['.'] = dot
def startXMLinput(self):
'''Renew writer for new input.'''
self.writer.resetWriter()
def finishXMLinput(self, outputFile=''):
'''Finishes input of XML file and closes the file.'''
self.writer.writeToFile(outputFile)
def writeToFile(self, fileName, shell='sh'):
'''Creates an output file with a specified name to be used for setting variables by sourcing this file'''
f = open(fileName, 'w')
if shell == 'sh':
f.write('#!/bin/bash\n')
for variable in self.variables:
if not self[variable].local:
f.write('export ' + variable + '=' +
self[variable].value(True, os.pathsep) + '\n')
elif shell == 'csh':
f.write('#!/bin/csh\n')
for variable in self.variables:
if not self[variable].local:
f.write('setenv ' + variable + ' ' +
self[variable].value(True, os.pathsep) + '\n')
else:
f.write('')
f.write('REM This is an enviroment settings file generated on ' +
strftime("%a, %d %b %Y %H:%M:%S\n", gmtime()))
for variable in self.variables:
if not self[variable].local:
f.write('set ' + variable + '=' +
self[variable].value(True, os.pathsep) + '\n')
f.close()
def writeToXMLFile(self, fileName):
'''Writes the current state of environment to a XML file.
NOTE: There is no trace of actions taken, variables are written with a set action only.
'''
writer = xmlModule.XMLFile()
for varName in self.variables:
if varName == '.':
continue # this is an internal transient variable
writer.writeVar(
varName, 'set', self.variables[varName].value(True, self.separator))
writer.writeToFile(fileName)
def presetFromSystem(self):
'''Loads all variables from the current system settings.'''
for k, v in os.environ.items():
if k not in self.variables:
self.set(k, v)
def process(self):
'''
Call the variable processors on all the variables.
'''
for v in self.variables.values():
v.val = v.process(v.val, self.variables)
def _concatenate(self, value):
'''Returns a variable string with separator separator from the values list'''
stri = ""
for it in value:
stri += it + self.separator
stri = stri[0:len(stri) - 1]
return stri
def _writeVarToXML(self, name, action, value, vartype='list', local='false'):
'''Writes single variable to XML file.'''
if isinstance(value, list):
value = self._concatenate(value)
self.writer.writeVar(name, action, value, vartype, local)
def __getitem__(self, key):
return self.variables[key]
def __setitem__(self, key, value):
if key in self.variables.keys():
self.log.warning(
'Addition canceled because of duplicate entry. Var: "%s" value: "%s".', key, value)
else:
self.append(key, value)
def __delitem__(self, key):
del self.variables[key]
def __iter__(self):
for i in self.variables:
yield i
def __contains__(self, item):
return item in self.variables.keys()
def __len__(self):
return len(self.variables.keys())
cmake/EnvConfig/TestEnvOps.py deleted 100644 → 0
+ 0
− 677
View file @ 6ebc5d99
This diff is collapsed.
cmake/EnvConfig/TestXMLOps.py deleted 100644 → 0
+ 0
− 125
View file @ 6ebc5d99
'''
Created on Jul 15, 2011
@author: mplajner
'''
import unittest
import os
from StringIO import StringIO
from EnvConfig import Control
from EnvConfig import xmlModule
from tempfile import mkstemp
class Test(unittest.TestCase):
def setUp(self):
pass
def tearDown(self):
pass
def testFileLoad(self):
'''Test loading of previously written file.'''
self.control = Control.Environment(useAsWriter=True)
self.control.unset('varToUnset')
self.control.declare('myVar', 'list', True)
self.control.set('myVar', 'setVal:$local')
self.control.append('myVar', 'appVal:appVal2')
self.control.prepend('myVar', 'prepVal:prepVal2')
self.control.declare('myScalar', 'scalar', False)
self.control.set('myScalar', 'setValscal')
self.control.append('myScalar', 'appValscal')
self.control.prepend('myScalar', 'prepValscal')
self.control.declare('myScalar2', 'scalar', True)
self.control.finishXMLinput('testOutputFile.xml')
loader = xmlModule.XMLFile()
variables = loader.variable('testOutputFile.xml')
expected = [('declare', ('varToUnset', 'list', 'false')),
('unset', ('varToUnset', '', None)),
('declare', ('myVar', 'list', 'true')),
('set', ('myVar', 'setVal:$local', None)),
('append', ('myVar', 'appVal:appVal2', None)),
('prepend', ('myVar', 'prepVal:prepVal2', None)),
('declare', ('myScalar', 'scalar', 'false')),
('set', ('myScalar', 'setValscal', None)),
('append', ('myScalar', 'appValscal', None)),
('prepend', ('myScalar', 'prepValscal', None)),
('declare', ('myScalar2', 'scalar', 'true'))]
self.assertEqual(variables, expected)
os.remove('testOutputFile.xml')
def testParsing(self):
data = StringIO('''<?xml version="1.0" ?>
<env:config xmlns:env="EnvSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="EnvSchema ./EnvSchema.xsd ">
<env:declare local="false" type="list" variable="varToUnset"/>
<env:unset variable="varToUnset"/>
<env:declare local="true" type="list" variable="myVar"/>
<env:set variable="myVar">setVal:$local</env:set>
<env:append variable="myVar">appVal:appVal2</env:append>
<env:prepend variable="myVar">prepVal:prepVal2</env:prepend>
<env:declare local="false" type="scalar" variable="myScalar"/>
<env:set variable="myScalar">setValscal</env:set>
<env:append variable="myScalar">appValscal</env:append>
<env:prepend variable="myScalar">prepValscal</env:prepend>
<env:declare local="true" type="scalar" variable="myScalar2"/>
<env:include>some_file.xml</env:include>
<env:include hints="some:place">another_file.xml</env:include>
</env:config>''')
loader = xmlModule.XMLFile()
variables = loader.variable(data)
expected = [('declare', ('varToUnset', 'list', 'false')),
('unset', ('varToUnset', '', None)),
('declare', ('myVar', 'list', 'true')),
('set', ('myVar', 'setVal:$local', None)),
('append', ('myVar', 'appVal:appVal2', None)),
('prepend', ('myVar', 'prepVal:prepVal2', None)),
('declare', ('myScalar', 'scalar', 'false')),
('set', ('myScalar', 'setValscal', None)),
('append', ('myScalar', 'appValscal', None)),
('prepend', ('myScalar', 'prepValscal', None)),
('declare', ('myScalar2', 'scalar', 'true')),
('include', ('some_file.xml', None, '')),
('include', ('another_file.xml', None, 'some:place'))]
self.assertEqual(variables, expected)
def testParsingError(self):
fd, filename = mkstemp()
f = os.fdopen(fd, 'w')
f.write('''<?xml version="1.0" ?>
<env:config xmlns:env="EnvSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="EnvSchema ./EnvSchema.xsd ">
<env:set variable="error">invalid</env:default>
</env:config>''')
f.close()
try:
import logging
stream = StringIO()
hdlr = logging.StreamHandler(stream)
logging.getLogger().addHandler(hdlr)
loader = xmlModule.XMLFile()
self.assertRaises(SystemExit, loader.variable, filename)
self.assertTrue(('Failed to parse %s:' % filename)
in stream.getvalue(), 'missing error message')
finally:
logging.getLogger().removeHandler(hdlr)
os.remove(filename)
if __name__ == "__main__":
#import sys;sys.argv = ['', 'Test.testName']
unittest.main()
cmake/EnvConfig/Variable.py deleted 100644 → 0
+ 0
− 413
View file @ 6ebc5d99
'''
Created on Jun 27, 2011
@author: mplajner
'''
import re
import os
import logging
import functools
from os import listdir
from os.path import normpath
from zipfile import is_zipfile
class VariableProcessor(object):
'''
Base class for the objects used to process the variables.
'''
def __init__(self, env):
'''
@param env: dictionary with the reference environment to use
'''
if env is None:
env = {}
self._env = env
def isTarget(self, variable):
'''
Return True if this processor can operate on the given variable.
'''
return True
def process(self, variable, value):
'''
Process the variable.
@param value: the content of the variable to be processed
@return: the processed value
'''
# by default do nothing
return value
def __call__(self, variable, value):
return self.process(variable, value)
class ListProcessor(VariableProcessor):
'''
Base class for processors operating only on lists.
'''
def isTarget(self, variable):
'''
Return True if this variable is a list.
'''
return isinstance(variable, List)
class ScalarProcessor(VariableProcessor):
'''
Base class for processors operating only on scalars.
'''
def isTarget(self, variable):
'''
Return True if this variable is a scalar.
'''
return isinstance(variable, Scalar)
class EnvExpander(VariableProcessor):
'''
Variable processor to expand the reference to environment variables.
'''
def __init__(self, env):
super(EnvExpander, self).__init__(env)
self._exp = re.compile(
r"\$([A-Za-z_][A-Za-z0-9_]*)|\$\(([A-Za-z_][A-Za-z0-9_]*)\)|\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$\{(\.)\}")
def isTarget(self, variable):
return (super(EnvExpander, self).isTarget(variable)
and variable.expandVars)
def _repl(self, value):
m = self._exp.search(value)
if m:
try:
value = (value[:m.start()]
+ str(self._env[filter(None, m.groups())[0]])
+ value[m.end():])
except KeyError, k:
logging.debug(
'KeyError: %s unknown while expanding %s', k, value)
return value
return self._repl(value)
else:
return value
def process(self, variable, value):
if isinstance(value, str):
value = self._repl(value)
else:
# expand only in the elements that are new
old_values = set(variable.val)
value = map(
lambda v: v if v in old_values else self._repl(v), value)
return value
class PathNormalizer(VariableProcessor):
'''
Call os.path.normpath for all the entries of the variable.
'''
def process(self, variable, value):
if not value: # do not process empty strings/lists
return value
if isinstance(value, str):
if '://' not in value: # this might be a URL
value = normpath(value)
else:
value = [normpath(v) for v in value if v]
return value
class DuplicatesRemover(ListProcessor):
'''
Remove duplicates entries from lists.
'''
def process(self, variable, value):
val = []
for s in value:
if s not in val:
val.append(s)
return val
def memoize(func):
'''
Memoization decorator for a function taking a single argument.
'''
cache = func.cache = {}
@functools.wraps(func)
def memoizer(arg):
try:
return cache[arg]
except KeyError:
return cache.setdefault(arg, func(arg))
return memoizer
normpath = memoize(normpath)
is_zipfile = memoize(is_zipfile)
@memoize
def isnonemptydir(path):
'''
Return whether a path is a list-able non-empty directory.
'''
try:
return bool(listdir(path))
except OSError:
return False
class EmptyDirsRemover(ListProcessor):
'''
Remove empty or not existing directories from lists.
'''
def process(self, variable, value):
return [s for s in value if s.endswith('.zip') or isnonemptydir(s)]
class UsePythonZip(ListProcessor):
'''
Use .zip files instead of regular directories in PYTHONPATH when possible.
'''
def isTarget(self, variable):
return (super(UsePythonZip, self).isTarget(variable)
and variable.varName == 'PYTHONPATH')
def process(self, variable, value):
val = []
for s in value:
z = s + '.zip'
if is_zipfile(z):
val.append(z)
else:
val.append(s)
return val
# Default (minimal) set of processors.
processors = [EnvExpander, PathNormalizer, DuplicatesRemover,
# special processors
EmptyDirsRemover, UsePythonZip
]
# FIXME: these are back-ward compatibility hacks: we need a proper way to add/remove processors
if ('no-strip-path' in os.environ.get('CMTEXTRATAGS', '')
or 'GAUDI_NO_STRIP_PATH' in os.environ
or 'LB_NO_STRIP_PATH' in os.environ):
processors.remove(EmptyDirsRemover)
if 'no-pyzip' in os.environ.get('CMTEXTRATAGS', ''):
processors.remove(UsePythonZip)
class VariableBase(object):
'''
Base class for the classes used to manipulate the environment.
'''
def __init__(self, name, local=False):
self.varName = name
self.local = local
self.expandVars = True
self.log = logging.getLogger('Variable')
def process(self, value, env):
'''
Call all the processors defined in the processors list on 'value'.
@return: the processed value
'''
for p in [c(env) for c in processors]:
if p.isTarget(self):
value = p(self, value)
return value
class List(VariableBase):
'''
Class for manipulating with environment lists.
It holds its name and values represented by a list.
Some operations are done with separator, which is usually colon. For windows use semicolon.
'''
def __init__(self, name, local=False):
super(List, self).__init__(name, local)
self.val = []
def name(self):
'''Returns the name of the List.'''
return self.varName
def set(self, value, separator=':', environment=None):
'''Sets the value of the List. Any previous value is overwritten.'''
if isinstance(value, str):
value = value.split(separator)
self.val = self.process(value, environment)
def unset(self, value, separator=':', environment=None): # pylint: disable=W0613
'''Sets the value of the List to empty. Any previous value is overwritten.'''
self.val = []
def value(self, asString=False, separator=':'):
'''Returns values of the List. Either as a list or string with desired separator.'''
if asString:
return separator.join(self.val)
else:
# clone the list
return list(self.val)
def remove_regexp(self, value, separator=':'):
self.remove(value, separator, True)
def remove(self, value, separator=':', regexp=False):
'''Removes value(s) from List. If value is not found, removal is canceled.'''
if regexp:
value = self.search(value, True)
elif isinstance(value, str):
value = value.split(separator)
for i in range(len(value)):
val = value[i]
if val not in value:
self.log.info(
'Value "%s" not found in List: "%s". Removal canceled.', val, self.varName)
while val in self.val:
self.val.remove(val)
def append(self, value, separator=':', environment=None):
'''Adds value(s) at the end of the list.'''
if isinstance(value, str):
value = value.split(separator)
self.val = self.process(self.val + value, environment)
def prepend(self, value, separator=':', environment=None):
'''Adds value(s) at the beginning of the list.
resolve references and duplications'''
if isinstance(value, str):
value = value.split(separator)
self.val = self.process(value + self.val, environment)
def search(self, expr, regExp):
'''Searches in List's values for a match
Use string value or set regExp to True.
In the first case search is done only for an exact match for one of List`s value ('^' and '$' added).
'''
if not regExp:
expr = '^' + expr + '$'
v = re.compile(expr)
res = []
for val in self.val:
if v.search(val):
res.append(val)
return res
def __getitem__(self, key):
return self.val[key]
def __setitem__(self, key, value):
if value in self.val:
self.log.info(
'Var: "%s" value: "%s". Addition canceled because of duplicate entry.', self.varName, value)
else:
self.val.insert(key, value)
def __delitem__(self, key):
self.remove(self.val[key])
def __iter__(self):
for i in self.val:
yield i
def __contains__(self, item):
return item in self.val
def __len__(self):
return len(self.val)
def __str__(self):
return ':'.join(self.val)
class Scalar(VariableBase):
'''Class for manipulating with environment scalars.'''
def __init__(self, name, local=False):
super(Scalar, self).__init__(name, local)
self.val = ''
def name(self):
'''Returns the name of the scalar.'''
return self.varName
def set(self, value, separator=':', environment=None): # pylint: disable=W0613
'''Sets the value of the scalar. Any previous value is overwritten.'''
self.val = self.process(value, environment)
def unset(self, value, separator=':', environment=None): # pylint: disable=W0613
'''Sets the value of the variable to empty. Any previous value is overwritten.'''
self.val = ''
def value(self, asString=False, separator=':'): # pylint: disable=W0613
'''Returns values of the scalar.'''
return self.val
def remove_regexp(self, value, separator=':'):
self.remove(value, separator, True)
def remove(self, value, separator=':', regexp=True): # pylint: disable=W0613
'''Removes value(s) from the scalar. If value is not found, removal is canceled.'''
value = self.search(value)
for val in value:
self.val = self.val.replace(val, '')
def append(self, value, separator=':', environment=None): # pylint: disable=W0613
'''Adds value(s) at the end of the scalar.'''
self.val += self.process(value, environment)
def prepend(self, value, separator=':', environment=None): # pylint: disable=W0613
'''Adds value(s) at the beginning of the scalar.'''
self.val = self.process(value, environment) + self.val
def search(self, expr):
'''Searches in scalar`s values for a match'''
return re.findall(expr, self.val)
def __str__(self):
return self.val
class EnvError(Exception):
'''Class which defines errors for locals operations.'''
def __init__(self, value, code):
super(EnvError, self).__init__()
self.val = value
self.code = code
def __str__(self):
if self.code == 'undefined':
return 'Reference to undefined environment element: "' + self.val + '".'
elif self.code == 'ref2var':
return 'Reference to list from the middle of string.'
elif self.code == 'redeclaration':
return 'Wrong redeclaration of environment element "' + self.val + '".'
cmake/EnvConfig/__init__.py deleted 100644 → 0
+ 0
− 281
View file @ 6ebc5d99
__author__ = "Marco Clemencic <marco.clemencic@cern.ch>"
import os
import sys
assert sys.version_info >= (2, 6), "Python 2.6 required"
import logging
from string import Template
__all__ = []
# Prepare the search path for environment XML files
path = ['.']
if 'ENVXMLPATH' in os.environ:
path.extend(os.environ['ENVXMLPATH'].split(os.pathsep))
import Control
class EnvError(RuntimeError):
'''
Simple class to wrap errors in the environment configuration.
'''
pass
def splitNameValue(name_value):
"""split the "NAME=VALUE" string into the tuple ("NAME", "VALUE")
replacing '[:]' with os.pathsep in VALUE"""
if '=' not in name_value:
raise EnvError("Invalid variable argument '%s'." % name_value)
n, v = name_value.split('=', 1)
return n, v.replace('[:]', os.pathsep)
class Script(object):
'''
Environment Script class used to control the logic of the script and allow
extensions.
'''
__usage__ = "Usage: %prog [OPTION]... [NAME=VALUE]... [COMMAND [ARG]...]"
__desc__ = "Set each NAME to VALUE in the environment and run COMMAND."
__epilog__ = ("The operations are performed in the order they appear on the "
"command line. If no COMMAND is provided, print the resulting "
"environment. (Note: this command is modeled after the Unix "
"command 'env', see \"man env\")")
def __init__(self, args=None):
'''
Initializes the script instance parsing the command line arguments (or
the explicit arguments provided).
'''
self.parser = None
self.opts = None
self.cmd = None
self.control = None
self.log = None
self.env = {}
# Run the core code of the script
self._prepare_parser()
self._parse_args(args)
self._check_args()
def _prepare_parser(self):
'''
Prepare an OptionParser instance used to analyze the command line
options and arguments.
'''
from optparse import OptionParser, OptionValueError
parser = OptionParser(prog=os.path.basename(sys.argv[0]),
usage=self.__usage__,
description=self.__desc__,
epilog=self.__epilog__)
self.log = logging.getLogger(parser.prog)
def addOperation(option, opt, value, parser, action):
'''
Append to the list of actions the tuple (action, (<args>, ...)).
'''
if action not in ('unset', 'loadXML'):
try:
value = splitNameValue(value)
except EnvError:
raise OptionValueError(
"Invalid value for option %s: '%s', it requires NAME=VALUE." % (opt, value))
else:
value = (value,)
parser.values.actions.append((action, value))
parser.add_option("-i", "--ignore-environment",
action="store_true",
help="start with an empty environment")
parser.add_option("-u", "--unset",
metavar="NAME",
action="callback", callback=addOperation,
type="str", nargs=1, callback_args=('unset',),
help="remove variable from the environment")
parser.add_option("-s", "--set",
metavar="NAME=VALUE",
action="callback", callback=addOperation,
type="str", nargs=1, callback_args=('set',),
help="set the variable NAME to VALUE")
parser.add_option("-a", "--append",
metavar="NAME=VALUE",
action="callback", callback=addOperation,
type="str", nargs=1, callback_args=('append',),
help="append VALUE to the variable NAME (with a '%s' as separator)" % os.pathsep)
parser.add_option("-p", "--prepend",
metavar="NAME=VALUE",
action="callback", callback=addOperation,
type="str", nargs=1, callback_args=('prepend',),
help="prepend VALUE to the variable NAME (with a '%s' as separator)" % os.pathsep)
parser.add_option("-x", "--xml",
action="callback", callback=addOperation,
type="str", nargs=1, callback_args=('loadXML',),
help="XML file describing the changes to the environment")
parser.add_option("--sh",
action="store_const", const="sh", dest="shell",
help="Print the changes to the environment as shell commands for 'sh'-derived shells.")
parser.add_option("--csh",
action="store_const", const="csh", dest="shell",
help="Print the changes to the environment as shell commands for 'csh'-derived shells.")
parser.add_option("--py",
action="store_const", const="py", dest="shell",
help="Print the changes to the environment as Python dictionary.")
parser.add_option("-A", "--all",
action="store_true",
help="Print all variables, instead of just the changes, with --sh, --csh and --py.")
parser.add_option('--verbose', action='store_const',
const=logging.INFO, dest='log_level',
help='print more information')
parser.add_option('--debug', action='store_const',
const=logging.DEBUG, dest='log_level',
help='print debug messages')
parser.add_option('--quiet', action='store_const',
const=logging.WARNING, dest='log_level',
help='print only warning messages (default)')
parser.disable_interspersed_args()
parser.set_defaults(actions=[],
ignore_environment=False,
log_level=logging.WARNING)
self.parser = parser
def _parse_args(self, args=None):
'''
Parse the command line arguments.
'''
opts, args = self.parser.parse_args(args)
# set the logging level
logging.basicConfig(level=opts.log_level)
cmd = []
# find the (implicit) 'set' arguments in the list of arguments
# and put the rest in the command
try:
for i, a in enumerate(args):
opts.actions.append(('set', splitNameValue(a)))
except EnvError:
cmd = args[i:]
self.opts, self.cmd = opts, cmd
def _check_args(self):
'''
Check consistency of command line options and arguments.
'''
if self.opts.shell and self.cmd:
self.parser.error(
"Invalid arguments: --%s cannot be used with a command." % self.opts.shell)
def _makeEnv(self):
'''
Generate a dictionary of the environment variables after applying all
the required actions.
'''
# prepare the environment control instance
control = Control.Environment()
if not self.opts.ignore_environment:
control.presetFromSystem()
# apply all the actions
for action, args in self.opts.actions:
apply(getattr(control, action), args)
# extract the result env dictionary
env = control.vars()
# set the library search path correctly for the non-Linux platforms
if "LD_LIBRARY_PATH" in env:
# replace LD_LIBRARY_PATH with the corresponding one on other systems
if sys.platform.startswith("win"):
other = "PATH"
elif sys.platform.startswith("darwin"):
other = "DYLD_LIBRARY_PATH"
else:
other = None
if other:
if other in env:
env[other] = env[other] + \
os.pathsep + env["LD_LIBRARY_PATH"]
else:
env[other] = env["LD_LIBRARY_PATH"]
del env["LD_LIBRARY_PATH"]
self.env = env
def dump(self):
'''
Print to standard output the final environment in the required format.
'''
if not self.opts.shell or self.opts.all:
# 'env' behaviour: print the whole environment
env = self.env
else:
# special dumps: print only the diffs
env = dict((name, value)
for name, value in sorted(self.env.items())
if os.environ.get(name) != value)
if self.opts.shell == 'py':
from pprint import pprint
pprint(env)
else:
template = {'sh': "export {0}='{1}'",
'csh': "setenv {0} '{1}';"}.get(self.opts.shell,
"{0}={1}")
print '\n'.join(template.format(name, value)
for name, value in sorted(env.items()))
def expandEnvVars(self, iterable):
'''
Return a copy of iterable where all the elements have the environment
variables expanded.
>>> s = Script([])
>>> s.env = {'A': '1', 'B': 'test'}
>>> s.expandEnvVars(['$A', '${B}-$A', '$DUMMY-$A', '$$B'])
['1', 'test-1', '$DUMMY-1', '$B']
'''
return [Template(elem).safe_substitute(self.env) for elem in iterable]
def runCmd(self):
'''
Execute a command in the modified environment and return the exit code.
'''
from subprocess import Popen
cmd = self.expandEnvVars(self.cmd)
try:
proc = Popen(cmd, env=self.env)
except OSError, x:
print >> sys.stderr, '{0}: {1}: {2}'.format(sys.argv[0],
cmd[0],
x.strerror)
sys.exit(127)
while proc.poll() is None:
try:
proc.wait()
except KeyboardInterrupt:
self.log.fatal('KeyboardInterrupt, '
'waiting for subprocess %d to end', proc.pid)
rc = proc.returncode
# There is a mismatch between Popen return code and sys.exit argument in
# case of signal.
# E.g. Popen returns -6 that is translated to 250 instead of 134
return rc if rc >= 0 else 128 - rc
def main(self):
'''
Main function of the script.
'''
self._makeEnv()
if not self.cmd:
self.dump()
else:
sys.exit(self.runCmd())
cmake/EnvConfig/_runtests.py deleted 100644 → 0
+ 0
− 7
View file @ 6ebc5d99
"""
Wrapper to run the tests.
"""
import os
from nose import main
main(defaultTest=os.path.normpath(__file__ + "/../.."))
cmake/EnvConfig/xmlModule.py deleted 100644 → 0
+ 0
− 142
View file @ 6ebc5d99
'''
Created on Jul 2, 2011
@author: mplajner
'''
import logging
from cPickle import load, dump
from hashlib import md5 # pylint: disable=E0611
from xml.dom import minidom
from xml.parsers.expat import ExpatError
class XMLFile(object):
'''Takes care of XML file operations such as reading and writing.'''
def __init__(self):
self.xmlResult = '<?xml version="1.0" encoding="UTF-8"?><env:config xmlns:env="EnvSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="EnvSchema ./EnvSchema.xsd ">\n'
self.declaredVars = []
self.log = logging.getLogger('XMLFile')
def variable(self, path, namespace='EnvSchema', name=None):
'''Returns list containing name of variable, action and value.
@param path: a file name or a file-like object
If no name given, returns list of lists of all variables and locals(instead of action 'local' is filled).
'''
isFilename = type(path) is str
if isFilename:
checksum = md5()
checksum.update(open(path, 'rb').read())
checksum = checksum.digest()
cpath = path + "c" # preparsed file
try:
f = open(cpath, 'rb')
oldsum, data = load(f)
if oldsum == checksum:
return data
except IOError:
pass
except EOFError:
pass
caller = path
else:
caller = None
# Get file
try:
doc = minidom.parse(path)
except ExpatError, exc:
self.log.fatal('Failed to parse %s: %s', path, exc)
self.log.fatal(list(open(path))[exc.lineno - 1].rstrip())
self.log.fatal(' ' * exc.offset + '^')
raise SystemExit(1)
if namespace == '':
namespace = None
ELEMENT_NODE = minidom.Node.ELEMENT_NODE
# Get all variables
nodes = doc.getElementsByTagNameNS(namespace, "config")[0].childNodes
variables = []
for node in nodes:
# if it is an element node
if node.nodeType == ELEMENT_NODE:
action = str(node.localName)
if action == 'include':
if node.childNodes:
value = str(node.childNodes[0].data)
else:
value = ''
variables.append(
(action, (value, caller, str(node.getAttribute('hints')))))
elif action == 'search_path':
if node.childNodes:
value = str(node.childNodes[0].data)
else:
value = ''
variables.append((action, (value, None, None)))
else:
varname = str(node.getAttribute('variable'))
if name and varname != name:
continue
if action == 'declare':
variables.append((action, (varname, str(
node.getAttribute('type')), str(node.getAttribute('local')))))
else:
if node.childNodes:
value = str(node.childNodes[0].data)
else:
value = ''
variables.append((action, (varname, value, None)))
if isFilename:
try:
f = open(cpath, 'wb')
dump((checksum, variables), f, protocol=2)
f.close()
except IOError:
pass
return variables
def resetWriter(self):
'''resets the buffer of writer'''
self.xmlResult = '<?xml version="1.0" encoding="UTF-8"?><env:config xmlns:env="EnvSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="EnvSchema ./EnvSchema.xsd ">\n'
self.declaredVars = []
def writeToFile(self, outputFile=None):
'''Finishes the XML input and writes XML to file.'''
if outputFile is None:
raise IOError("No output file given")
self.xmlResult += '</env:config>'
doc = minidom.parseString(self.xmlResult)
with open(outputFile, "w") as f:
f.write(doc.toxml())
return outputFile
def writeVar(self, varName, action, value, vartype='list', local=False):
'''Writes a action to a file. Declare undeclared elements (non-local list is default type).'''
if action == 'declare':
self.xmlResult += '<env:declare variable="' + varName + '" type="' + \
vartype.lower() + '" local="' + (str(local)).lower() + '" />\n'
self.declaredVars.append(varName)
return
if varName not in self.declaredVars:
self.xmlResult += '<env:declare variable="' + varName + '" type="' + \
vartype + '" local="' + (str(local)).lower() + '" />\n'
self.declaredVars.append(varName)
self.xmlResult += '<env:' + action + ' variable="' + \
varName + '">' + value + '</env:' + action + '>\n'
cmake/GaudiProjectConfig.cmake
+ 31
− 5
View file @ 561db725
......@@ -398,8 +398,37 @@ macro(gaudi_project project version)
# (python scripts are located as such but run through python)
set(binary_paths ${CMAKE_SOURCE_DIR}/cmake ${CMAKE_SOURCE_DIR}/GaudiPolicy/scripts ${CMAKE_SOURCE_DIR}/GaudiKernel/scripts ${CMAKE_SOURCE_DIR}/Gaudi/scripts ${binary_paths})
find_program(env_cmd xenv HINTS ${binary_paths})
set(env_cmd ${PYTHON_EXECUTABLE} ${env_cmd})
find_program(env_cmd NAMES xenv)
if(NOT env_cmd)
if(NOT EXISTS ${CMAKE_BINARY_DIR}/contrib/xenv)
execute_process(COMMAND git clone https://gitlab.cern.ch/gaudi/xenv.git ${CMAKE_BINARY_DIR}/contrib/xenv)
endif()
# I'd like to generate the script with executable permission, but I only
# found this workaround: https://stackoverflow.com/a/45515418
# - write a temporary file
file(WRITE "${CMAKE_BINARY_DIR}/xenv"
"#!/usr/bin/env python
from os.path import join, dirname, pardir
import sys
sys.path.append(join(dirname(__file__), pardir, 'python'))
from xenv import main
main()")
# - copy it to the right place with right permissions
file(COPY "${CMAKE_BINARY_DIR}/xenv" DESTINATION "${CMAKE_BINARY_DIR}/bin"
FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
# - remove the temporary
file(REMOVE "${CMAKE_BINARY_DIR}/xenv")
# we have to copy the Python package to the bin directory to be able to run xenv
# without setting PYTHONPATH
file(COPY ${CMAKE_BINARY_DIR}/contrib/xenv/xenv DESTINATION "${CMAKE_BINARY_DIR}/python")
set(env_cmd "${CMAKE_BINARY_DIR}/bin/xenv" CACHE FILEPATH "Path to xenv command" FORCE)
install(PROGRAMS ${CMAKE_BINARY_DIR}/bin/xenv DESTINATION scripts)
install(DIRECTORY ${CMAKE_BINARY_DIR}/contrib/xenv/xenv DESTINATION python
FILES_MATCHING PATTERN "*.py" PATTERN "*.conf")
endif()
find_program(default_merge_cmd quick-merge HINTS ${binary_paths})
set(default_merge_cmd ${PYTHON_EXECUTABLE} ${default_merge_cmd})
......@@ -462,9 +491,6 @@ macro(gaudi_project project version)
PATTERN "*.cmake.in"
PATTERN "*.py"
PATTERN ".svn" EXCLUDE)
install(PROGRAMS cmake/xenv DESTINATION scripts OPTIONAL)
install(DIRECTORY cmake/EnvConfig DESTINATION scripts
FILES_MATCHING PATTERN "*.py" PATTERN "*.conf")
if (CMAKE_EXPORT_COMPILE_COMMANDS)
install(FILES ${CMAKE_BINARY_DIR}/compile_commands.json DESTINATION . OPTIONAL)
......
cmake/tests/test_LBCORE_716.py
+ 1
− 1
View file @ 561db725
......@@ -7,7 +7,7 @@ from subprocess import Popen, PIPE
base_dir = os.path.join(os.path.dirname(__file__), 'data', 'LBCORE-716')
xenv_cmd = os.path.join(os.path.dirname(__file__), os.pardir, 'xenv')
xenv_cmd = 'xenv'
build_log = None
build_returncode = None
......
cmake/tests/test_LBCORE_725.py
+ 1
− 1
View file @ 561db725
......@@ -8,7 +8,7 @@ base_dir = os.path.join(os.path.dirname(__file__), 'data', 'LBCORE-725')
build_dir = os.path.join(base_dir, 'tmp')
release_dir = os.path.join(base_dir, 'dest')
xenv_cmd = os.path.join(os.path.dirname(__file__), os.pardir, 'xenv')
xenv_cmd = 'xenv'
build_log = None
build_returncode = None
......
cmake/xenv deleted 100755 → 0
+ 0
− 3
View file @ 6ebc5d99
#!/usr/bin/env python
import EnvConfig
EnvConfig.Script().main()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment