Commit 0e0deb97 authored by Gitlab CI's avatar Gitlab CI Committed by Marco Clemencic
Browse files

Fixed formatting

patch generated by standalone job
parent b59ca0d1
# File: Gaudi/CommonGaudiConfigurables.py
# Author: Pere Mato (pere.mato@cern.ch)
"""
This module would scan all known Gaudi configurable modules for
'Configurable' classes and fill __all__ such that it can be imported
......@@ -11,16 +10,17 @@ from GaudiKernel.Configurable import Configurable
from GaudiKernel.ConfigurableMeta import ConfigurableMeta
__all__ = []
packages = ['GaudiCoreSvc', 'GaudiCommonSvc', 'GaudiSvc', 'GaudiAlg',
'GaudiAud', 'GaudiPoolDb', 'RootHistCnv', 'GaudiUtils',
'RootCnv']
packages = [
'GaudiCoreSvc', 'GaudiCommonSvc', 'GaudiSvc', 'GaudiAlg', 'GaudiAud',
'GaudiPoolDb', 'RootHistCnv', 'GaudiUtils', 'RootCnv'
]
# --Loop open all listed packages and populate __all__ with the names and
# the local scope with the Configurable classes
for package in packages:
try:
mod = __import__('%s.%sConf' % (package, package),
globals(), locals(), ['%sConf' % package])
mod = __import__('%s.%sConf' % (package, package), globals(), locals(),
['%sConf' % package])
for nam in dir(mod):
cls = getattr(mod, nam)
if type(cls) is ConfigurableMeta and issubclass(cls, Configurable):
......@@ -32,29 +32,26 @@ for package in packages:
# --Fix some of the name idiosyncrasies in Gaudi
aliases = {
'EventDataSvc': 'EvtDataSvc',
'DetectorDataSvc': 'DetDataSvc',
'HistogramDataSvc': 'HistogramSvc',
'HbookHistSvc': 'HbookCnv__PersSvc',
'RootHistSvc': 'RootHistCnv__PersSvc',
'EventPersistencySvc': 'EvtPersistencySvc',
'DetectorPersistencySvc': 'DetPersistencySvc',
'HistogramPersistencySvc': 'HistogramPersistencySvc',
'EventDataSvc': 'EvtDataSvc',
'DetectorDataSvc': 'DetDataSvc',
'HistogramDataSvc': 'HistogramSvc',
'HbookHistSvc': 'HbookCnv__PersSvc',
'RootHistSvc': 'RootHistCnv__PersSvc',
'EventPersistencySvc': 'EvtPersistencySvc',
'DetectorPersistencySvc': 'DetPersistencySvc',
'HistogramPersistencySvc': 'HistogramPersistencySvc',
'FileRecordPersistencySvc': 'PersistencySvc',
'FileCatalog': 'Gaudi__MultiFileCatalog',
'IODataManager': 'Gaudi__IODataManager',
'RootCnvSvc': 'Gaudi__RootCnvSvc',
'RootEvtSelector': 'Gaudi__RootEvtSelector',
'FileCatalog': 'Gaudi__MultiFileCatalog',
'IODataManager': 'Gaudi__IODataManager',
'RootCnvSvc': 'Gaudi__RootCnvSvc',
'RootEvtSelector': 'Gaudi__RootEvtSelector',
}
_gbl = globals() # optimization
# This would be nicer with dict comprehension (http://www.python.org/dev/peps/pep-0274)
# but it is available only in Python 2.7
aliases = dict([(new, _gbl[old])
for new, old in aliases.items()
if old in _gbl]) # do the aliasing only if the original is available
aliases = dict([(new, _gbl[old]) for new, old in aliases.items() if old in _gbl
]) # do the aliasing only if the original is available
# change the default name
for new in aliases:
aliases[new].DefaultedName = new
......
......@@ -26,14 +26,12 @@ def _fillConfDict():
log.warning("Found %d duplicates among the %d genConfDb files :",
len(cfgDb.duplicates()), nFiles)
log.warning("--------------------------------------------------")
log.warning(" -%s: %s - %s",
"<component name>", "<module>", "[ <duplicates> ]")
log.warning(" -%s: %s - %s", "<component name>", "<module>",
"[ <duplicates> ]")
log.warning("--------------------------------------------------")
dups = cfgDb.duplicates()
for cfgName in dups.keys():
log.warning(" -%s: %s - %s",
cfgName,
cfgDb[cfgName]['module'],
log.warning(" -%s: %s - %s", cfgName, cfgDb[cfgName]['module'],
str([d['module'] for d in dups[cfgName]]))
pass
del dups
......@@ -134,21 +132,22 @@ class GaudiPersistency(ConfigurableUser):
def __apply_configuration__(self):
"""Apply low-level configuration"""
from Configurables import (ApplicationMgr,
PersistencySvc,
FileRecordDataSvc,
EventPersistencySvc,
)
from Configurables import (
ApplicationMgr,
PersistencySvc,
FileRecordDataSvc,
EventPersistencySvc,
)
# aliased names
from Configurables import (RootCnvSvc,
RootEvtSelector,
IODataManager,
FileCatalog,
)
from Configurables import (
RootCnvSvc,
RootEvtSelector,
IODataManager,
FileCatalog,
)
cnvSvcs = [RootCnvSvc()]
EventPersistencySvc().CnvServices += cnvSvcs
PersistencySvc("FileRecordPersistencySvc").CnvServices += cnvSvcs
app = ApplicationMgr()
app.SvcOptMapping += [FileCatalog(), IODataManager(),
RootCnvSvc()]
app.SvcOptMapping += [FileCatalog(), IODataManager(), RootCnvSvc()]
app.ExtSvc += [FileRecordDataSvc()]
......@@ -14,6 +14,7 @@ class BootstrapHelper(object):
def __bool__(self):
return self.value
__nonzero__ = __bool__
def isSuccess(self):
......@@ -31,6 +32,7 @@ class BootstrapHelper(object):
def __str__(self):
return str(self.value)
toString = __str__
class AppMgr(object):
......@@ -40,34 +42,43 @@ class BootstrapHelper(object):
self._as_parameter_ = ptr
def configure(self):
return BootstrapHelper.StatusCode(self.lib.py_bootstrap_fsm_configure(self.ptr))
return BootstrapHelper.StatusCode(
self.lib.py_bootstrap_fsm_configure(self.ptr))
def initialize(self):
return BootstrapHelper.StatusCode(self.lib.py_bootstrap_fsm_initialize(self.ptr))
return BootstrapHelper.StatusCode(
self.lib.py_bootstrap_fsm_initialize(self.ptr))
def start(self):
return BootstrapHelper.StatusCode(self.lib.py_bootstrap_fsm_start(self.ptr))
return BootstrapHelper.StatusCode(
self.lib.py_bootstrap_fsm_start(self.ptr))
def run(self, nevt):
return BootstrapHelper.StatusCode(self.lib.py_bootstrap_app_run(self.ptr, nevt))
return BootstrapHelper.StatusCode(
self.lib.py_bootstrap_app_run(self.ptr, nevt))
def stop(self):
return BootstrapHelper.StatusCode(self.lib.py_bootstrap_fsm_stop(self.ptr))
return BootstrapHelper.StatusCode(
self.lib.py_bootstrap_fsm_stop(self.ptr))
def finalize(self):
return BootstrapHelper.StatusCode(self.lib.py_bootstrap_fsm_finalize(self.ptr))
return BootstrapHelper.StatusCode(
self.lib.py_bootstrap_fsm_finalize(self.ptr))
def terminate(self):
return BootstrapHelper.StatusCode(self.lib.py_bootstrap_fsm_terminate(self.ptr))
return BootstrapHelper.StatusCode(
self.lib.py_bootstrap_fsm_terminate(self.ptr))
def getService(self, name):
return self.lib.py_bootstrap_getService(self.ptr, name)
def setProperty(self, name, value):
return BootstrapHelper.StatusCode(self.lib.py_bootstrap_setProperty(self.ptr, name, value))
return BootstrapHelper.StatusCode(
self.lib.py_bootstrap_setProperty(self.ptr, name, value))
def getProperty(self, name):
return BootstrapHelper.Property(self.lib.py_bootstrap_getProperty(self.ptr, name))
return BootstrapHelper.Property(
self.lib.py_bootstrap_getProperty(self.ptr, name))
def printAlgsSequences(self):
return self.lib.py_helper_printAlgsSequences(self.ptr)
......@@ -75,13 +86,15 @@ class BootstrapHelper(object):
def __init__(self):
from ctypes import (PyDLL, util, c_void_p, c_bool, c_char_p, c_int,
RTLD_GLOBAL)
# Helper class to avoid void* to int conversion
# (see http://stackoverflow.com/questions/17840144)
class IInterface_p(c_void_p):
def __repr__(self):
return "IInterface_p(0x%x)" % (0 if self.value is None
else self.value)
return "IInterface_p(0x%x)" % (0 if self.value is None else
self.value)
self.log = logging.getLogger('BootstrapHelper')
libname = util.find_library('GaudiKernel') or 'libGaudiKernel.so'
self.log.debug('loading GaudiKernel (%s)', libname)
......@@ -90,15 +103,15 @@ class BootstrapHelper(object):
# Python functions are not protected with the GIL.
self.lib = gkl = PyDLL(libname, mode=RTLD_GLOBAL)
functions = [('createApplicationMgr', IInterface_p, []),
('getService', IInterface_p, [IInterface_p, c_char_p]),
('setProperty', c_bool, [
IInterface_p, c_char_p, c_char_p]),
('getProperty', c_char_p, [IInterface_p, c_char_p]),
('addPropertyToCatalogue', c_bool, [
IInterface_p, c_char_p, c_char_p, c_char_p]),
('ROOT_VERSION_CODE', c_int, []),
]
functions = [
('createApplicationMgr', IInterface_p, []),
('getService', IInterface_p, [IInterface_p, c_char_p]),
('setProperty', c_bool, [IInterface_p, c_char_p, c_char_p]),
('getProperty', c_char_p, [IInterface_p, c_char_p]),
('addPropertyToCatalogue', c_bool,
[IInterface_p, c_char_p, c_char_p, c_char_p]),
('ROOT_VERSION_CODE', c_int, []),
]
for name, restype, argtypes in functions:
f = getattr(gkl, 'py_bootstrap_%s' % name)
......@@ -108,8 +121,8 @@ class BootstrapHelper(object):
if name not in self.__class__.__dict__:
setattr(self, name, f)
for name in ('configure', 'initialize', 'start',
'stop', 'finalize', 'terminate'):
for name in ('configure', 'initialize', 'start', 'stop', 'finalize',
'terminate'):
f = getattr(gkl, 'py_bootstrap_fsm_%s' % name)
f.restype, f.argtypes = c_bool, [IInterface_p]
gkl.py_bootstrap_app_run.restype = c_bool
......@@ -153,8 +166,9 @@ def toOpt(value):
if isinstance(value, basestring):
return '"{0}"'.format(value.replace('"', '\\"'))
elif isinstance(value, dict):
return '{{{0}}}'.format(', '.join('{0}: {1}'.format(toOpt(k), toOpt(v))
for k, v in value.iteritems()))
return '{{{0}}}'.format(', '.join(
'{0}: {1}'.format(toOpt(k), toOpt(v))
for k, v in value.iteritems()))
elif hasattr(value, '__iter__'):
return '[{0}]'.format(', '.join(map(toOpt, value)))
else:
......@@ -261,8 +275,8 @@ class gaudimain(object):
if ext in write:
write[ext](filename, all)
else:
log.error("Unknown file type '%s'. Must be any of %r.",
ext, write.keys())
log.error("Unknown file type '%s'. Must be any of %r.", ext,
write.keys())
sys.exit(1)
# Instantiate and run the application.
......@@ -279,8 +293,9 @@ class gaudimain(object):
def hookDebugger(self, debugger='gdb'):
import os
self.log.info('attaching debugger to PID ' + str(os.getpid()))
pid = os.spawnvp(os.P_NOWAIT,
debugger, [debugger, '-q', 'python', str(os.getpid())])
pid = os.spawnvp(os.P_NOWAIT, debugger,
[debugger, '-q', 'python',
str(os.getpid())])
# give debugger some time to attach to the python process
import time
......@@ -299,7 +314,8 @@ class gaudimain(object):
from GaudiKernel.Proxy.Configurable import expandvars
except ImportError:
# pass-through implementation if expandvars is not defined (AthenaCommon)
def expandvars(data): return data
def expandvars(data):
return data
from GaudiKernel.Proxy.Configurable import Configurable, getNeededConfigurables
......@@ -355,7 +371,8 @@ class gaudimain(object):
for p, v in c.getValuedProperties().items():
v = expandvars(v)
# Note: AthenaCommon.Configurable does not have Configurable.PropertyReference
if hasattr(Configurable, "PropertyReference") and type(v) == Configurable.PropertyReference:
if hasattr(Configurable, "PropertyReference") and type(
v) == Configurable.PropertyReference:
# this is done in "getFullName", but the exception is ignored,
# so we do it again to get it
v = v.__resolve__()
......@@ -381,8 +398,7 @@ class gaudimain(object):
def runSerial(self, attach_debugger):
# --- Instantiate the ApplicationMgr------------------------------
if (self.mainLoop or
os.environ.get('GAUDIRUN_USE_GAUDIPYTHON')):
if (self.mainLoop or os.environ.get('GAUDIRUN_USE_GAUDIPYTHON')):
self.gaudiPythonInit()
else:
self.basicInit()
......@@ -395,6 +411,7 @@ class gaudimain(object):
if self.mainLoop:
runner = self.mainLoop
else:
def runner(app, nevt):
self.log.debug('initialize')
sc = app.initialize()
......@@ -424,8 +441,8 @@ class gaudimain(object):
self.hookDebugger()
try:
statuscode = runner(self.g, int(
self.ip.getProperty('EvtMax').toString()))
statuscode = runner(self.g,
int(self.ip.getProperty('EvtMax').toString()))
except SystemError:
# It may not be 100% correct, but usually it means a segfault in C++
self.ip.setProperty('ReturnCode', str(128 + 11))
......@@ -445,15 +462,16 @@ class gaudimain(object):
self.ip.setProperty('ReturnCode', '1')
sysTime = time() - sysStart
self.log.debug('-' * 80)
self.log.debug(
'%s: serial system finished, time taken: %5.4fs', __name__, sysTime)
self.log.debug('%s: serial system finished, time taken: %5.4fs',
__name__, sysTime)
self.log.debug('-' * 80)
return int(self.ip.getProperty('ReturnCode').toString())
def runParallel(self, ncpus):
if self.mainLoop:
self.log.fatal(
"Cannot use custom main loop in multi-process mode, check your options")
"Cannot use custom main loop in multi-process mode, check your options"
)
return 1
self.setupParallelLogging()
from Gaudi.Configuration import Configurable
......@@ -461,14 +479,15 @@ class gaudimain(object):
c = Configurable.allConfigurables
self.log.info('-' * 80)
self.log.info('%s: Parallel Mode : %i ', __name__, ncpus)
for name, value in [('platrofm', ' '.join(os.uname())),
('config', os.environ.get('BINARY_TAG') or
os.environ.get('CMTCONFIG')),
('app. name', os.environ.get('GAUDIAPPNAME')),
('app. version', os.environ.get('GAUDIAPPVERSION')),
]:
self.log.info('%s: %30s : %s ', __name__,
name, value or 'Undefined')
for name, value in [
('platrofm', ' '.join(os.uname())),
('config', os.environ.get('BINARY_TAG')
or os.environ.get('CMTCONFIG')),
('app. name', os.environ.get('GAUDIAPPNAME')),
('app. version', os.environ.get('GAUDIAPPVERSION')),
]:
self.log.info('%s: %30s : %s ', __name__, name, value
or 'Undefined')
try:
events = str(c['ApplicationMgr'].EvtMax)
except:
......@@ -485,7 +504,7 @@ class gaudimain(object):
sysTime = time() - sysStart
self.log.name = 'Gaudi/Main.py Logger'
self.log.info('-' * 80)
self.log.info(
'%s: parallel system finished, time taken: %5.4fs', __name__, sysTime)
self.log.info('%s: parallel system finished, time taken: %5.4fs',
__name__, sysTime)
self.log.info('-' * 80)
return 0
......@@ -42,8 +42,8 @@ class _ConfigurablesModule(object):
else:
# We raise an AttributeError exception if the configurable could not be found
# to respect the Python semantic.
raise AttributeError("module '%s' does not have attribute '%s'" % (
__configurables_module_fullname__, name))
raise AttributeError("module '%s' does not have attribute '%s'" %
(__configurables_module_fullname__, name))
return retval
......
......@@ -43,12 +43,13 @@ def setLibraryPreload(newpreload):
preload = []
for libname in set(preload).intersection(newpreload):
logging.warning("Ignoring preload of library %s because it is "
"already in LD_PRELOAD.", libname)
logging.warning(
"Ignoring preload of library %s because it is "
"already in LD_PRELOAD.", libname)
to_load = [libname
for libname in newpreload
if libname not in set(preload)]
to_load = [
libname for libname in newpreload if libname not in set(preload)
]
if to_load:
preload += to_load
......@@ -90,8 +91,9 @@ def getArgsFromQmt(qmtfile):
if options is not None: # options need to be dumped in a temporary file
from tempfile import NamedTemporaryFile
import re
if re.search(r"from\s+Gaudi.Configuration\s+import\s+\*"
r"|from\s+Configurables\s+import", options.text):
if re.search(
r"from\s+Gaudi.Configuration\s+import\s+\*"
r"|from\s+Configurables\s+import", options.text):
tmp_opts = NamedTemporaryFile(suffix='.py')
else:
tmp_opts = NamedTemporaryFile(suffix='.opts')
......@@ -128,108 +130,165 @@ if __name__ == "__main__":
from optparse import OptionParser
parser = OptionParser(usage="%prog [options] <opts_file> ...")
parser.add_option("-n", "--dry-run", action="store_true",
help="do not run the application, just parse option files")
parser.add_option("-p", "--pickle-output", action="store", type="string",
metavar="FILE",
help="DEPRECATED: use '--output file.pkl' instead. Write "
"the parsed options as a pickle file (static option "
"file)")
parser.add_option("-v", "--verbose", action="store_true",
help="print the parsed options")
parser.add_option("--old-opts", action="store_true",
help="format printed options in old option files style")
parser.add_option("--all-opts", action="store_true",
help="print all the option (even if equal to default)")
parser.add_option(
"-n",
"--dry-run",
action="store_true",
help="do not run the application, just parse option files")
parser.add_option(
"-p",
"--pickle-output",
action="store",
type="string",
metavar="FILE",
help="DEPRECATED: use '--output file.pkl' instead. Write "
"the parsed options as a pickle file (static option "
"file)")
parser.add_option(
"-v",
"--verbose",
action="store_true",
help="print the parsed options")
parser.add_option(
"--old-opts",
action="store_true",
help="format printed options in old option files style")
parser.add_option(
"--all-opts",
action="store_true",
help="print all the option (even if equal to default)")
# GaudiPython Parallel Mode Option
# Argument must be an integer in range [ -1, sys_cpus ]
# -1 : All available cpus
# 0 : Serial Mode (traditional gaudirun)
# n>0 : parallel with n cpus (n <= sys_cpus)
parser.add_option("--ncpus", action="store", type="int", default=0,
help="start the application in parallel mode using NCPUS processes. "
"0 => serial mode (default), -1 => use all CPUs")
parser.add_option(
"--ncpus",
action="store",
type="int",
default=0,
help="start the application in parallel mode using NCPUS processes. "
"0 => serial mode (default), -1 => use all CPUs")
def option_cb(option, opt, value, parser):
"""Add the option line to a list together with its position in the
argument list.
"""
parser.values.options.append((len(parser.largs), value))
parser.add_option("--option", action="callback", callback=option_cb,
type="string", nargs=1,
help="add a single line (Python) option to the configuration. "
"All options lines are executed, one after the other, in "
"the same context.")
parser.add_option("--no-conf-user-apply", action="store_true",
help="disable the automatic application of configurable "
"users (for backward compatibility)")
parser.add_option("--old-conf-user-apply", action="store_true",
help="use the old logic when applying ConfigurableUsers "
"(with bug #103803) [default]")
parser.add_option("--new-conf-user-apply", action="store_false",
dest="old_conf_user_apply",
help="use the new (correct) logic when applying "
"ConfigurableUsers (fixed bug #103803), can be "
"turned on also with the environment variable "
"GAUDI_FIXED_APPLY_CONF")
parser.add_option("-o", "--output", action="store", type="string",
help="dump the configuration to a file. The format of "
"the options is determined by the extension of the "
"file name: .pkl = pickle, .py = python, .opts = "
"old style options. The python format cannot be "
"used to run the application and it contains the "
"same dictionary printed with -v")
parser.add_option("--post-option", action="append", type="string",
dest="post_options",
help="Python options to be executed after the ConfigurableUser "
"are applied. "
"All options lines are executed, one after the other, in "
"the same context.")
parser.add_option("--debug", action="store_true",
help="enable some debug print-out")
parser.add_option("--gdb", action="store_true",
help="attach gdb")
parser.add_option("--printsequence", action="store_true",
help="print the sequence")
parser.add_option(
"--option",
action="callback",
callback=option_cb,
type="string",
nargs=1,
help="add a single line (Python) option to the configuration. "
"All options lines are executed, one after the other, in "
"the same context.")
parser.add_option(
"--no-conf-user-apply",
action="store_true",
help="disable the automatic application of configurable "
"users (for backward compatibility)")
parser.add_option(
"--old-conf-user-apply",
action="store_true",
help="use the old logic when applying ConfigurableUsers "
"(with bug #103803) [default]")
parser.add_option(
"--new-conf-user-apply",
action="store_false",
dest="old_conf_user_apply",
help="use the new (correct) logic when applying "
"ConfigurableUsers (fixed bug #103803), can be "
"turned on also with the environment variable "
"GAUDI_FIXED_APPLY_CONF")
parser.add_option(
"-o",
"--output",
action="store",
type="string",
help="dump the configuration to a file. The format of "
"the options is determined by the extension of the "
"file name: .pkl = pickle, .py = python, .opts = "
"old style options. The python format cannot be "
"used to run the application and it contains the "
"same dictionary printed with -v")
parser.add_option(
"--post-option",
action="append",
type="string",
dest="post_options",
help="Python options to be executed after the ConfigurableUser "
"are applied. "
"All options lines are executed, one after the other, in "
"the same context.")
parser.add_option(
"--debug", action="store_true", help="enable some debug print-out")
parser.add_option("--gdb", action="store_true", help="attach gdb")
parser.add_option(
"--printsequence", action="store_true", help="print the sequence")
if not sys.platform.startswith("win"):
# These options can be used only on unix platforms
parser.add_option("-T", "--tcmalloc", action="store_true",
help="Use the Google malloc replacement. The environment "
"variable TCMALLOCLIB can be used to specify a different "
"name for the library (the default is libtcmalloc.so)")
parser.add_option("--preload", action="append",
help="Allow pre-loading of special libraries (e.g. Google "
"profiling libraries).")
parser.add_option(
"-T",
"--tcmalloc",
action="store_true",
help="Use the Google malloc replacement. The environment "
"variable TCMALLOCLIB can be used to specify a different "
"name for the library (the default is libtcmalloc.so)")
parser.add_option(
"--preload",
action="append",
help="Allow pre-loading of special libraries (e.g. Google "
"profiling libraries).")
# Option to use a profiler
parser.add_option("--profilerName", type="string",