Skip to content
Snippets Groups Projects

Backport of Git CondDB to hlt2016-patches

Merged Marco Clemencic requested to merge marcocle/backport-git-conddb into hlt2016-patches
166 files
+ 15385
720
Compare changes
  • Side-by-side
  • Inline
Files
166
@@ -3,10 +3,56 @@ High level configuration tools for the detector description.
"""
__author__ = "Marco Clemencic <Marco.Clemencic@cern.ch>"
import re
from Gaudi.Configuration import *
from Configurables import ( CondDBEntityResolver,
XmlCnvSvc,
XmlParserSvc )
GIT_CONDDBS = {}
# GitEntityResolver may not be available
import Configurables
if ('EntityResolverDispatcher' in Configurables.__all__ and
'GitEntityResolver' in Configurables.__all__):
from Configurables import EntityResolverDispatcher, GitEntityResolver
# look for git DBs
def _gitconddbpath():
from os.path import join, isdir
from itertools import ifilter, imap, chain
def pathenv(name):
from os import environ as env, pathsep
return env.get(name, '').split(pathsep)
def deduplicate(iterable):
used = set()
for i in iterable:
if i not in used:
used.add(i)
yield i
user_path = pathenv('GITCONDDBPATH')
main_path = imap(lambda p: join(p, 'git-conddb'),
pathenv('CMAKE_PREFIX_PATH') +
pathenv('CMTPROJECTPATH'))
return ifilter(isdir, deduplicate(chain(user_path, main_path)))
for p in _gitconddbpath():
for dbname in [f.split('.')[0] for f in os.listdir(p)
if re.match(r'^[A-Z0-9_]+(\.git)?$', f)]:
if dbname not in GIT_CONDDBS:
if os.path.isdir(os.path.join(p, dbname, '.git')):
GIT_CONDDBS[dbname] = os.path.join(p, dbname)
elif os.path.isdir(os.path.join(p, dbname + '.git')):
GIT_CONDDBS[dbname] = os.path.join(p, dbname + '.git')
# allow forcing COOL CondDB if Git CondDBs are found
if 'NO_GIT_CONDDB' in os.environ:
GIT_CONDDBS = {}
from DetCond.Configuration import CondDB
from datetime import datetime, timedelta
import xmlrpclib
@@ -17,12 +63,14 @@ class DDDBConf(ConfigurableUser):
"""
ConfigurableUser for the configuration of the detector description.
"""
__slots__ = { "DbRoot" : "conddb:/lhcb.xml",
__slots__ = { "DbRoot" : "git:/lhcb.xml" if "DDDB" in GIT_CONDDBS else "conddb:/lhcb.xml",
"DataType" : "2012",
"Simulation": False,
"AutoTags" : False,
"InitialTime" : "Safe",
"OnlineMode" : False
"OnlineMode" : False,
"IgnoreHeartBeat": False,
"EnableRunStampCheck": False
}
_propertyDocDct = {
'DbRoot' : """ Root file of the detector description """,
@@ -30,7 +78,9 @@ class DDDBConf(ConfigurableUser):
'Simulation' : """ Boolean flag to select the simulation or real-data configuration """,
'AutoTags' : """ Perform automatic resolution of CondDB tags """,
'InitialTime' : """ How to set the initial time. None/'Safe' uses a list of dummy times for each year and sets that time. 'Now' uses the current time. Sepcifying a number assumes that is a time in utc.""",
'OnlineMode' : """ To use to run online jobs (Monitoring, ...) """
'OnlineMode' : """ To use to run online jobs (Monitoring, ...) """,
'IgnoreHeartBeat': """ Disable check on latest update of ONLINE CondDB """,
'EnableRunStampCheck': """ Enable the check for run stamp (valid data for the run) """
}
__used_configurables__ = [ CondDB ]
@@ -53,6 +103,7 @@ class DDDBConf(ConfigurableUser):
DetDbLocation = self.getProp("DbRoot"),
DetStorageType = 7 )
using_git = self.getProp("DbRoot").startswith('git:')
##########################################################################
# XML Conversion Service configuration
##########################################################################
@@ -63,10 +114,49 @@ class DDDBConf(ConfigurableUser):
##########################################################################
xmlCnvSvc = XmlCnvSvc(AllowGenericConversion = True)
# fine tune the defaults for heart-beat and run-stamp checks
ignore_hb = self.getProp("OnlineMode") or self.getProp("Simulation")
self._properties["IgnoreHeartBeat"].setDefault(ignore_hb)
self._properties["EnableRunStampCheck"].setDefault(not ignore_hb)
if using_git:
if self.getProp("Simulation"):
resolvers = [
GitEntityResolver('GitDDDB', Ignore="Conditions/.*"),
GitEntityResolver('GitSIMCOND'),
]
else:
resolvers = [
GitEntityResolver('GitDDDB', Ignore="Conditions/.*"),
GitEntityResolver('GitLHCBCOND', Ignore="Conditions/(Online|DQ).*"),
GitEntityResolver('GitONLINE', Ignore="Conditions/DQ.*",
LimitToLastCommitTime=not self.getProp("IgnoreHeartBeat")),
GitEntityResolver('GitDQFLAGS', Ignore="Conditions/Online.*"),
]
# set PathToRepository if needed and available
for r in resolvers:
r.PathToRepository = (r.PathToRepository if r.isPropertySet('PathToRepository')
else GIT_CONDDBS.get(r.name()[11:], ''))
# keep only Git resolvers that can be actually used
resolvers = [ r for r in resolvers if r.PathToRepository ]
# Leave the outputLevel at INFO even if global level is different
for r in resolvers:
if not r.isPropertySet('OutputLevel') : r.setProp( 'OutputLevel', INFO )
# add failover to COOL CondDB
resolvers.append(EntityResolverDispatcher('FallbackResolver',
EntityResolvers=[CondDBEntityResolver()],
Mappings=[(r'^git:', 'conddb:')]))
resolver = EntityResolverDispatcher(EntityResolvers=resolvers,
Mappings=[(r'^conddb:', 'git:')])
else:
resolver = CondDBEntityResolver()
xmlParserSvc = XmlParserSvc(
# Set the maximum number of files to be put in the
# cache of the XmlCnvSvc
MaxDocNbInCache = 10,
MaxDocNbInCache = 15,
# Change the behavior of the cache. A 0 value makes
# it FIFO while a bigger value tends to make it keep
@@ -75,7 +165,7 @@ class DDDBConf(ConfigurableUser):
# Tell to the XmlParserSvc how to resolve entities
# in the CondDB
EntityResolver = CondDBEntityResolver()
EntityResolver = resolver
)
##########################################################################
@@ -112,7 +202,11 @@ class DDDBConf(ConfigurableUser):
# Get particle properties table from condDB
from Configurables import LHCb__ParticlePropertySvc
LHCb__ParticlePropertySvc( ParticlePropertiesFile = 'conddb:///param/ParticleTable.txt' )
LHCb__ParticlePropertySvc(
ParticlePropertiesFile = '{protocol}///param/ParticleTable.txt'.format(
protocol = 'git:' if using_git else 'conddb:'
)
)
def __auto_tags_conf__(self, question, criterion):
""" Automatic configuration of CondDB tags through the Ariadne system """
@@ -137,7 +231,7 @@ class DDDBConf(ConfigurableUser):
def __set_tag__(self, partitions, tag):
cdb = CondDB()
online = False
online = False
if self.isPropertySet( "OnlineMode" ):
if self.getProp( "OnlineMode" ):
online = True
@@ -150,7 +244,7 @@ class DDDBConf(ConfigurableUser):
log.warning("Using default tag %s for partition %s", tag, p)
elif cdb.Tags[p].upper() == "DEFAULT" :
cdb.Tags[p] = tag
if online:
if online:
log.info("Default tag requested for partition %s (using %s)", p, tag )
else:
log.warning("Default tag requested for partition %s (using %s)", p, tag )
@@ -265,7 +359,7 @@ class DDDBConf(ConfigurableUser):
self.__set_init_time__(datetime(2009, 12, 31, 21, 0)) # 31/12/2009 21:00
else:
self.__set_tag__(["SIMCOND"], "MC-20101026-vc15mm-md100")
def __2008_conf__(self):
"""
Default configuration for 2008 data and corresponding MonteCarlo
@@ -278,7 +372,7 @@ class DDDBConf(ConfigurableUser):
self.__set_init_time__(datetime(2008, 12, 31, 21, 0)) # 31/12/2008 21:00
else:
self.__set_tag__(["SIMCOND"], "sim-20090212")
def __Upgrade_conf__(self):
"""
Default database for Upgrade MonteCarlo production and analysis
Loading