From fefb20029408eb0862b1ede0a2468615956e0e89 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Thu, 10 Sep 2020 22:02:49 +0200 Subject: [PATCH 01/28] initial commit --- .../python/DaVinci/ConfigurationUpgrade.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py new file mode 100644 index 00000000..6ad64e75 --- /dev/null +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -0,0 +1,22 @@ +##################################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb and ATLAS collaborations # +# # +# This software is distributed under the terms of the Apache version 2 licence, # +# copied verbatim in the file "LICENSE". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +##################################################################################### +from GaudiConfig2.Configurables import ApplicationMgr, MessageSvc +from GaudiConfig2 import mergeConfigs + +def main(EvtMax=-1, OutputLevel=3, **kwargs): + """ + DaVinci application main configuration. + """ + msgSvc = MessageSvc(OutputLevel=OutputLevel) + appMgr = ApplicationMgr(EvtMax=EvtMax, MessageSvcType=msgSvc) + + return mergeConfigs([appMgr , msgSvc]) + -- GitLab From b431f654e44a540d5ee657871aa8db0c5dff2690 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Thu, 10 Sep 2020 23:16:01 +0200 Subject: [PATCH 02/28] correct license --- .../python/DaVinci/ConfigurationUpgrade.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index 6ad64e75..87d16297 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -1,13 +1,13 @@ -##################################################################################### -# (c) Copyright 2020 CERN for the benefit of the LHCb and ATLAS collaborations # -# # -# This software is distributed under the terms of the Apache version 2 licence, # -# copied verbatim in the file "LICENSE". # -# # -# In applying this licence, CERN does not waive the privileges and immunities # -# granted to it by virtue of its status as an Intergovernmental Organization # -# or submit itself to any jurisdiction. # -##################################################################################### +############################################################################### +# (c) Copyright 2000-2019 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### from GaudiConfig2.Configurables import ApplicationMgr, MessageSvc from GaudiConfig2 import mergeConfigs -- GitLab From cc105fb27b813d6e285b6b9f11328bedf4b55d8a Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Fri, 11 Sep 2020 07:40:53 +0000 Subject: [PATCH 03/28] Update ConfigurationUpgrade.py --- Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index 87d16297..e0b46657 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -1,5 +1,5 @@ ############################################################################### -# (c) Copyright 2000-2019 CERN for the benefit of the LHCb Collaboration # +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # # # # This software is distributed under the terms of the GNU General Public # # Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # -- GitLab From f3674914fc5ec872825b824751280f9e5d4ae97e Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Thu, 24 Sep 2020 10:32:14 +0200 Subject: [PATCH 04/28] first steps in new DaVinci Configurable --- .../python/DaVinci/ConfigurationUpgrade.py | 136 +++++++++++++++++- 1 file changed, 134 insertions(+), 2 deletions(-) diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index e0b46657..ec71ea86 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -8,15 +8,147 @@ # granted to it by virtue of its status as an Intergovernmental Organization # # or submit itself to any jurisdiction. # ############################################################################### +""" +High level configuration tools for DaVinci +""" + +from GaudiConfig2 import Configurables as C from GaudiConfig2.Configurables import ApplicationMgr, MessageSvc from GaudiConfig2 import mergeConfigs -def main(EvtMax=-1, OutputLevel=3, **kwargs): +def main(EvtMax, OutputLevel, **kwargs): """ DaVinci application main configuration. """ + msgSvc = MessageSvc(OutputLevel=OutputLevel) appMgr = ApplicationMgr(EvtMax=EvtMax, MessageSvcType=msgSvc) + + __slots__ = initializeSlots() + _propertyDocDct = setPropDct() + + checkOptions(__slots__) + + #return mergeConfigs([appMgr , msgSvc], setUpAlgorithms()) + return mergeConfigs([appMgr, msgSvc]) - return mergeConfigs([appMgr , msgSvc]) +def mc(EvtMax=-1, OutputLevel=1, **kwargs): + return main(EvtMax, OutputLevel, **kwargs) + +def data(EvtMax=-1, OutputLevel=1, **kwargs): + return main(EvtMax, OutputLevel, **kwargs) +def checkOptions(slots): + """ + Checks options. Changes a few if needed. + """ + + __legacy_datatypes__ = [ "MC09", "2008", "2009", "2010", "2011", "2012", "2015", "2016", "2017", "2018" ] + __known_datatypes__ = ["Upgrade"] + __known_default_detectors__ = ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] + + dataType = slots["DataType"] + if (not dataType): + raise TypeError( "You must set DataType" ) + if dataType in __legacy_datatypes__: + raise TypeError( "Invalid DataType '{}'. DaVinci versions higher than v50 " \ + "should be used for Run3 data only. " \ + "See http://lhcbdoc.web.cern.ch/lhcbdoc/davinci/releases/".format(dataType) ) + if dataType not in __known_datatypes__: + raise TypeError( "Invalid DataType '%s'" %dataType ) + + inputType = slots["InputType"].upper() + if inputType not in [ "MDF", "DST", "DIGI", "RDST", "MDST", "SDST", "XDST", "LDST" ]: + raise TypeError( "Invalid inputType '%s'"%inputType ) + + if slots["Simulation"] and not inputType in ( "MDF" , "DIGI" , "MDST" ) : + redo = slots["RedoMCLinks"] + if (inputType == "RDST") and (redo) : + log.warning("Re-doing MC links not possible for RDST") + slots["RedoMCLinks"] = False + + if slots["Simulation"] and slots['Lumi'] : + log.warning('Lumi not valid for Simulation. Setting Lumi = False') + slots['Lumi'] = False + + ## for simulation, it is very important to specify proper DB-tags: + if slots['Simulation']: + if not slots['DDDBtag']: + log.warning("``DDDBtag'' is not specified for simulated data") + if not slots['CondDBtag'] : + log.warning("``CondDBtag'' is not specified for simulated data") + + if slots["MergeGenFSR"] and not slots["Simulation"] : + raise TypeError("Cannot MergeGenFSR on real data"); + +def initializeSlots(): + """ + Initialize options with default values. + """ + slots = { + # Application Configuration : sent to LHCbApp and Gaudi + "EvtMax" : -1 + , "SkipEvents" : 0 + , "PrintFreq" : 1000 + , "DataType" : 'Upgrade' + , "Simulation" : True + , "DDDBtag" : "dddb-20171126" + , "CondDBtag" : "sim-20171127-vc-md100" + , "DQFLAGStag" : "" + # Input + , "Input" : [] + , "InputType" : "DST" + , 'EnableUnpack' : None + # Output + , "HistogramFile" : "" + , "TupleFile" : "" + , "WriteFSR" : True + , "MergeGenFSR" : False + , "RootCompressionLevel" : "LZMA:6" + # DQ + , "IgnoreDQFlags" : False + # DaVinci Options + , "MainOptions" : "" + , "UserAlgorithms" : [] + , "RedoMCLinks" : False + , "Lumi" : False + , "EventPreFilters" : [] + , "VerboseMessages" : False + , "RootInTES" : "" + , "Detectors" : ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] + } + return slots + +def setPropDct(): + """ + Define a dictionary with option details. + """ + + propDct = { + "EvtMax" : """ Number of events to analyse """ + , "SkipEvents" : """ Number of events to skip at beginning for file """ + , "PrintFreq" : """ The frequency at which to print event numbers """ + , "DataType" : """ Data type, can be ['Upgrade'] Forwarded to PhysConf, AnalysisConf, DstConf and LHCbApp """ + , "Simulation" : """ set to True to use SimCond. Forwarded to PhysConf """ + , "DDDBtag" : """ Tag for DDDB. Default as set in DDDBConf for DataType """ + , "CondDBtag" : """ Tag for CondDB. Default as set in DDDBConf for DataType """ + , "DQFLAGStag" : """ Tag for DQFLAGS. Default as set in DDDBConf for DataType """ + , "Input" : """ Input data. Can also be passed as a second option file. """ + , "InputType" : """ 'DST', 'DIGI', 'RDST', 'MDST', 'XDST' or 'LDST'. Nothing means the input type is compatible with being a DST. """ + , 'EnableUnpack' : """Explicitly enable/disable unpacking for input data (if specified) """ + , "HistogramFile" : """ Write name of output Histogram file """ + , "TupleFile" : """ Write name of output Tuple file """ + , 'WriteFSR' : """ Flags whether to write out an FSR """ + , 'MergeGenFSR' : """ Flags whether to merge the generatore level FSRs """ + , "RootCompressionLevel" : """ ROOT Compression level for ntuples """ + , 'IgnoreDQFlags' : """ If False, process only events with good DQ. Default is False """ + , "MainOptions" : """ Main option file to execute """ + , "UserAlgorithms" : """ User algorithms to run. """ + , "RedoMCLinks" : """ On some stripped DST one needs to redo the Track<->MC link table. Set to true if problems with association. """ + , "Lumi" : """ Run event count and Lumi accounting (should normally be True) """ + , "EventPreFilters" : """Set of event filtering algorithms to be run before DaVinci initializaton sequence. Only events passing these filters will be processed.""" + , "VerboseMessages" : """ Enable additional verbose printouts """ + , "RootInTES" : """ RootInTES (for uDst input type) """ + , "Detectors" : """ List of detectors """ + } + return propDct -- GitLab From 7836a867de186319fc4f99ce0a26858bfd39d2a1 Mon Sep 17 00:00:00 2001 From: Maurizio Martinelli Date: Thu, 24 Sep 2020 15:30:14 +0200 Subject: [PATCH 05/28] updated following first comments from Eduardo --- .../python/DaVinci/ConfigurationUpgrade.py | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index ec71ea86..e8e5ea2f 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -16,51 +16,46 @@ from GaudiConfig2 import Configurables as C from GaudiConfig2.Configurables import ApplicationMgr, MessageSvc from GaudiConfig2 import mergeConfigs -def main(EvtMax, OutputLevel, **kwargs): +def main(EvtMax=-1, OutputLevel=1, **kwargs): """ DaVinci application main configuration. """ - + msgSvc = MessageSvc(OutputLevel=OutputLevel) appMgr = ApplicationMgr(EvtMax=EvtMax, MessageSvcType=msgSvc) __slots__ = initializeSlots() - _propertyDocDct = setPropDct() + _propertyDocDct = setPropertiesDict() checkOptions(__slots__) #return mergeConfigs([appMgr , msgSvc], setUpAlgorithms()) return mergeConfigs([appMgr, msgSvc]) - + def mc(EvtMax=-1, OutputLevel=1, **kwargs): return main(EvtMax, OutputLevel, **kwargs) def data(EvtMax=-1, OutputLevel=1, **kwargs): return main(EvtMax, OutputLevel, **kwargs) - + def checkOptions(slots): """ Checks options. Changes a few if needed. """ - - __legacy_datatypes__ = [ "MC09", "2008", "2009", "2010", "2011", "2012", "2015", "2016", "2017", "2018" ] + __known_datatypes__ = ["Upgrade"] __known_default_detectors__ = ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] dataType = slots["DataType"] if (not dataType): raise TypeError( "You must set DataType" ) - if dataType in __legacy_datatypes__: - raise TypeError( "Invalid DataType '{}'. DaVinci versions higher than v50 " \ - "should be used for Run3 data only. " \ - "See http://lhcbdoc.web.cern.ch/lhcbdoc/davinci/releases/".format(dataType) ) if dataType not in __known_datatypes__: raise TypeError( "Invalid DataType '%s'" %dataType ) - + inputType = slots["InputType"].upper() if inputType not in [ "MDF", "DST", "DIGI", "RDST", "MDST", "SDST", "XDST", "LDST" ]: raise TypeError( "Invalid inputType '%s'"%inputType ) - + if slots["Simulation"] and not inputType in ( "MDF" , "DIGI" , "MDST" ) : redo = slots["RedoMCLinks"] if (inputType == "RDST") and (redo) : @@ -69,7 +64,7 @@ def checkOptions(slots): if slots["Simulation"] and slots['Lumi'] : log.warning('Lumi not valid for Simulation. Setting Lumi = False') - slots['Lumi'] = False + slots['Lumi'] = False ## for simulation, it is very important to specify proper DB-tags: if slots['Simulation']: @@ -80,10 +75,10 @@ def checkOptions(slots): if slots["MergeGenFSR"] and not slots["Simulation"] : raise TypeError("Cannot MergeGenFSR on real data"); - + def initializeSlots(): """ - Initialize options with default values. + Initialize options with default values. """ slots = { # Application Configuration : sent to LHCbApp and Gaudi @@ -110,7 +105,7 @@ def initializeSlots(): # DaVinci Options , "MainOptions" : "" , "UserAlgorithms" : [] - , "RedoMCLinks" : False + , "RedoMCLinks" : False , "Lumi" : False , "EventPreFilters" : [] , "VerboseMessages" : False @@ -119,7 +114,7 @@ def initializeSlots(): } return slots -def setPropDct(): +def setPropertiesDict(): """ Define a dictionary with option details. """ -- GitLab From a16d1aa42a23101433155a936113e7c5abe2aed2 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Wed, 21 Oct 2020 09:09:54 +0200 Subject: [PATCH 06/28] creating module structure --- Phys/DaVinci/options/jobOptions_Example.json | 5 + .../python/DaVinci/ConfigurationUpgrade.py | 149 ++++------------ Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py | 43 +++++ Phys/DaVinci/python/DaVinci/DaVinciDB_data.py | 14 ++ Phys/DaVinci/python/DaVinci/algorithms.py | 46 +++++ Phys/DaVinci/python/DaVinci/configOptions.py | 161 ++++++++++++++++++ 6 files changed, 298 insertions(+), 120 deletions(-) create mode 100644 Phys/DaVinci/options/jobOptions_Example.json create mode 100644 Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py create mode 100644 Phys/DaVinci/python/DaVinci/DaVinciDB_data.py create mode 100644 Phys/DaVinci/python/DaVinci/algorithms.py create mode 100644 Phys/DaVinci/python/DaVinci/configOptions.py diff --git a/Phys/DaVinci/options/jobOptions_Example.json b/Phys/DaVinci/options/jobOptions_Example.json new file mode 100644 index 00000000..f29fe58a --- /dev/null +++ b/Phys/DaVinci/options/jobOptions_Example.json @@ -0,0 +1,5 @@ +{ + "EvtMax": 100, + "SkipEvents": 2, + "Simulation": "False" +} diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index e8e5ea2f..032adfec 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -16,134 +16,43 @@ from GaudiConfig2 import Configurables as C from GaudiConfig2.Configurables import ApplicationMgr, MessageSvc from GaudiConfig2 import mergeConfigs -def main(EvtMax=-1, OutputLevel=1, **kwargs): - """ - DaVinci application main configuration. - """ +from . import configOptions, algorithms +from .configOptions import initializeSlots, setPropertiesDict, checkOptions, setFileOptions, setJobOptions +from .algorithms import dvInit, defineInput, setSlotValue, getSlotValue - msgSvc = MessageSvc(OutputLevel=OutputLevel) - appMgr = ApplicationMgr(EvtMax=EvtMax, MessageSvcType=msgSvc) +from AnalysisPython.Logger import getLogger +log = getLogger ( 'DaVinci' ) - __slots__ = initializeSlots() - _propertyDocDct = setPropertiesDict() +def mc(fileName='test', jobOptFile='jobOptions.json'): + return main(fileName, jobOptFile, True) - checkOptions(__slots__) - - #return mergeConfigs([appMgr , msgSvc], setUpAlgorithms()) - return mergeConfigs([appMgr, msgSvc]) +def data(fileName='Upgrade_Bd2KstarMuMu', jobOptFile='jobOptions.json'): + return main(fileName, jobOptFile, False) -def mc(EvtMax=-1, OutputLevel=1, **kwargs): - return main(EvtMax, OutputLevel, **kwargs) - -def data(EvtMax=-1, OutputLevel=1, **kwargs): - return main(EvtMax, OutputLevel, **kwargs) - -def checkOptions(slots): +def main(fileName, jobOptFile, isMC): """ - Checks options. Changes a few if needed. - """ - - __known_datatypes__ = ["Upgrade"] - __known_default_detectors__ = ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] - - dataType = slots["DataType"] - if (not dataType): - raise TypeError( "You must set DataType" ) - if dataType not in __known_datatypes__: - raise TypeError( "Invalid DataType '%s'" %dataType ) - - inputType = slots["InputType"].upper() - if inputType not in [ "MDF", "DST", "DIGI", "RDST", "MDST", "SDST", "XDST", "LDST" ]: - raise TypeError( "Invalid inputType '%s'"%inputType ) + DaVinci application main configuration. + """ + + slots = initializeSlots() + propertyDocDct = setPropertiesDict() + + setFileOptions(slots, fileName, isMC) + setJobOptions(slots, jobOptFile, isMC) - if slots["Simulation"] and not inputType in ( "MDF" , "DIGI" , "MDST" ) : - redo = slots["RedoMCLinks"] - if (inputType == "RDST") and (redo) : - log.warning("Re-doing MC links not possible for RDST") - slots["RedoMCLinks"] = False + log.info("Applying DaVinci configuration") - if slots["Simulation"] and slots['Lumi'] : - log.warning('Lumi not valid for Simulation. Setting Lumi = False') - slots['Lumi'] = False + checkOptions(slots) - ## for simulation, it is very important to specify proper DB-tags: - if slots['Simulation']: - if not slots['DDDBtag']: - log.warning("``DDDBtag'' is not specified for simulated data") - if not slots['CondDBtag'] : - log.warning("``CondDBtag'' is not specified for simulated data") + from Configurables import EventSelector + EventSelector().PrintFreq = 1000 - if slots["MergeGenFSR"] and not slots["Simulation"] : - raise TypeError("Cannot MergeGenFSR on real data"); + #importOptions("$STDOPTS/PreloadUnits.opts") # to get units in .opts files + defineInput(slots) -def initializeSlots(): - """ - Initialize options with default values. - """ - slots = { - # Application Configuration : sent to LHCbApp and Gaudi - "EvtMax" : -1 - , "SkipEvents" : 0 - , "PrintFreq" : 1000 - , "DataType" : 'Upgrade' - , "Simulation" : True - , "DDDBtag" : "dddb-20171126" - , "CondDBtag" : "sim-20171127-vc-md100" - , "DQFLAGStag" : "" - # Input - , "Input" : [] - , "InputType" : "DST" - , 'EnableUnpack' : None - # Output - , "HistogramFile" : "" - , "TupleFile" : "" - , "WriteFSR" : True - , "MergeGenFSR" : False - , "RootCompressionLevel" : "LZMA:6" - # DQ - , "IgnoreDQFlags" : False - # DaVinci Options - , "MainOptions" : "" - , "UserAlgorithms" : [] - , "RedoMCLinks" : False - , "Lumi" : False - , "EventPreFilters" : [] - , "VerboseMessages" : False - , "RootInTES" : "" - , "Detectors" : ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] - } - return slots + algSequence = [] + + msgSvc = MessageSvc(OutputLevel=getSlotValue(slots,"OutputLevel")) + appMgr = ApplicationMgr(EvtMax=getSlotValue(slots,"EvtMax"), MessageSvcType=msgSvc, TopAlg=algSequence) -def setPropertiesDict(): - """ - Define a dictionary with option details. - """ - - propDct = { - "EvtMax" : """ Number of events to analyse """ - , "SkipEvents" : """ Number of events to skip at beginning for file """ - , "PrintFreq" : """ The frequency at which to print event numbers """ - , "DataType" : """ Data type, can be ['Upgrade'] Forwarded to PhysConf, AnalysisConf, DstConf and LHCbApp """ - , "Simulation" : """ set to True to use SimCond. Forwarded to PhysConf """ - , "DDDBtag" : """ Tag for DDDB. Default as set in DDDBConf for DataType """ - , "CondDBtag" : """ Tag for CondDB. Default as set in DDDBConf for DataType """ - , "DQFLAGStag" : """ Tag for DQFLAGS. Default as set in DDDBConf for DataType """ - , "Input" : """ Input data. Can also be passed as a second option file. """ - , "InputType" : """ 'DST', 'DIGI', 'RDST', 'MDST', 'XDST' or 'LDST'. Nothing means the input type is compatible with being a DST. """ - , 'EnableUnpack' : """Explicitly enable/disable unpacking for input data (if specified) """ - , "HistogramFile" : """ Write name of output Histogram file """ - , "TupleFile" : """ Write name of output Tuple file """ - , 'WriteFSR' : """ Flags whether to write out an FSR """ - , 'MergeGenFSR' : """ Flags whether to merge the generatore level FSRs """ - , "RootCompressionLevel" : """ ROOT Compression level for ntuples """ - , 'IgnoreDQFlags' : """ If False, process only events with good DQ. Default is False """ - , "MainOptions" : """ Main option file to execute """ - , "UserAlgorithms" : """ User algorithms to run. """ - , "RedoMCLinks" : """ On some stripped DST one needs to redo the Track<->MC link table. Set to true if problems with association. """ - , "Lumi" : """ Run event count and Lumi accounting (should normally be True) """ - , "EventPreFilters" : """Set of event filtering algorithms to be run before DaVinci initializaton sequence. Only events passing these filters will be processed.""" - , "VerboseMessages" : """ Enable additional verbose printouts """ - , "RootInTES" : """ RootInTES (for uDst input type) """ - , "Detectors" : """ List of detectors """ - } - return propDct + return mergeConfigs([appMgr, msgSvc]) diff --git a/Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py b/Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py new file mode 100644 index 00000000..61064926 --- /dev/null +++ b/Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py @@ -0,0 +1,43 @@ +filesDV = {'genFSR_upgrade_ldst': {'filenames': ['root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst', + 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade1.ldst', + 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade2.ldst', + 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade3.ldst'], + 'qualifiers': {'DataType': 'Upgrade', + 'InputType': 'LDST', + 'Simulation': True, + 'CondDB': 'sim-20171127-vc-md100', + 'DDDB': 'dddb-20171126', + }, + 'extra': {'Author': 'dfazzini', + 'Date': '2020-06-17 18:34:43.57756' + }, + 'comment': 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci', + }, + 'Upgrade_Bd2KstarMuMu': {'filenames': ['root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000002_1.ldst', + 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000004_1.ldst', + 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000043_1.ldst', + 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000068_1.ldst'], + 'qualifiers': {'DataType': 'Upgrade', + 'InputType': 'LDST', + 'Simulation': True, + 'CondDB': 'sim-20171127-vc-md100', + 'DDDB': 'dddb-20171126' + }, + 'extra': {'Author': 'Patrick Koppenburg', + 'Date': '2020-05-28 11:12:55' + }, + 'comment':'K*mumu by Moore Hlt1-filtered with TCK-0x52000000' + }, + 'test': {'filenames': ['/afs/cern.ch/work/d/dfazzini/public/TestFSR/genfsr_upgrade0.ldst'], + 'qualifiers': {'DataType': 'Upgrade', + 'InputType': 'LDST', + 'Simulation': True, + 'CondDB': 'sim-20171127-vc-md100', + 'DDDB': 'dddb-20171126', + }, + 'extra': {'Author': 'dfazzini', + 'Date': '2020-06-17 18:34:43.57756' + }, + 'comment': 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci', + } + } diff --git a/Phys/DaVinci/python/DaVinci/DaVinciDB_data.py b/Phys/DaVinci/python/DaVinci/DaVinciDB_data.py new file mode 100644 index 00000000..4e5d5dff --- /dev/null +++ b/Phys/DaVinci/python/DaVinci/DaVinciDB_data.py @@ -0,0 +1,14 @@ +filesDV = {'test': {'filenames': ['/afs/cern.ch/work/d/dfazzini/public/TestFSR/genfsr_upgrade0.ldst'], + 'qualifiers': {'DataType': 'Upgrade', + 'InputType': 'LDST', + 'Simulation': True, + 'CondDB': 'sim-20171127-vc-md100', + 'DDDB': 'dddb-20171126', + }, + 'extra': {'Author': 'dfazzini', + 'Date': '2020-06-17 18:34:43.57756' + }, + 'comment': 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci', + } + } + diff --git a/Phys/DaVinci/python/DaVinci/algorithms.py b/Phys/DaVinci/python/DaVinci/algorithms.py new file mode 100644 index 00000000..908c7e39 --- /dev/null +++ b/Phys/DaVinci/python/DaVinci/algorithms.py @@ -0,0 +1,46 @@ +################################################################################ +# Manage slots properties +# + +def getSlotValue(slots, name): + return slots[name] + +def setSlotValue(slots, name, value): + slots[name] = value + +################################################################################ +# Event Initialisation sequence +# + +def dvInit(slots): + """ + Initialisation sequence + """ + from Configurables import DaVinciInit, MemoryTool + + di = DaVinciInit('DaVinciInitAlg', Increment=getSlotValue(slots, "PrintFreq")) + di.addTool(MemoryTool) + di.MemoryTool.HistoSize = 5000 + return di + + +################################################################################ +# Defines input +# +def defineInput(slots): + """ + Define Input + """ + input = getSlotValue(slots, "Input") + + if ( len(input) > 0 ): + from GaudiConf import IOHelper + + persistency=None + inputType = getSlotValue(slots, "InputType").upper() + if inputType == "MDF" : persistency = "MDF" + #support connection strings and lists of files + input=IOHelper(persistency,persistency).convertConnectionStrings(input,"I") + #clear selector to maintain the same behaviour + IOHelper(persistency,persistency).inputFiles(input, clear=True) + diff --git a/Phys/DaVinci/python/DaVinci/configOptions.py b/Phys/DaVinci/python/DaVinci/configOptions.py new file mode 100644 index 00000000..e0886219 --- /dev/null +++ b/Phys/DaVinci/python/DaVinci/configOptions.py @@ -0,0 +1,161 @@ +from . import algorithms +from .algorithms import getSlotValue, setSlotValue + +def initializeSlots(): + """ + Initialize options with default values. + """ + slots = { + # Application Configuration : sent to LHCbApp and Gaudi + "EvtMax" : -1 + , "SkipEvents" : 0 + , "PrintFreq" : 1000 + , "DataType" : 'Upgrade' + , "Simulation" : True + , "DDDBtag" : "dddb-20171126" + , "CondDBtag" : "sim-20171127-vc-md100" + , "DQFLAGStag" : "" + , "OutputLevel" : 0 + # Input + , "Input" : [] + , "InputType" : "DST" + , 'EnableUnpack' : None + # Output + , "HistogramFile" : "" + , "TupleFile" : "" + , "WriteFSR" : True + , "MergeGenFSR" : False + , "RootCompressionLevel" : "LZMA:6" + # DQ + , "IgnoreDQFlags" : False + # DaVinci Options + , "MainOptions" : "" + , "UserAlgorithms" : [] + , "RedoMCLinks" : False + , "Lumi" : False + , "EventPreFilters" : [] + , "VerboseMessages" : False + , "RootInTES" : "" + , "Detectors" : ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] + } + return slots + +def setFileOptions(slots, name, isMC): + + if isMC: + from . import DaVinciDB_MC + from .DaVinciDB_MC import filesDV + else: + from . import DaVinciDB_data + from .DaVinciDB_data import filesDV + + for key, config in filesDV.items(): + if name == key: + for item, value in filesDV[key]['qualifiers'].items(): + setSlotValue(slots, item, value) + input = filesDV[key]['filenames'] + setSlotValue(slots, "Input", input) + +def setJobOptions(slots, filename, isMC): + import json + + fileOptions = listFileOptions(isMC) + + with open(filename) as config_file: + config = json.load(config_file) + + for key, value in config.items(): + if key in fileOptions: + log.warning('Option %s should not be set, it will be taken from DaVinciDB instead!') + else: + setSlotValue(slots, key, value) + + +def listFileOptions(isMC): + + if isMC: + from . import DaVinciDB_MC + from .DaVinciDB_MC import filesDV + else: + from . import DaVinciDB_data + from .DaVinciDB_data import filesDV + + optionList = [] + + for key in filesDV: + optionList.append(key) + + return optionList + + +def checkOptions(slots): + """ + Checks options. Changes a few if needed. + """ + + known_datatypes = ["Upgrade"] + known_default_detectors = ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] + + dataType = getSlotValue(slots,"DataType") + if (not dataType): + raise TypeError( "You must set DataType" ) + if dataType not in known_datatypes: + raise TypeError( "Invalid DataType '%s'" %dataType ) + + inputType = getSlotValue(slots, "InputType").upper() + if inputType not in [ "MDF", "DST", "DIGI", "RDST", "MDST", "SDST", "XDST", "LDST" ]: + raise TypeError( "Invalid inputType '%s'"%inputType ) + + if getSlotValue(slots, "Simulation") and not inputType in ( "MDF" , "DIGI" , "MDST" ): + redo = getSlotValue(slots, "RedoMCLinks") + if (inputType == "RDST") and (redo): + log.warning("Re-doing MC links not possible for RDST") + setSlotValue(slots, "RedoMCLinks", False) + + if getSlotValue(slots, "Simulation") and getSlotValue(slots,'Lumi'): + log.warning('Lumi not valid for Simulation. Setting Lumi = False') + setSlotValue(slots, 'Lumi', False) + + ## for simulation, it is very important to specify proper DB-tags: + if getSlotValue(slots, 'Simulation'): + if not getSlotValue(slots,'DDDBtag'): + log.warning("``DDDBtag'' is not specified for simulated data") + if not getSlotValue(slots, 'CondDBtag'): + log.warning("``CondDBtag'' is not specified for simulated data") + + if getSlotValue(slots, "MergeGenFSR") and not getSlotValue(slots, "Simulation"): + raise TypeError("Cannot MergeGenFSR on real data"); + +def setPropertiesDict(): + """ + Define a dictionary with option details. + """ + + propDct = { + "EvtMax" : """ Number of events to analyse """ + , "SkipEvents" : """ Number of events to skip at beginning for file """ + , "PrintFreq" : """ The frequency at which to print event numbers """ + , "DataType" : """ Data type, can be ['Upgrade'] Forwarded to PhysConf, AnalysisConf, DstConf and LHCbApp """ + , "Simulation" : """ set to True to use SimCond. Forwarded to PhysConf """ + , "DDDBtag" : """ Tag for DDDB. Default as set in DDDBConf for DataType """ + , "CondDBtag" : """ Tag for CondDB. Default as set in DDDBConf for DataType """ + , "DQFLAGStag" : """ Tag for DQFLAGS. Default as set in DDDBConf for DataType """ + , "Input" : """ Input data. Can also be passed as a second option file. """ + , "InputType" : """ 'DST', 'DIGI', 'RDST', 'MDST', 'XDST' or 'LDST'. Nothing means the input type is compatible with being a DST. """ + , 'EnableUnpack' : """Explicitly enable/disable unpacking for input data (if specified) """ + , "HistogramFile" : """ Write name of output Histogram file """ + , "TupleFile" : """ Write name of output Tuple file """ + , 'WriteFSR' : """ Flags whether to write out an FSR """ + , 'MergeGenFSR' : """ Flags whether to merge the generatore level FSRs """ + , "RootCompressionLevel" : """ ROOT Compression level for ntuples """ + , 'IgnoreDQFlags' : """ If False, process only events with good DQ. Default is False """ + , "MainOptions" : """ Main option file to execute """ + , "UserAlgorithms" : """ User algorithms to run. """ + , "RedoMCLinks" : """ On some stripped DST one needs to redo the Track<->MC link table. Set to true if problems with association. """ + , "Lumi" : """ Run event count and Lumi accounting (should normally be True) """ + , "EventPreFilters" : """Set of event filtering algorithms to be run before DaVinci initializaton sequence. Only events passing these filters will be processed.""" + , "VerboseMessages" : """ Enable additional verbose printouts """ + , "RootInTES" : """ RootInTES (for uDst input type) """ + , "Detectors" : """ List of detectors """ + } + return propDct -- GitLab From 0b7623c47238588b9029154821444e9e45f39f56 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Wed, 21 Oct 2020 09:42:10 +0200 Subject: [PATCH 07/28] adding copyright --- Phys/DaVinci/options/jobOptions_Example.json | 5 ---- .../python/DaVinci/ConfigurationUpgrade.py | 4 +--- Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py | 10 ++++++++ Phys/DaVinci/python/DaVinci/DaVinciDB_data.py | 10 ++++++++ Phys/DaVinci/python/DaVinci/algorithms.py | 16 +++++++++++-- Phys/DaVinci/python/DaVinci/configOptions.py | 24 +++++++++++++++---- 6 files changed, 55 insertions(+), 14 deletions(-) delete mode 100644 Phys/DaVinci/options/jobOptions_Example.json diff --git a/Phys/DaVinci/options/jobOptions_Example.json b/Phys/DaVinci/options/jobOptions_Example.json deleted file mode 100644 index f29fe58a..00000000 --- a/Phys/DaVinci/options/jobOptions_Example.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "EvtMax": 100, - "SkipEvents": 2, - "Simulation": "False" -} diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index 032adfec..f830ab5d 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -11,7 +11,6 @@ """ High level configuration tools for DaVinci """ - from GaudiConfig2 import Configurables as C from GaudiConfig2.Configurables import ApplicationMgr, MessageSvc from GaudiConfig2 import mergeConfigs @@ -32,8 +31,7 @@ def data(fileName='Upgrade_Bd2KstarMuMu', jobOptFile='jobOptions.json'): def main(fileName, jobOptFile, isMC): """ DaVinci application main configuration. - """ - + """ slots = initializeSlots() propertyDocDct = setPropertiesDict() diff --git a/Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py b/Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py index 61064926..d774b7e3 100644 --- a/Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py +++ b/Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py @@ -1,3 +1,13 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### filesDV = {'genFSR_upgrade_ldst': {'filenames': ['root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade1.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade2.ldst', diff --git a/Phys/DaVinci/python/DaVinci/DaVinciDB_data.py b/Phys/DaVinci/python/DaVinci/DaVinciDB_data.py index 4e5d5dff..60334918 100644 --- a/Phys/DaVinci/python/DaVinci/DaVinciDB_data.py +++ b/Phys/DaVinci/python/DaVinci/DaVinciDB_data.py @@ -1,3 +1,13 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### filesDV = {'test': {'filenames': ['/afs/cern.ch/work/d/dfazzini/public/TestFSR/genfsr_upgrade0.ldst'], 'qualifiers': {'DataType': 'Upgrade', 'InputType': 'LDST', diff --git a/Phys/DaVinci/python/DaVinci/algorithms.py b/Phys/DaVinci/python/DaVinci/algorithms.py index 908c7e39..9590b77e 100644 --- a/Phys/DaVinci/python/DaVinci/algorithms.py +++ b/Phys/DaVinci/python/DaVinci/algorithms.py @@ -1,7 +1,20 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + ################################################################################ # Manage slots properties # - +""" +Set and retrieve the value of a specific slot property +""" def getSlotValue(slots, name): return slots[name] @@ -11,7 +24,6 @@ def setSlotValue(slots, name, value): ################################################################################ # Event Initialisation sequence # - def dvInit(slots): """ Initialisation sequence diff --git a/Phys/DaVinci/python/DaVinci/configOptions.py b/Phys/DaVinci/python/DaVinci/configOptions.py index e0886219..568dc4fb 100644 --- a/Phys/DaVinci/python/DaVinci/configOptions.py +++ b/Phys/DaVinci/python/DaVinci/configOptions.py @@ -1,3 +1,14 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + from . import algorithms from .algorithms import getSlotValue, setSlotValue @@ -41,7 +52,9 @@ def initializeSlots(): return slots def setFileOptions(slots, name, isMC): - + """ + Set the properties related to the dataset + """ if isMC: from . import DaVinciDB_MC from .DaVinciDB_MC import filesDV @@ -57,6 +70,9 @@ def setFileOptions(slots, name, isMC): setSlotValue(slots, "Input", input) def setJobOptions(slots, filename, isMC): + """ + Set the properties related to the job + """ import json fileOptions = listFileOptions(isMC) @@ -72,7 +88,9 @@ def setJobOptions(slots, filename, isMC): def listFileOptions(isMC): - + """ + List of the properties that are set automatically given a dataset + """ if isMC: from . import DaVinciDB_MC from .DaVinciDB_MC import filesDV @@ -92,7 +110,6 @@ def checkOptions(slots): """ Checks options. Changes a few if needed. """ - known_datatypes = ["Upgrade"] known_default_detectors = ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] @@ -130,7 +147,6 @@ def setPropertiesDict(): """ Define a dictionary with option details. """ - propDct = { "EvtMax" : """ Number of events to analyse """ , "SkipEvents" : """ Number of events to skip at beginning for file """ -- GitLab From 6f428cde6852090dd1e503c92d89cd690479ca95 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Wed, 21 Oct 2020 15:08:04 +0200 Subject: [PATCH 08/28] adding path to jobOption file for testing --- Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index f830ab5d..cf35fe18 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -22,10 +22,10 @@ from .algorithms import dvInit, defineInput, setSlotValue, getSlotValue from AnalysisPython.Logger import getLogger log = getLogger ( 'DaVinci' ) -def mc(fileName='test', jobOptFile='jobOptions.json'): +def mc(fileName='test', jobOptFile='/eos/lhcb/user/d/dfazzini/public/DaVinciUpgrade/jobOptions.json'): return main(fileName, jobOptFile, True) -def data(fileName='Upgrade_Bd2KstarMuMu', jobOptFile='jobOptions.json'): +def data(fileName='Upgrade_Bd2KstarMuMu', jobOptFile='/eos/lhcb/user/d/dfazzini/public/DaVinciUpgrade/jobOptions.json'): return main(fileName, jobOptFile, False) def main(fileName, jobOptFile, isMC): -- GitLab From c228a58943244a875f8e940a92caf1a29f2b65c3 Mon Sep 17 00:00:00 2001 From: Maurizio Martinelli Date: Wed, 21 Oct 2020 16:06:15 +0200 Subject: [PATCH 09/28] small modifications to configurations import --- .../python/DaVinci/ConfigurationUpgrade.py | 23 +++++++++---------- Phys/DaVinci/python/DaVinci/configurations.py | 15 ++++++++++++ 2 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 Phys/DaVinci/python/DaVinci/configurations.py diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index cf35fe18..a7ee60c7 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -12,20 +12,19 @@ High level configuration tools for DaVinci """ from GaudiConfig2 import Configurables as C -from GaudiConfig2.Configurables import ApplicationMgr, MessageSvc from GaudiConfig2 import mergeConfigs -from . import configOptions, algorithms -from .configOptions import initializeSlots, setPropertiesDict, checkOptions, setFileOptions, setJobOptions -from .algorithms import dvInit, defineInput, setSlotValue, getSlotValue +from DaVinci.configOptions import initializeSlots, setPropertiesDict, checkOptions, setFileOptions, setJobOptions +from DaVinci.algorithms import dvInit, defineInput, setSlotValue, getSlotValue +from DaVinci.configurations import configureTiming from AnalysisPython.Logger import getLogger log = getLogger ( 'DaVinci' ) -def mc(fileName='test', jobOptFile='/eos/lhcb/user/d/dfazzini/public/DaVinciUpgrade/jobOptions.json'): +def mc(fileName='test', jobOptFile='jobOptions.json'): return main(fileName, jobOptFile, True) -def data(fileName='Upgrade_Bd2KstarMuMu', jobOptFile='/eos/lhcb/user/d/dfazzini/public/DaVinciUpgrade/jobOptions.json'): +def data(fileName='Upgrade_Bd2KstarMuMu', jobOptFile='jobOptions.json'): return main(fileName, jobOptFile, False) def main(fileName, jobOptFile, isMC): @@ -34,7 +33,7 @@ def main(fileName, jobOptFile, isMC): """ slots = initializeSlots() propertyDocDct = setPropertiesDict() - + setFileOptions(slots, fileName, isMC) setJobOptions(slots, jobOptFile, isMC) @@ -43,14 +42,14 @@ def main(fileName, jobOptFile, isMC): checkOptions(slots) from Configurables import EventSelector - EventSelector().PrintFreq = 1000 + EventSelector().PrintFreq = 1000 #importOptions("$STDOPTS/PreloadUnits.opts") # to get units in .opts files defineInput(slots) algSequence = [] - - msgSvc = MessageSvc(OutputLevel=getSlotValue(slots,"OutputLevel")) - appMgr = ApplicationMgr(EvtMax=getSlotValue(slots,"EvtMax"), MessageSvcType=msgSvc, TopAlg=algSequence) - return mergeConfigs([appMgr, msgSvc]) + msgSvc = C.MessageSvc(OutputLevel=getSlotValue(slots,"OutputLevel")) + appMgr = C.ApplicationMgr(EvtMax=getSlotValue(slots,"EvtMax"), MessageSvcType=msgSvc, TopAlg=algSequence) + + return mergeConfigs([appMgr, msgSvc],configureTiming()) diff --git a/Phys/DaVinci/python/DaVinci/configurations.py b/Phys/DaVinci/python/DaVinci/configurations.py new file mode 100644 index 00000000..7a61468e --- /dev/null +++ b/Phys/DaVinci/python/DaVinci/configurations.py @@ -0,0 +1,15 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +from GaudiConfig2 import Configurables as C + +def configureTiming(): + return [C.AuditorSvc('AuditorSvc', Auditors=['ChronoAuditor'])] -- GitLab From f1c97d805d21ac0913a925e4f283b3b3bad09bbd Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Wed, 21 Oct 2020 16:27:58 +0200 Subject: [PATCH 10/28] adding external services --- .../python/DaVinci/ConfigurationUpgrade.py | 17 +++++++++++++---- Phys/DaVinci/python/DaVinci/configurations.py | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index a7ee60c7..a349587a 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -16,7 +16,7 @@ from GaudiConfig2 import mergeConfigs from DaVinci.configOptions import initializeSlots, setPropertiesDict, checkOptions, setFileOptions, setJobOptions from DaVinci.algorithms import dvInit, defineInput, setSlotValue, getSlotValue -from DaVinci.configurations import configureTiming +from DaVinci.configurations import configureTiming, configExtSvc from AnalysisPython.Logger import getLogger log = getLogger ( 'DaVinci' ) @@ -44,12 +44,21 @@ def main(fileName, jobOptFile, isMC): from Configurables import EventSelector EventSelector().PrintFreq = 1000 - #importOptions("$STDOPTS/PreloadUnits.opts") # to get units in .opts files + from Configurables import DataOnDemandSvc + verbosePrint = getSlotValue(slots, "VerboseMessages") + DataOnDemandSvc().Dump = verbosePrint + + #import GaudiConf.options.PreloadUnits.opts defineInput(slots) + extSvc = configExtSvc() algSequence = [] msgSvc = C.MessageSvc(OutputLevel=getSlotValue(slots,"OutputLevel")) - appMgr = C.ApplicationMgr(EvtMax=getSlotValue(slots,"EvtMax"), MessageSvcType=msgSvc, TopAlg=algSequence) + appMgr = C.ApplicationMgr(EvtMax=getSlotValue(slots,"EvtMax"), + MessageSvcType=msgSvc, + ExtSvc=extSvc, + TopAlg=algSequence, + HistogramPersistency='ROOT') - return mergeConfigs([appMgr, msgSvc],configureTiming()) + return mergeConfigs([appMgr, msgSvc], configureTiming(), dvInit(slots)) diff --git a/Phys/DaVinci/python/DaVinci/configurations.py b/Phys/DaVinci/python/DaVinci/configurations.py index 7a61468e..49384b39 100644 --- a/Phys/DaVinci/python/DaVinci/configurations.py +++ b/Phys/DaVinci/python/DaVinci/configurations.py @@ -13,3 +13,21 @@ from GaudiConfig2 import Configurables as C def configureTiming(): return [C.AuditorSvc('AuditorSvc', Auditors=['ChronoAuditor'])] + +################################################################################ +# Define external services +# +def configExtSvc(): + extSvc = ['DetDataSvc/DetectorDataSvc', + 'ToolSvc', + 'AuditorSvc', + 'NTupleSvc/NTupleSvc', + 'LoKiSvc/LoKiSvc', + 'ToolSvc/ToolSvc', + 'DataOnDemandSvc/DataOnDemandSvc', + 'Gaudi::MultiFileCatalog/FileCatalog', + 'Gaudi::IODataManager/IODataManager', + 'Gaudi::RootCnvSvc/RootCnvSvc', + 'Gaudi::RootCnvSvc/FileRecordCnvSvc', + ] + return extSvc -- GitLab From 394ed4b0d26e0d73fe96c4c2de59928d3ae442e0 Mon Sep 17 00:00:00 2001 From: Marco Clemencic Date: Fri, 23 Oct 2020 16:13:48 +0200 Subject: [PATCH 11/28] Configure EventPersistencySvc and use instances instead of strings in extSvc --- .../python/DaVinci/ConfigurationUpgrade.py | 5 ++++- Phys/DaVinci/python/DaVinci/configurations.py | 21 +++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index a349587a..461ecc11 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -61,4 +61,7 @@ def main(fileName, jobOptFile, isMC): TopAlg=algSequence, HistogramPersistency='ROOT') - return mergeConfigs([appMgr, msgSvc], configureTiming(), dvInit(slots)) + evtPers = C.EvtPersistencySvc("EventPersistencySvc") + evtPers.CnvServices.append("Gaudi::RootCnvSvc/RootCnvSvc") + + return mergeConfigs([appMgr, msgSvc, evtPers], configureTiming(), dvInit(slots), extSvc) diff --git a/Phys/DaVinci/python/DaVinci/configurations.py b/Phys/DaVinci/python/DaVinci/configurations.py index 49384b39..a33b0440 100644 --- a/Phys/DaVinci/python/DaVinci/configurations.py +++ b/Phys/DaVinci/python/DaVinci/configurations.py @@ -18,16 +18,15 @@ def configureTiming(): # Define external services # def configExtSvc(): - extSvc = ['DetDataSvc/DetectorDataSvc', - 'ToolSvc', - 'AuditorSvc', - 'NTupleSvc/NTupleSvc', - 'LoKiSvc/LoKiSvc', - 'ToolSvc/ToolSvc', - 'DataOnDemandSvc/DataOnDemandSvc', - 'Gaudi::MultiFileCatalog/FileCatalog', - 'Gaudi::IODataManager/IODataManager', - 'Gaudi::RootCnvSvc/RootCnvSvc', - 'Gaudi::RootCnvSvc/FileRecordCnvSvc', + extSvc = [C.DetDataSvc('DetectorDataSvc'), + C.ToolSvc(), + C.AuditorSvc(), + C.NTupleSvc(), + C.LoKiSvc(), + C.DataOnDemandSvc(), + C.Gaudi.MultiFileCatalog('FileCatalog'), + C.Gaudi.IODataManager('IODataManager'), + C.Gaudi.RootCnvSvc('RootCnvSvc'), + C.Gaudi.RootCnvSvc('FileRecordCnvSvc'), ] return extSvc -- GitLab From 61692a53deaad611102cdd449f9c3cec00316427 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Sun, 8 Nov 2020 20:47:47 +0100 Subject: [PATCH 12/28] fixing service instatiation and add new yaml DB --- Phys/DaVinci/options/DaVinciDB_MC.yaml | 59 +++++ Phys/DaVinci/options/jobOptions.yaml | 15 ++ .../python/DaVinci/ConfigurationUpgrade.py | 63 +++--- Phys/DaVinci/python/DaVinci/algorithms.py | 39 +++- Phys/DaVinci/python/DaVinci/configOptions.py | 203 ++++++++++-------- Phys/DaVinci/python/DaVinci/configurations.py | 24 ++- 6 files changed, 271 insertions(+), 132 deletions(-) create mode 100644 Phys/DaVinci/options/DaVinciDB_MC.yaml create mode 100644 Phys/DaVinci/options/jobOptions.yaml diff --git a/Phys/DaVinci/options/DaVinciDB_MC.yaml b/Phys/DaVinci/options/DaVinciDB_MC.yaml new file mode 100644 index 00000000..3a7c1b45 --- /dev/null +++ b/Phys/DaVinci/options/DaVinciDB_MC.yaml @@ -0,0 +1,59 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +test: + filenames: + - '/afs/cern.ch/work/d/dfazzini/public/TestFSR/genfsr_upgrade0.ldst' + qualifiers: + DataType: Upgrade + InputType: LDST + Simulation: true + CondDB: sim-20171127-vc-md100 + DDDB: dddb-20171126 + extra: + Author: 'dfazzini' + Date: '2020-06-17 18:34:43.57756' + comment: 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci' + +genFSR_upgrade_ldst: + filenames: + - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst' + - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade1.ldst' + - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade2.ldst' + - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade3.ldst' + qualifiers: + DataType: Upgrade + InputType: LDST + Simulation: true + CondDB: sim-20171127-vc-md100 + DDDB: dddb-20171126 + extra: + Author: 'dfazzini' + Date: '2020-06-17 18:34:43.57756' + comment: 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci' + +Upgrade_Bd2KstarMuMu: + filenames: + - 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000002_1.ldst' + - 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000004_1.ldst' + - 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000043_1.ldst' + - 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000068_1.ldst' + qualifiers: + DataType: Upgrade + InputType: LDST + Simulation: true + CondDB: sim-20171127-vc-md100 + DDDB: dddb-20171126 + extra: + Author: 'Patrick Koppenburg' + Date: '2020-05-28 11:12:55' + comment: 'K*mumu by Moore Hlt1-filtered with TCK-0x52000000' + \ No newline at end of file diff --git a/Phys/DaVinci/options/jobOptions.yaml b/Phys/DaVinci/options/jobOptions.yaml new file mode 100644 index 00000000..aaf8e681 --- /dev/null +++ b/Phys/DaVinci/options/jobOptions.yaml @@ -0,0 +1,15 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +EvtMax: 100 +SkipEvents: 2 +TupleFile: 'DVNtuple.root' +HistogramFile: 'DVHistos.root' diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index 461ecc11..339f50eb 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -1,4 +1,4 @@ -############################################################################### +############################################################################## # (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # # # # This software is distributed under the terms of the GNU General Public # @@ -15,16 +15,17 @@ from GaudiConfig2 import Configurables as C from GaudiConfig2 import mergeConfigs from DaVinci.configOptions import initializeSlots, setPropertiesDict, checkOptions, setFileOptions, setJobOptions -from DaVinci.algorithms import dvInit, defineInput, setSlotValue, getSlotValue -from DaVinci.configurations import configureTiming, configExtSvc +from DaVinci.algorithms import defineInput, setSlotValue, getSlotValue, setupAlgorithms +from DaVinci.configurations import configureTiming, configureMessage, configExtSvc from AnalysisPython.Logger import getLogger log = getLogger ( 'DaVinci' ) -def mc(fileName='test', jobOptFile='jobOptions.json'): +def mc(fileName='genFSR_upgrade_ldst', jobOptFile='jobOptions.yaml'): return main(fileName, jobOptFile, True) - -def data(fileName='Upgrade_Bd2KstarMuMu', jobOptFile='jobOptions.json'): + +def data(fileName='', jobOptFile=''): + raise ValueError('Data file with upgrade conditions are not yet available. Please use :mc function instead.') return main(fileName, jobOptFile, False) def main(fileName, jobOptFile, isMC): @@ -35,33 +36,41 @@ def main(fileName, jobOptFile, isMC): propertyDocDct = setPropertiesDict() setFileOptions(slots, fileName, isMC) - setJobOptions(slots, jobOptFile, isMC) + setJobOptions(slots, jobOptFile, isMC, log) log.info("Applying DaVinci configuration") - - checkOptions(slots) - - from Configurables import EventSelector - EventSelector().PrintFreq = 1000 - - from Configurables import DataOnDemandSvc - verbosePrint = getSlotValue(slots, "VerboseMessages") - DataOnDemandSvc().Dump = verbosePrint + checkOptions(slots, log) #import GaudiConf.options.PreloadUnits.opts defineInput(slots) - extSvc = configExtSvc() - algSequence = [] + config = [] + appMgr = C.ApplicationMgr() + config.append(appMgr) + config.append(configureTiming()) + #config.append(configureMessage(slots)) + configExtSvc(config, slots) + setupAlgorithms(config, slots) - msgSvc = C.MessageSvc(OutputLevel=getSlotValue(slots,"OutputLevel")) - appMgr = C.ApplicationMgr(EvtMax=getSlotValue(slots,"EvtMax"), - MessageSvcType=msgSvc, - ExtSvc=extSvc, - TopAlg=algSequence, - HistogramPersistency='ROOT') + messageSvc = 0 + for c in config: + if c.name == 'MessageSvc': + messageSvc = c + + #appMgr.MessageSvcType = messageSvc + appMgr.EvtMax = getSlotValue(slots,"EvtMax") + appMgr.HistogramPersistency = 'ROOT' + + appMgr.ExtSvc = [c for c in config + if c.__component_type__ == 'Service' + and c.name not in ['ApplicationMgr']] + appMgr.TopAlg = [c for c in config + if c.__component_type__ == 'Algorithm' + and c.name not in ['ApplicationMgr']] + + config = mergeConfigs(config) + return config + + - evtPers = C.EvtPersistencySvc("EventPersistencySvc") - evtPers.CnvServices.append("Gaudi::RootCnvSvc/RootCnvSvc") - return mergeConfigs([appMgr, msgSvc, evtPers], configureTiming(), dvInit(slots), extSvc) diff --git a/Phys/DaVinci/python/DaVinci/algorithms.py b/Phys/DaVinci/python/DaVinci/algorithms.py index 9590b77e..132f7eea 100644 --- a/Phys/DaVinci/python/DaVinci/algorithms.py +++ b/Phys/DaVinci/python/DaVinci/algorithms.py @@ -9,6 +9,8 @@ # or submit itself to any jurisdiction. # ############################################################################### +from GaudiConfig2 import Configurables as C + ################################################################################ # Manage slots properties # @@ -27,12 +29,10 @@ def setSlotValue(slots, name, value): def dvInit(slots): """ Initialisation sequence - """ - from Configurables import DaVinciInit, MemoryTool - - di = DaVinciInit('DaVinciInitAlg', Increment=getSlotValue(slots, "PrintFreq")) - di.addTool(MemoryTool) - di.MemoryTool.HistoSize = 5000 + """ + di = C.DaVinciInit('DaVinciInitAlg', Increment = getSlotValue(slots, "PrintFreq")) + #di.addTool(MemoryTool) + #di.MemoryTool.HistoSize = 5000 return di @@ -48,11 +48,30 @@ def defineInput(slots): if ( len(input) > 0 ): from GaudiConf import IOHelper - persistency=None + persistency = None inputType = getSlotValue(slots, "InputType").upper() - if inputType == "MDF" : persistency = "MDF" + if inputType == "MDF" : persistency = "MDF" #support connection strings and lists of files - input=IOHelper(persistency,persistency).convertConnectionStrings(input,"I") + input = IOHelper(persistency, persistency).convertConnectionStrings(input, "I") #clear selector to maintain the same behaviour - IOHelper(persistency,persistency).inputFiles(input, clear=True) + IOHelper(persistency, persistency).inputFiles(input, clear = True) + else: + log.warning('No input files available along with the selected key. Check the related DaVinciDB.') + + +################################################################################ +# Defines algorithms +# +def setupAlgorithms(config, slots): + """ + Define DaVinci algorithms + """ + init = dvInit(slots) + config.append(init) + + algorithms = [C.GaudiHistoAlgorithm('SimpleHistos', + HistoPrint=True, + OutputLevel=getSlotValue(slots, "OutputLevel"))] + for alg in algorithms: + config.append(alg) diff --git a/Phys/DaVinci/python/DaVinci/configOptions.py b/Phys/DaVinci/python/DaVinci/configOptions.py index 568dc4fb..486cfb6e 100644 --- a/Phys/DaVinci/python/DaVinci/configOptions.py +++ b/Phys/DaVinci/python/DaVinci/configOptions.py @@ -9,8 +9,9 @@ # or submit itself to any jurisdiction. # ############################################################################### -from . import algorithms -from .algorithms import getSlotValue, setSlotValue +import os +from DaVinci import algorithms +from DaVinci.algorithms import getSlotValue, setSlotValue def initializeSlots(): """ @@ -18,36 +19,37 @@ def initializeSlots(): """ slots = { # Application Configuration : sent to LHCbApp and Gaudi - "EvtMax" : -1 - , "SkipEvents" : 0 - , "PrintFreq" : 1000 - , "DataType" : 'Upgrade' - , "Simulation" : True - , "DDDBtag" : "dddb-20171126" - , "CondDBtag" : "sim-20171127-vc-md100" - , "DQFLAGStag" : "" - , "OutputLevel" : 0 + "EvtMax" : -1 + , "SkipEvents" : 0 + , "PrintFreq" : 1000 + , "DataType" : 'Upgrade' + , "Simulation" : True + , "DDDBtag" : "dddb-20171126" + , "CondDBtag" : "sim-20171127-vc-md100" + , "DQFLAGStag" : "" + , "OutputLevel" : 3 # Input - , "Input" : [] - , "InputType" : "DST" - , 'EnableUnpack' : None + , "Input" : [] + , "InputType" : "DST" + , 'EnableUnpack' : None + , "OverwriteDataOptions" : False # Output - , "HistogramFile" : "" - , "TupleFile" : "" - , "WriteFSR" : True - , "MergeGenFSR" : False + , "HistogramFile" : "" + , "TupleFile" : "" + , "WriteFSR" : True + , "MergeGenFSR" : False , "RootCompressionLevel" : "LZMA:6" # DQ - , "IgnoreDQFlags" : False + , "IgnoreDQFlags" : False # DaVinci Options - , "MainOptions" : "" - , "UserAlgorithms" : [] - , "RedoMCLinks" : False - , "Lumi" : False - , "EventPreFilters" : [] - , "VerboseMessages" : False - , "RootInTES" : "" - , "Detectors" : ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] + , "MainOptions" : "" + , "UserAlgorithms" : [] + , "RedoMCLinks" : False + , "Lumi" : False + , "EventPreFilters" : [] + , "VerboseMessages" : False + , "RootInTES" : "" + , "Detectors" : ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] } return slots @@ -55,58 +57,76 @@ def setFileOptions(slots, name, isMC): """ Set the properties related to the dataset """ - if isMC: - from . import DaVinciDB_MC - from .DaVinciDB_MC import filesDV - else: - from . import DaVinciDB_data - from .DaVinciDB_data import filesDV - - for key, config in filesDV.items(): - if name == key: - for item, value in filesDV[key]['qualifiers'].items(): - setSlotValue(slots, item, value) - input = filesDV[key]['filenames'] - setSlotValue(slots, "Input", input) - -def setJobOptions(slots, filename, isMC): + import yaml + dbName = 'DaVinciDB_MC.yaml' if isMC else 'DaVinciDB_data.yaml' + + dbName = '$DAVINCIROOT/options/%s' %dbName + with open(os.path.expandvars(dbName)) as dbFile: + dataDV = yaml.safe_load(dbFile) + + for key, config in dataDV.items(): + if key == name: + for item, obj in config.items(): + if item == 'qualifiers': + for prop, value in obj.items(): + setSlotValue(slots, prop, value) + elif item == 'filenames': + setSlotValue(slots, "Input", obj) + +def setJobOptions(slots, filename, isMC, log): """ Set the properties related to the job """ - import json - - fileOptions = listFileOptions(isMC) - - with open(filename) as config_file: - config = json.load(config_file) - - for key, value in config.items(): - if key in fileOptions: - log.warning('Option %s should not be set, it will be taken from DaVinciDB instead!') + if filename == '': + log.warning('No jobOption file selected, the default values are used.') + else: + dataOptions = listDataOptions(isMC) + + filename = '$DAVINCIROOT/options/%s' %filename + with open(os.path.expandvars(filename)) as config_file: + if filename.endswith('.json'): + import json + config = json.load(config_file) + elif filename.endswith('.yaml') or filename.endswith('.yml'): + import yaml + config = yaml.safe_load(config_file) else: - setSlotValue(slots, key, value) - - -def listFileOptions(isMC): + raise ValueError('JobOption file extension not known! Please use only a json or yaml file!') + + for key, value in config.items(): + if key in dataOptions: + if getSlotValue(slots, "OverwriteDataOptions"): + log.warning('Default value for option %s is taken from DaVinciDB but a new value is found in option file. ' + 'Since "OverwriteDataOptions" is active the new value is used. Make sure this is the configuration you want to use for this job.' %key) + setSlotValue(slots, key, value) + else: + log.warning('Default value for option %s is taken from DaVinciDB but a new value is found in option file. ' + 'Since "OverwriteDataOptions" is not active the default value is used. Make sure this is the configuration you want to use for this job.' %key) + else: + setSlotValue(slots, key, value) + +def listDataOptions(isMC): """ List of the properties that are set automatically given a dataset """ - if isMC: - from . import DaVinciDB_MC - from .DaVinciDB_MC import filesDV - else: - from . import DaVinciDB_data - from .DaVinciDB_data import filesDV - - optionList = [] + import yaml + dbName = 'DaVinciDB_MC.yaml' if isMC else 'DaVinciDB_data.yaml' - for key in filesDV: - optionList.append(key) + dbName = '$DAVINCIROOT/options/%s' %dbName + with open(os.path.expandvars(dbName)) as dbFile: + dataDV = yaml.safe_load(dbFile) + optionList = [] + for key, config in dataDV.items(): + for item, obj in config.items(): + if item == 'qualifiers': + for prop, value in obj.items(): + optionList.append(prop) + break return optionList -def checkOptions(slots): +def checkOptions(slots, log): """ Checks options. Changes a few if needed. """ @@ -148,30 +168,31 @@ def setPropertiesDict(): Define a dictionary with option details. """ propDct = { - "EvtMax" : """ Number of events to analyse """ - , "SkipEvents" : """ Number of events to skip at beginning for file """ - , "PrintFreq" : """ The frequency at which to print event numbers """ - , "DataType" : """ Data type, can be ['Upgrade'] Forwarded to PhysConf, AnalysisConf, DstConf and LHCbApp """ - , "Simulation" : """ set to True to use SimCond. Forwarded to PhysConf """ - , "DDDBtag" : """ Tag for DDDB. Default as set in DDDBConf for DataType """ - , "CondDBtag" : """ Tag for CondDB. Default as set in DDDBConf for DataType """ - , "DQFLAGStag" : """ Tag for DQFLAGS. Default as set in DDDBConf for DataType """ - , "Input" : """ Input data. Can also be passed as a second option file. """ - , "InputType" : """ 'DST', 'DIGI', 'RDST', 'MDST', 'XDST' or 'LDST'. Nothing means the input type is compatible with being a DST. """ - , 'EnableUnpack' : """Explicitly enable/disable unpacking for input data (if specified) """ - , "HistogramFile" : """ Write name of output Histogram file """ - , "TupleFile" : """ Write name of output Tuple file """ - , 'WriteFSR' : """ Flags whether to write out an FSR """ - , 'MergeGenFSR' : """ Flags whether to merge the generatore level FSRs """ + "EvtMax" : """ Number of events to analyse """ + , "SkipEvents" : """ Number of events to skip at beginning for file """ + , "PrintFreq" : """ The frequency at which to print event numbers """ + , "DataType" : """ Data type, can be ['Upgrade'] Forwarded to PhysConf, AnalysisConf, DstConf and LHCbApp """ + , "Simulation" : """ set to True to use SimCond. Forwarded to PhysConf """ + , "DDDBtag" : """ Tag for DDDB. Default as set in DDDBConf for DataType """ + , "CondDBtag" : """ Tag for CondDB. Default as set in DDDBConf for DataType """ + , "DQFLAGStag" : """ Tag for DQFLAGS. Default as set in DDDBConf for DataType """ + , "Input" : """ Input data. Can also be passed as a second option file. """ + , "InputType" : """ 'DST', 'DIGI', 'RDST', 'MDST', 'XDST' or 'LDST'. Nothing means the input type is compatible with being a DST. """ + , 'EnableUnpack' : """Explicitly enable/disable unpacking for input data (if specified) """ + , "OverwriteDataOptions" : """ Allow overwriting default data options defined in DaVinci Database. """ + , "HistogramFile" : """ Write name of output Histogram file """ + , "TupleFile" : """ Write name of output Tuple file """ + , 'WriteFSR' : """ Flags whether to write out an FSR """ + , 'MergeGenFSR' : """ Flags whether to merge the generatore level FSRs """ , "RootCompressionLevel" : """ ROOT Compression level for ntuples """ - , 'IgnoreDQFlags' : """ If False, process only events with good DQ. Default is False """ - , "MainOptions" : """ Main option file to execute """ - , "UserAlgorithms" : """ User algorithms to run. """ - , "RedoMCLinks" : """ On some stripped DST one needs to redo the Track<->MC link table. Set to true if problems with association. """ - , "Lumi" : """ Run event count and Lumi accounting (should normally be True) """ - , "EventPreFilters" : """Set of event filtering algorithms to be run before DaVinci initializaton sequence. Only events passing these filters will be processed.""" - , "VerboseMessages" : """ Enable additional verbose printouts """ - , "RootInTES" : """ RootInTES (for uDst input type) """ - , "Detectors" : """ List of detectors """ + , 'IgnoreDQFlags' : """ If False, process only events with good DQ. Default is False """ + , "MainOptions" : """ Main option file to execute """ + , "UserAlgorithms" : """ User algorithms to run. """ + , "RedoMCLinks" : """ On some stripped DST one needs to redo the Track<->MC link table. Set to true if problems with association. """ + , "Lumi" : """ Run event count and Lumi accounting (should normally be True) """ + , "EventPreFilters" : """Set of event filtering algorithms to be run before DaVinci initializaton sequence. Only events passing these filters will be processed.""" + , "VerboseMessages" : """ Enable additional verbose printouts """ + , "RootInTES" : """ RootInTES (for uDst input type) """ + , "Detectors" : """ List of detectors """ } return propDct diff --git a/Phys/DaVinci/python/DaVinci/configurations.py b/Phys/DaVinci/python/DaVinci/configurations.py index a33b0440..b3fc4d53 100644 --- a/Phys/DaVinci/python/DaVinci/configurations.py +++ b/Phys/DaVinci/python/DaVinci/configurations.py @@ -10,23 +10,39 @@ ############################################################################### from GaudiConfig2 import Configurables as C +from DaVinci.algorithms import getSlotValue def configureTiming(): - return [C.AuditorSvc('AuditorSvc', Auditors=['ChronoAuditor'])] + return C.AuditorSvc('AuditorSvc', Auditors=['ChronoAuditor']) + +def configureMessage(slots): + return C.MessageSvc(OutputLevel = getSlotValue(slots,"OutputLevel")) ################################################################################ # Define external services # -def configExtSvc(): +def configExtSvc(config, slots): extSvc = [C.DetDataSvc('DetectorDataSvc'), C.ToolSvc(), C.AuditorSvc(), C.NTupleSvc(), C.LoKiSvc(), C.DataOnDemandSvc(), + C.EventSelector(), + C.EvtPersistencySvc("EventPersistencySvc"), C.Gaudi.MultiFileCatalog('FileCatalog'), C.Gaudi.IODataManager('IODataManager'), - C.Gaudi.RootCnvSvc('RootCnvSvc'), C.Gaudi.RootCnvSvc('FileRecordCnvSvc'), ] - return extSvc + + # Define extra option and append services to main config list + for svc in extSvc: + if svc.name == "EventSelector": + svc.PrintFreq = getSlotValue(slots, "PrintFreq") + elif svc.name == "DataOnDemandSvc": + svc.Dump = getSlotValue(slots, "VerboseMessages") + elif svc.name == "EventPersistencySvc": + #CnvServices.append(C.Gaudi.RootCnvSvc("RootCnvSvc")) + svc.CnvServices.append("Gaudi::RootCnvSvc/RootCnvSvc") + + config.append(svc) -- GitLab From ec928f88e57a114829f391dbddc1bcdefeea7468 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Sun, 8 Nov 2020 20:50:57 +0100 Subject: [PATCH 13/28] removing old python database --- Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py | 53 ------------------- Phys/DaVinci/python/DaVinci/DaVinciDB_data.py | 24 --------- 2 files changed, 77 deletions(-) delete mode 100644 Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py delete mode 100644 Phys/DaVinci/python/DaVinci/DaVinciDB_data.py diff --git a/Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py b/Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py deleted file mode 100644 index d774b7e3..00000000 --- a/Phys/DaVinci/python/DaVinci/DaVinciDB_MC.py +++ /dev/null @@ -1,53 +0,0 @@ -############################################################################### -# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # -# # -# This software is distributed under the terms of the GNU General Public # -# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # -# # -# In applying this licence, CERN does not waive the privileges and immunities # -# granted to it by virtue of its status as an Intergovernmental Organization # -# or submit itself to any jurisdiction. # -############################################################################### -filesDV = {'genFSR_upgrade_ldst': {'filenames': ['root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst', - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade1.ldst', - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade2.ldst', - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade3.ldst'], - 'qualifiers': {'DataType': 'Upgrade', - 'InputType': 'LDST', - 'Simulation': True, - 'CondDB': 'sim-20171127-vc-md100', - 'DDDB': 'dddb-20171126', - }, - 'extra': {'Author': 'dfazzini', - 'Date': '2020-06-17 18:34:43.57756' - }, - 'comment': 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci', - }, - 'Upgrade_Bd2KstarMuMu': {'filenames': ['root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000002_1.ldst', - 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000004_1.ldst', - 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000043_1.ldst', - 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000068_1.ldst'], - 'qualifiers': {'DataType': 'Upgrade', - 'InputType': 'LDST', - 'Simulation': True, - 'CondDB': 'sim-20171127-vc-md100', - 'DDDB': 'dddb-20171126' - }, - 'extra': {'Author': 'Patrick Koppenburg', - 'Date': '2020-05-28 11:12:55' - }, - 'comment':'K*mumu by Moore Hlt1-filtered with TCK-0x52000000' - }, - 'test': {'filenames': ['/afs/cern.ch/work/d/dfazzini/public/TestFSR/genfsr_upgrade0.ldst'], - 'qualifiers': {'DataType': 'Upgrade', - 'InputType': 'LDST', - 'Simulation': True, - 'CondDB': 'sim-20171127-vc-md100', - 'DDDB': 'dddb-20171126', - }, - 'extra': {'Author': 'dfazzini', - 'Date': '2020-06-17 18:34:43.57756' - }, - 'comment': 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci', - } - } diff --git a/Phys/DaVinci/python/DaVinci/DaVinciDB_data.py b/Phys/DaVinci/python/DaVinci/DaVinciDB_data.py deleted file mode 100644 index 60334918..00000000 --- a/Phys/DaVinci/python/DaVinci/DaVinciDB_data.py +++ /dev/null @@ -1,24 +0,0 @@ -############################################################################### -# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # -# # -# This software is distributed under the terms of the GNU General Public # -# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # -# # -# In applying this licence, CERN does not waive the privileges and immunities # -# granted to it by virtue of its status as an Intergovernmental Organization # -# or submit itself to any jurisdiction. # -############################################################################### -filesDV = {'test': {'filenames': ['/afs/cern.ch/work/d/dfazzini/public/TestFSR/genfsr_upgrade0.ldst'], - 'qualifiers': {'DataType': 'Upgrade', - 'InputType': 'LDST', - 'Simulation': True, - 'CondDB': 'sim-20171127-vc-md100', - 'DDDB': 'dddb-20171126', - }, - 'extra': {'Author': 'dfazzini', - 'Date': '2020-06-17 18:34:43.57756' - }, - 'comment': 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci', - } - } - -- GitLab From c2f92676614154a00b812228db7ae5a07c641609 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Sun, 8 Nov 2020 20:53:52 +0100 Subject: [PATCH 14/28] removing old test file from database --- Phys/DaVinci/options/DaVinciDB_MC.yaml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/Phys/DaVinci/options/DaVinciDB_MC.yaml b/Phys/DaVinci/options/DaVinciDB_MC.yaml index 3a7c1b45..478e3cdc 100644 --- a/Phys/DaVinci/options/DaVinciDB_MC.yaml +++ b/Phys/DaVinci/options/DaVinciDB_MC.yaml @@ -9,20 +9,6 @@ # or submit itself to any jurisdiction. # ############################################################################### -test: - filenames: - - '/afs/cern.ch/work/d/dfazzini/public/TestFSR/genfsr_upgrade0.ldst' - qualifiers: - DataType: Upgrade - InputType: LDST - Simulation: true - CondDB: sim-20171127-vc-md100 - DDDB: dddb-20171126 - extra: - Author: 'dfazzini' - Date: '2020-06-17 18:34:43.57756' - comment: 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci' - genFSR_upgrade_ldst: filenames: - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst' -- GitLab From 21dde110d356aab662dca5403c7b365295903b31 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Sun, 8 Nov 2020 20:56:54 +0100 Subject: [PATCH 15/28] fix small typo --- Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py | 2 +- Phys/DaVinci/python/DaVinci/algorithms.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index 339f50eb..a3f00220 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -42,7 +42,7 @@ def main(fileName, jobOptFile, isMC): checkOptions(slots, log) #import GaudiConf.options.PreloadUnits.opts - defineInput(slots) + defineInput(slots, log) config = [] appMgr = C.ApplicationMgr() diff --git a/Phys/DaVinci/python/DaVinci/algorithms.py b/Phys/DaVinci/python/DaVinci/algorithms.py index 132f7eea..46763517 100644 --- a/Phys/DaVinci/python/DaVinci/algorithms.py +++ b/Phys/DaVinci/python/DaVinci/algorithms.py @@ -39,7 +39,7 @@ def dvInit(slots): ################################################################################ # Defines input # -def defineInput(slots): +def defineInput(slots, log): """ Define Input """ -- GitLab From 12cd574dd5c558d0663d03508b6eeb4105ffa65a Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Fri, 27 Nov 2020 23:13:18 +0100 Subject: [PATCH 16/28] finalize minimal version --- Phys/DaVinci/options/DaVinciDB_MC.yaml | 8 +- Phys/DaVinci/options/jobOptions-Example.py | 16 +++ .../python/DaVinci/ConfigurationUpgrade.py | 60 ++++------ Phys/DaVinci/python/DaVinci/algorithms.py | 109 +++++++++++++----- Phys/DaVinci/python/DaVinci/configOptions.py | 88 ++++++++++---- Phys/DaVinci/python/DaVinci/configurations.py | 97 +++++++++++----- 6 files changed, 255 insertions(+), 123 deletions(-) create mode 100644 Phys/DaVinci/options/jobOptions-Example.py diff --git a/Phys/DaVinci/options/DaVinciDB_MC.yaml b/Phys/DaVinci/options/DaVinciDB_MC.yaml index 478e3cdc..c40977e3 100644 --- a/Phys/DaVinci/options/DaVinciDB_MC.yaml +++ b/Phys/DaVinci/options/DaVinciDB_MC.yaml @@ -19,8 +19,8 @@ genFSR_upgrade_ldst: DataType: Upgrade InputType: LDST Simulation: true - CondDB: sim-20171127-vc-md100 - DDDB: dddb-20171126 + CondDBtag: sim-20171127-vc-md100 + DDDBtag: dddb-20171126 extra: Author: 'dfazzini' Date: '2020-06-17 18:34:43.57756' @@ -36,8 +36,8 @@ Upgrade_Bd2KstarMuMu: DataType: Upgrade InputType: LDST Simulation: true - CondDB: sim-20171127-vc-md100 - DDDB: dddb-20171126 + CondDBtag: sim-20171127-vc-md100 + DDDBtag: dddb-20171126 extra: Author: 'Patrick Koppenburg' Date: '2020-05-28 11:12:55' diff --git a/Phys/DaVinci/options/jobOptions-Example.py b/Phys/DaVinci/options/jobOptions-Example.py new file mode 100644 index 00000000..8a25cdfc --- /dev/null +++ b/Phys/DaVinci/options/jobOptions-Example.py @@ -0,0 +1,16 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +{'EvtMax': 20, + 'SkipEvents': 1, + 'TupleFile': 'DVNtuple.root', + 'HistogramFile': 'DVHistos.root' +} diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index a3f00220..9cc659a9 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -15,59 +15,49 @@ from GaudiConfig2 import Configurables as C from GaudiConfig2 import mergeConfigs from DaVinci.configOptions import initializeSlots, setPropertiesDict, checkOptions, setFileOptions, setJobOptions -from DaVinci.algorithms import defineInput, setSlotValue, getSlotValue, setupAlgorithms -from DaVinci.configurations import configureTiming, configureMessage, configExtSvc +from DaVinci.algorithms import dvInit, defineInput, setupAlgorithms, setupExtSvc, rootFiles, defineMonitors, defineLog +from DaVinci.configurations import configureAppMgr -from AnalysisPython.Logger import getLogger -log = getLogger ( 'DaVinci' ) - -def mc(fileName='genFSR_upgrade_ldst', jobOptFile='jobOptions.yaml'): - return main(fileName, jobOptFile, True) +def mc(key='genFSR_upgrade_ldst', dbFile='$DAVINCIROOT/options/DaVinciDB_MC.yaml', jobOptFile='$DAVINCIROOT/options/jobOptions.py'): + return main(key, dbFile, jobOptFile, True) -def data(fileName='', jobOptFile=''): +def data(key='', dbFile='', jobOptFile=''): raise ValueError('Data file with upgrade conditions are not yet available. Please use :mc function instead.') - return main(fileName, jobOptFile, False) + return main(key, dbFile, jobOptFile, False) -def main(fileName, jobOptFile, isMC): - """ +def main(key, dbFile, jobOptFile, isMC): + """ DaVinci application main configuration. """ slots = initializeSlots() propertyDocDct = setPropertiesDict() - setFileOptions(slots, fileName, isMC) - setJobOptions(slots, jobOptFile, isMC, log) + defineLog(slots) + + setFileOptions(slots, key, dbFile, isMC) + setJobOptions(slots, jobOptFile, key, dbFile) - log.info("Applying DaVinci configuration") - checkOptions(slots, log) + checkOptions(slots) #import GaudiConf.options.PreloadUnits.opts - defineInput(slots, log) + defineInput(slots) config = [] appMgr = C.ApplicationMgr() config.append(appMgr) - config.append(configureTiming()) - #config.append(configureMessage(slots)) - configExtSvc(config, slots) - setupAlgorithms(config, slots) - messageSvc = 0 - for c in config: - if c.name == 'MessageSvc': - messageSvc = c - - #appMgr.MessageSvcType = messageSvc - appMgr.EvtMax = getSlotValue(slots,"EvtMax") - appMgr.HistogramPersistency = 'ROOT' - - appMgr.ExtSvc = [c for c in config - if c.__component_type__ == 'Service' - and c.name not in ['ApplicationMgr']] - appMgr.TopAlg = [c for c in config - if c.__component_type__ == 'Algorithm' - and c.name not in ['ApplicationMgr']] + dvInit(config, slots) + defineMonitors(config, slots) + + setupAlgorithms(config, slots) + + setupExtSvc(config, slots) + + rootFiles(config, slots) + + configureAppMgr(appMgr, slots, config) + config = mergeConfigs(config) return config diff --git a/Phys/DaVinci/python/DaVinci/algorithms.py b/Phys/DaVinci/python/DaVinci/algorithms.py index 46763517..44aa7f86 100644 --- a/Phys/DaVinci/python/DaVinci/algorithms.py +++ b/Phys/DaVinci/python/DaVinci/algorithms.py @@ -10,39 +10,21 @@ ############################################################################### from GaudiConfig2 import Configurables as C - -################################################################################ -# Manage slots properties -# -""" -Set and retrieve the value of a specific slot property -""" -def getSlotValue(slots, name): - return slots[name] - -def setSlotValue(slots, name, value): - slots[name] = value +from DaVinci.configOptions import getSlotValue, setSlotValue +from DaVinci.configurations import * ################################################################################ # Event Initialisation sequence # -def dvInit(slots): - """ - Initialisation sequence - """ - di = C.DaVinciInit('DaVinciInitAlg', Increment = getSlotValue(slots, "PrintFreq")) - #di.addTool(MemoryTool) - #di.MemoryTool.HistoSize = 5000 - return di - +def dvInit(config, slots): + dvInit = C.DaVinciInit('DaVinciInitAlg') + configureInit(dvInit, slots) + config.append(dvInit) ################################################################################ # Defines input # -def defineInput(slots, log): - """ - Define Input - """ +def defineInput(slots): input = getSlotValue(slots, "Input") if ( len(input) > 0 ): @@ -56,8 +38,23 @@ def defineInput(slots, log): #clear selector to maintain the same behaviour IOHelper(persistency, persistency).inputFiles(input, clear = True) else: + log = getSlotValue(slots, "Log") log.warning('No input files available along with the selected key. Check the related DaVinciDB.') +################################################################################ +# Defines monitors +# +def defineMonitors(config, slots): + auditor = C.AuditorSvc() + timer = C.TimingAuditor() + sequencer = C.SequencerTimerTool() + configureTiming(auditor, timer, sequencer) + + evtSel = C.EventSelector() + configureEventSelector(evtSel, slots) + + for svc in [auditor, timer, sequencer, evtSel]: + config.append(svc) ################################################################################ # Defines algorithms @@ -66,12 +63,68 @@ def setupAlgorithms(config, slots): """ Define DaVinci algorithms """ - init = dvInit(slots) - config.append(init) - algorithms = [C.GaudiHistoAlgorithm('SimpleHistos', HistoPrint=True, OutputLevel=getSlotValue(slots, "OutputLevel"))] for alg in algorithms: config.append(alg) + +################################################################################ +# Defines external services +# +def setupExtSvc(config, slots): + """ + Define DaVinci services + """ + lokiSvc = C.LoKiSvc() + dataSvc = C.DataOnDemandSvc() + configurePrintOuts(lokiSvc, dataSvc, slots) + messageSvc = C.MessageSvc(OutputLevel = getSlotValue(slots,"OutputLevel")) + + evtPers = C.EvtPersistencySvc("EventPersistencySvc") + configureEventPers(evtPers) + + extSvc = [lokiSvc, + dataSvc, + evtPers, + #messageSvc, + C.ToolSvc(), + C.DetDataSvc('DetectorDataSvc'), + C.Gaudi.MultiFileCatalog('FileCatalog'), + C.Gaudi.IODataManager('IODataManager'), + C.Gaudi.RootCnvSvc('FileRecordCnvSvc'), + ] + + for svc in extSvc: + config.append(svc) + +################################################################################ +# Defines output root files +# +def rootFiles(config, slots): + """ + output files + """ + if getSlotValue(slots, "HistogramFile"): + histoSvc = C.HistogramPersistencySvc(OutputFile = getSlotValue(slots, "HistogramFile")) + config.append(histoSvc) + + if getSlotValue(slots, "TupleFile"): + ntSvc = C.NTupleSvc() + configureNtuplesROOT(ntSvc, slots) + config.append(ntSvc) + + # Set the compression level for the ROOT tuple file + from GaudiKernel.Configurable import ConfigurableGeneric as RFileCnv + RFileCnv('RFileCnv').GlobalCompression = getSlotValue(slots, "RootCompressionLevel") + +################################################################################ +# Defines DaVinci log +# +def defineLog(slots): + from AnalysisPython.Logger import getLogger + log = getLogger ( 'DaVinci' ) + log.info("Applying DaVinci configuration") + setSlotValue(slots, "Log", log) + diff --git a/Phys/DaVinci/python/DaVinci/configOptions.py b/Phys/DaVinci/python/DaVinci/configOptions.py index 486cfb6e..06c5cc24 100644 --- a/Phys/DaVinci/python/DaVinci/configOptions.py +++ b/Phys/DaVinci/python/DaVinci/configOptions.py @@ -10,8 +10,6 @@ ############################################################################### import os -from DaVinci import algorithms -from DaVinci.algorithms import getSlotValue, setSlotValue def initializeSlots(): """ @@ -50,48 +48,88 @@ def initializeSlots(): , "VerboseMessages" : False , "RootInTES" : "" , "Detectors" : ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] + # Log + , "Log" : "" } return slots -def setFileOptions(slots, name, isMC): +################################################################################ +# Manage slots properties and input job and data options +# +""" +Set and retrieve the value of a specific slot property +""" +def getSlotValue(slots, name): + return slots[name] + +def setSlotValue(slots, name, value): + if name in slots: + slots[name] = value + else: + import difflib + raise ValueError('Unknown qualifier called %s. Please check the DaVinci database or the job option file!\n ' + 'List of possible valid qualifiers: %s' %(name, difflib.get_close_matches(name, slots))) + +def setFileOptions(slots, myKey, dbName, isMC): """ Set the properties related to the dataset """ import yaml - dbName = 'DaVinciDB_MC.yaml' if isMC else 'DaVinciDB_data.yaml' - - dbName = '$DAVINCIROOT/options/%s' %dbName with open(os.path.expandvars(dbName)) as dbFile: dataDV = yaml.safe_load(dbFile) + idxFile = -1 + if ":" in myKey: + idxFile = int(myKey.split(":")[1]) + myKey = myKey.split(":")[0] + for key, config in dataDV.items(): - if key == name: + if key == myKey: for item, obj in config.items(): if item == 'qualifiers': for prop, value in obj.items(): setSlotValue(slots, prop, value) elif item == 'filenames': - setSlotValue(slots, "Input", obj) + if idxFile > -1: + if idxFile < len(obj): + setSlotValue(slots, "Input", [obj[idxFile]]) + else: + raise ValueError('Index file requested exceeds the number of available files related to the given key!') + """ + elif "*" in obj[0]: + import glob + listFiles = [] + #for idx in range[0, 1]: + print(glob.glob(str(obj[0]))) + print(obj[0]) + listFiles = glob.glob(obj[0]) + setSlotValue(slots, "Input", listFiles) + """ + else: + setSlotValue(slots, "Input", obj) -def setJobOptions(slots, filename, isMC, log): +def setJobOptions(slots, configName, myKey, dbName): """ Set the properties related to the job """ - if filename == '': + log = getSlotValue(slots, "Log") + if configName == '': log.warning('No jobOption file selected, the default values are used.') else: - dataOptions = listDataOptions(isMC) + dataOptions = listDataOptions(myKey.split(":")[0], dbName) - filename = '$DAVINCIROOT/options/%s' %filename - with open(os.path.expandvars(filename)) as config_file: - if filename.endswith('.json'): + with open(os.path.expandvars(configName)) as config_file: + if configName.endswith('.json'): import json config = json.load(config_file) - elif filename.endswith('.yaml') or filename.endswith('.yml'): + elif configName.endswith('.yaml') or configName.endswith('.yml'): import yaml config = yaml.safe_load(config_file) + elif configName.endswith('.py'): + import ast + config = ast.literal_eval(config_file.read()) else: - raise ValueError('JobOption file extension not known! Please use only a json or yaml file!') + raise ValueError('JobOption file extension not known! Please use only a .py, .json or .yaml (.yml) file!') for key, value in config.items(): if key in dataOptions: @@ -105,31 +143,31 @@ def setJobOptions(slots, filename, isMC, log): else: setSlotValue(slots, key, value) -def listDataOptions(isMC): +def listDataOptions(myKey, dbName): """ List of the properties that are set automatically given a dataset """ import yaml - dbName = 'DaVinciDB_MC.yaml' if isMC else 'DaVinciDB_data.yaml' - dbName = '$DAVINCIROOT/options/%s' %dbName with open(os.path.expandvars(dbName)) as dbFile: dataDV = yaml.safe_load(dbFile) optionList = [] for key, config in dataDV.items(): - for item, obj in config.items(): - if item == 'qualifiers': - for prop, value in obj.items(): - optionList.append(prop) - break + if key == myKey: + for item, obj in config.items(): + if item == 'qualifiers': + for prop, value in obj.items(): + optionList.append(prop) + break return optionList -def checkOptions(slots, log): +def checkOptions(slots): """ Checks options. Changes a few if needed. """ + log = getSlotValue(slots, "Log") known_datatypes = ["Upgrade"] known_default_detectors = ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] diff --git a/Phys/DaVinci/python/DaVinci/configurations.py b/Phys/DaVinci/python/DaVinci/configurations.py index b3fc4d53..e9fc7f17 100644 --- a/Phys/DaVinci/python/DaVinci/configurations.py +++ b/Phys/DaVinci/python/DaVinci/configurations.py @@ -10,39 +10,74 @@ ############################################################################### from GaudiConfig2 import Configurables as C -from DaVinci.algorithms import getSlotValue +from DaVinci.configOptions import getSlotValue -def configureTiming(): - return C.AuditorSvc('AuditorSvc', Auditors=['ChronoAuditor']) +################################################################################ +# Define configurations for applications and algorithms +# + +def configureAppMgr(appMgr, slots, config): + messageSvc = 0 + for c in config: + if c.name == 'MessageSvc': + messageSvc = c + + #appMgr.MessageSvcType = messageSvc + appMgr.EvtMax = getSlotValue(slots,"EvtMax") + appMgr.HistogramPersistency = 'ROOT' + appMgr.AuditAlgorithms = True + + appMgr.ExtSvc = [c for c in config + if c.__component_type__ == 'Service' + and c.name not in ['ApplicationMgr']] + appMgr.TopAlg = [c for c in config + if c.__component_type__ == 'Algorithm' + and c.name not in ['ApplicationMgr']] + +def configureInit(dvInit, slots): + dvInit.Increment = getSlotValue(slots, "PrintFreq") + #dvInit.addTool(C.MemoryTool()) + #dvInit.MemoryTool.HistoSize = 5000 -def configureMessage(slots): - return C.MessageSvc(OutputLevel = getSlotValue(slots,"OutputLevel")) ################################################################################ -# Define external services +# Define configurations for external services # -def configExtSvc(config, slots): - extSvc = [C.DetDataSvc('DetectorDataSvc'), - C.ToolSvc(), - C.AuditorSvc(), - C.NTupleSvc(), - C.LoKiSvc(), - C.DataOnDemandSvc(), - C.EventSelector(), - C.EvtPersistencySvc("EventPersistencySvc"), - C.Gaudi.MultiFileCatalog('FileCatalog'), - C.Gaudi.IODataManager('IODataManager'), - C.Gaudi.RootCnvSvc('FileRecordCnvSvc'), - ] - - # Define extra option and append services to main config list - for svc in extSvc: - if svc.name == "EventSelector": - svc.PrintFreq = getSlotValue(slots, "PrintFreq") - elif svc.name == "DataOnDemandSvc": - svc.Dump = getSlotValue(slots, "VerboseMessages") - elif svc.name == "EventPersistencySvc": - #CnvServices.append(C.Gaudi.RootCnvSvc("RootCnvSvc")) - svc.CnvServices.append("Gaudi::RootCnvSvc/RootCnvSvc") - - config.append(svc) + +def configureTiming(auditor, timer, sequencer): + auditor.Auditors = ['ChronoAuditor', 'TimingAuditor'] + sequencer.OutputLevel = 4 + #timer.addTool(sequencer, name="TIMER") + #if not timer.TIMER.isPropertySrt("NameSize"): + # timer.TIMER.NameSize = 50 + +def configureNtuplesROOT(svc, slots): + for _line in svc.Output : + if 0 <= _line.find ('FILE1') : + log = getSlotValue(slots, "Log") + log.warning ('Replace NTuple-LUN FILE1: ' + _line ) + svc.Output.remove ( _line ) + tupleFile = getSlotValue(slots, "TupleFile") + tupleStr = "FILE1 DATAFILE='%s' TYP='ROOT' OPT='NEW'" % tupleFile + svc.Output += [ tupleStr ] + svc.OutputLevel = 1 + +def configureEventPers(svc): + #CnvServices.append(C.Gaudi.RootCnvSvc("RootCnvSvc")) + svc.CnvServices.append("Gaudi::RootCnvSvc/RootCnvSvc") + +def configurePrintOuts(lokiSvc, dataSvc, slots): + verbosePrint = getSlotValue(slots, "VerboseMessages") + lokiSvc.Welcome = verbosePrint + dataSvc.Dump = verbosePrint + #if not verbosePrint: + # msgSvc = getConfigurable("MessageSvc") + # msgSvc.setWarning += ['RFileCnv'] + +def configureEventSelector(svc, slots): + printFreq = getSlotValue(slots, "PrintFreq") + if printFreq == 0: + log = getSlotValue(slots, "Log") + log.warning("Print frequence cannot be 0") + printFreq = 1000 + svc.PrintFreq = printFreq -- GitLab From 53e87e61ef34d354eec99a1498a619605c8c7495 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Sat, 28 Nov 2020 23:29:59 +0100 Subject: [PATCH 17/28] adding initial test --- .../test_davinci_initialise_upgrade.qmt | 40 +++++++++++++++++++ ...VinciDB_MC.yaml => DaVinciDB-Example.yaml} | 0 ...obOptions.yaml => jobOptions-Example.yaml} | 0 .../python/DaVinci/ConfigurationUpgrade.py | 2 +- 4 files changed, 41 insertions(+), 1 deletion(-) create mode 100755 DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt rename Phys/DaVinci/options/{DaVinciDB_MC.yaml => DaVinciDB-Example.yaml} (100%) rename Phys/DaVinci/options/{jobOptions.yaml => jobOptions-Example.yaml} (100%) diff --git a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt new file mode 100755 index 00000000..7fbce036 --- /dev/null +++ b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt @@ -0,0 +1,40 @@ + + + + + + gaudirun.py + DaVinci.ConfigurationUpgrade:mc + +findReferenceBlock(""" +TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- +TimingAuditor.T... INFO This machine has a speed about 2.94 times the speed of a 2.8 GHz Xeon. +TimingAuditor.T... INFO Algorithm (millisec) | | | min max sigma | entries | total (s) | +TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- +TimingAuditor.T... INFO EVENT LOOP | 1.000 | 1.086 | 0.088 19.2 4.26 | 20 | 0.022 | +TimingAuditor.T... INFO DaVinciInitAlg | 0.000 | 0.129 | 0.035 1.6 0.35 | 20 | 0.003 | +TimingAuditor.T... INFO SimpleHistos | 1.000 | 0.898 | 0.031 17.0 3.80 | 20 | 0.018 | +TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- +""", stdout, result, causes, signature_offset = 0, id='rest') +countErrorLines({"FATAL":0}) + + diff --git a/Phys/DaVinci/options/DaVinciDB_MC.yaml b/Phys/DaVinci/options/DaVinciDB-Example.yaml similarity index 100% rename from Phys/DaVinci/options/DaVinciDB_MC.yaml rename to Phys/DaVinci/options/DaVinciDB-Example.yaml diff --git a/Phys/DaVinci/options/jobOptions.yaml b/Phys/DaVinci/options/jobOptions-Example.yaml similarity index 100% rename from Phys/DaVinci/options/jobOptions.yaml rename to Phys/DaVinci/options/jobOptions-Example.yaml diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index 9cc659a9..26cb1e7c 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -18,7 +18,7 @@ from DaVinci.configOptions import initializeSlots, setPropertiesDict, checkOptio from DaVinci.algorithms import dvInit, defineInput, setupAlgorithms, setupExtSvc, rootFiles, defineMonitors, defineLog from DaVinci.configurations import configureAppMgr -def mc(key='genFSR_upgrade_ldst', dbFile='$DAVINCIROOT/options/DaVinciDB_MC.yaml', jobOptFile='$DAVINCIROOT/options/jobOptions.py'): +def mc(key='genFSR_upgrade_ldst', dbFile='$DAVINCIROOT/options/DaVinciDB-Example.yaml', jobOptFile='$DAVINCIROOT/options/jobOptions-Example.yaml'): return main(key, dbFile, jobOptFile, True) def data(key='', dbFile='', jobOptFile=''): -- GitLab From 33041b9a7664203ba7d76d5e30d12f55e5600405 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Mon, 30 Nov 2020 01:36:35 +0100 Subject: [PATCH 18/28] clean up the scripts --- .../test_davinci_initialise_upgrade.qmt | 3 - Phys/DaVinci/options/DVconfig-Default.yaml | 101 ++++++++++++++++ Phys/DaVinci/options/DaVinciDB-Example.yaml | 4 +- Phys/DaVinci/options/jobOptions-Example.py | 9 +- .../python/DaVinci/ConfigurationUpgrade.py | 9 +- Phys/DaVinci/python/DaVinci/algorithms.py | 41 +++---- Phys/DaVinci/python/DaVinci/configOptions.py | 114 ++++++------------ Phys/DaVinci/python/DaVinci/configurations.py | 36 ++++-- 8 files changed, 193 insertions(+), 124 deletions(-) create mode 100644 Phys/DaVinci/options/DVconfig-Default.yaml diff --git a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt index 7fbce036..6a7ad7d7 100755 --- a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt +++ b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt @@ -27,9 +27,6 @@ findReferenceBlock(""" TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- -TimingAuditor.T... INFO This machine has a speed about 2.94 times the speed of a 2.8 GHz Xeon. -TimingAuditor.T... INFO Algorithm (millisec) | | | min max sigma | entries | total (s) | -TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- TimingAuditor.T... INFO EVENT LOOP | 1.000 | 1.086 | 0.088 19.2 4.26 | 20 | 0.022 | TimingAuditor.T... INFO DaVinciInitAlg | 0.000 | 0.129 | 0.035 1.6 0.35 | 20 | 0.003 | TimingAuditor.T... INFO SimpleHistos | 1.000 | 0.898 | 0.031 17.0 3.80 | 20 | 0.018 | diff --git a/Phys/DaVinci/options/DVconfig-Default.yaml b/Phys/DaVinci/options/DVconfig-Default.yaml new file mode 100644 index 00000000..83803932 --- /dev/null +++ b/Phys/DaVinci/options/DVconfig-Default.yaml @@ -0,0 +1,101 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +# Application Configuration +EvtMax: + text: '""" Number of events to analyse """' + value: -1 +SkipEvents: + text: '""" Number of events to skip at beginning for file """' + value: 0 +PrintFreq: + text: '""" The frequency at which to print event numbers """' + value: 1000 +DataType: + text: '""" Data type, can be ["Upgrade"] Forwarded to PhysConf, AnalysisConf, DstConf and LHCbApp """' + value: 'Upgrade' +Simulation: + text: '""" set to True to use SimCond. Forwarded to PhysConf """' + value: False +DDDBtag: + text: '""" Data type, can be ["Upgrade"] Forwarded to PhysConf, AnalysisConf, DstConf and LHCbApp """' + value: 'dddb-20171126' +CondDBtag: + text: '""" Tag for CondDB. Default as set in DDDBConf for DataType """' + value: 'sim-20171127-vc-md100' +DQFLAGStag: + text: '""" Tag for DQFLAGS. Default as set in DDDBConf for DataType """' + value: '' +OutputLevel: + text: '""" Set output level used in the job """' + value: 3 +# Input +Input: + text: '""" Input data. Can also be passed as a second option file. """' + value: [] +InputType: + text: '""" "DST", "DIGI", "RDST", "MDST", "XDST" or "LDST". Nothing means the input type is compatible with being a DST """' + value: 'DST' +EnableUnpack: + text: '""" Explicitly enable/disable unpacking for input data (if specified) """' + value: None +OverwriteDataOptions: + text: '""" Allow overwriting default data options defined in DaVinci Database. """' + value: False +# Output +HistogramFile: + text: '""" Write name of output Histogram file """' + value: '' +TupleFile: + text: '""" Write name of output Tuple file """' + value: '' +WriteFSR: + text: '""" Flags whether to write out an FSR """' + value: True +MergeGenFSR: + text: '""" Flags whether to merge the generatore level FSRs """' + value: False +RootCompressionLevel: + text: '""" ROOT Compression level for ntuples """' + value: 'LZMA:6' +# DQ +IgnoreDQFlags: + text: '""" If False, process only events with good DQ. Default is False """' + value: False +# DaVinci Options +MainOptions: + text: '""" Main option file to execute """' + value: '' +UserAlgorithms: + text: '""" User algorithms to run. """' + value: [] +RedoMCLinks: + text: '""" On some stripped DST one needs to redo the Track<->MC link table. Set to true if problems with association. """' + value: False +Lumi: + text: '""" Run event count and Lumi accounting (should normally be True) """' + value: False +EventPreFilters: + text: '""" Set of event filtering algorithms to be run before DaVinci initializaton sequence. Only events passing these filters will be processed. """' + value: [] +VerboseMessages: + text: '""" Enable additional verbose printouts """' + value: False +RootInTES: + text: '""" RootInTES (for uDst input type) """' + value: '' +Detectors: + text: '""" List of detectors """' + value: ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr'] +# Log +Log: + text: '""" Logger used to print warning and error messages """' + value: '' \ No newline at end of file diff --git a/Phys/DaVinci/options/DaVinciDB-Example.yaml b/Phys/DaVinci/options/DaVinciDB-Example.yaml index c40977e3..1d06aaeb 100644 --- a/Phys/DaVinci/options/DaVinciDB-Example.yaml +++ b/Phys/DaVinci/options/DaVinciDB-Example.yaml @@ -9,7 +9,7 @@ # or submit itself to any jurisdiction. # ############################################################################### -genFSR_upgrade_ldst: +Upgrade_genFSR_ldst: filenames: - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst' - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade1.ldst' @@ -26,7 +26,7 @@ genFSR_upgrade_ldst: Date: '2020-06-17 18:34:43.57756' comment: 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci' -Upgrade_Bd2KstarMuMu: +Upgrade_Bd2KstarMuMu_ldst: filenames: - 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000002_1.ldst' - 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000004_1.ldst' diff --git a/Phys/DaVinci/options/jobOptions-Example.py b/Phys/DaVinci/options/jobOptions-Example.py index 8a25cdfc..6d429177 100644 --- a/Phys/DaVinci/options/jobOptions-Example.py +++ b/Phys/DaVinci/options/jobOptions-Example.py @@ -9,8 +9,9 @@ # or submit itself to any jurisdiction. # ############################################################################### -{'EvtMax': 20, - 'SkipEvents': 1, - 'TupleFile': 'DVNtuple.root', - 'HistogramFile': 'DVHistos.root' +{ + 'EvtMax': 100, + 'SkipEvents': 2, + 'TupleFile': 'DVNtuple.root', + 'HistogramFile': 'DVHistos.root' } diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index 26cb1e7c..9f7a0f88 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -14,11 +14,11 @@ High level configuration tools for DaVinci from GaudiConfig2 import Configurables as C from GaudiConfig2 import mergeConfigs -from DaVinci.configOptions import initializeSlots, setPropertiesDict, checkOptions, setFileOptions, setJobOptions +from DaVinci.configOptions import initializeSlots, checkOptions, setFileOptions, setJobOptions from DaVinci.algorithms import dvInit, defineInput, setupAlgorithms, setupExtSvc, rootFiles, defineMonitors, defineLog from DaVinci.configurations import configureAppMgr -def mc(key='genFSR_upgrade_ldst', dbFile='$DAVINCIROOT/options/DaVinciDB-Example.yaml', jobOptFile='$DAVINCIROOT/options/jobOptions-Example.yaml'): +def mc(key='Upgrade_genFSR_ldst', dbFile='$DAVINCIROOT/options/DaVinciDB-Example.yaml', jobOptFile='$DAVINCIROOT/options/jobOptions-Example.yaml'): return main(key, dbFile, jobOptFile, True) def data(key='', dbFile='', jobOptFile=''): @@ -29,8 +29,9 @@ def main(key, dbFile, jobOptFile, isMC): """ DaVinci application main configuration. """ - slots = initializeSlots() - propertyDocDct = setPropertiesDict() + slots = {} + propertyDocDct = {} + initializeSlots(slots, propertyDocDct) defineLog(slots) diff --git a/Phys/DaVinci/python/DaVinci/algorithms.py b/Phys/DaVinci/python/DaVinci/algorithms.py index 44aa7f86..54033823 100644 --- a/Phys/DaVinci/python/DaVinci/algorithms.py +++ b/Phys/DaVinci/python/DaVinci/algorithms.py @@ -13,18 +13,18 @@ from GaudiConfig2 import Configurables as C from DaVinci.configOptions import getSlotValue, setSlotValue from DaVinci.configurations import * -################################################################################ -# Event Initialisation sequence -# def dvInit(config, slots): + """ + Set initialisation algorithm + """ dvInit = C.DaVinciInit('DaVinciInitAlg') configureInit(dvInit, slots) config.append(dvInit) -################################################################################ -# Defines input -# def defineInput(slots): + """ + Set input files + """ input = getSlotValue(slots, "Input") if ( len(input) > 0 ): @@ -41,10 +41,10 @@ def defineInput(slots): log = getSlotValue(slots, "Log") log.warning('No input files available along with the selected key. Check the related DaVinciDB.') -################################################################################ -# Defines monitors -# def defineMonitors(config, slots): + """ + Set monitors + """ auditor = C.AuditorSvc() timer = C.TimingAuditor() sequencer = C.SequencerTimerTool() @@ -56,12 +56,9 @@ def defineMonitors(config, slots): for svc in [auditor, timer, sequencer, evtSel]: config.append(svc) -################################################################################ -# Defines algorithms -# def setupAlgorithms(config, slots): """ - Define DaVinci algorithms + Set DaVinci algorithms """ algorithms = [C.GaudiHistoAlgorithm('SimpleHistos', HistoPrint=True, @@ -70,12 +67,9 @@ def setupAlgorithms(config, slots): for alg in algorithms: config.append(alg) -################################################################################ -# Defines external services -# def setupExtSvc(config, slots): """ - Define DaVinci services + Set DaVinci services """ lokiSvc = C.LoKiSvc() dataSvc = C.DataOnDemandSvc() @@ -99,12 +93,9 @@ def setupExtSvc(config, slots): for svc in extSvc: config.append(svc) -################################################################################ -# Defines output root files -# def rootFiles(config, slots): """ - output files + Set output .root files """ if getSlotValue(slots, "HistogramFile"): histoSvc = C.HistogramPersistencySvc(OutputFile = getSlotValue(slots, "HistogramFile")) @@ -112,17 +103,17 @@ def rootFiles(config, slots): if getSlotValue(slots, "TupleFile"): ntSvc = C.NTupleSvc() - configureNtuplesROOT(ntSvc, slots) + configureOptions_rootNtuples(ntSvc, slots) config.append(ntSvc) # Set the compression level for the ROOT tuple file from GaudiKernel.Configurable import ConfigurableGeneric as RFileCnv RFileCnv('RFileCnv').GlobalCompression = getSlotValue(slots, "RootCompressionLevel") -################################################################################ -# Defines DaVinci log -# def defineLog(slots): + """ + Set logger + """ from AnalysisPython.Logger import getLogger log = getLogger ( 'DaVinci' ) log.info("Applying DaVinci configuration") diff --git a/Phys/DaVinci/python/DaVinci/configOptions.py b/Phys/DaVinci/python/DaVinci/configOptions.py index 06c5cc24..c54975ca 100644 --- a/Phys/DaVinci/python/DaVinci/configOptions.py +++ b/Phys/DaVinci/python/DaVinci/configOptions.py @@ -11,48 +11,6 @@ import os -def initializeSlots(): - """ - Initialize options with default values. - """ - slots = { - # Application Configuration : sent to LHCbApp and Gaudi - "EvtMax" : -1 - , "SkipEvents" : 0 - , "PrintFreq" : 1000 - , "DataType" : 'Upgrade' - , "Simulation" : True - , "DDDBtag" : "dddb-20171126" - , "CondDBtag" : "sim-20171127-vc-md100" - , "DQFLAGStag" : "" - , "OutputLevel" : 3 - # Input - , "Input" : [] - , "InputType" : "DST" - , 'EnableUnpack' : None - , "OverwriteDataOptions" : False - # Output - , "HistogramFile" : "" - , "TupleFile" : "" - , "WriteFSR" : True - , "MergeGenFSR" : False - , "RootCompressionLevel" : "LZMA:6" - # DQ - , "IgnoreDQFlags" : False - # DaVinci Options - , "MainOptions" : "" - , "UserAlgorithms" : [] - , "RedoMCLinks" : False - , "Lumi" : False - , "EventPreFilters" : [] - , "VerboseMessages" : False - , "RootInTES" : "" - , "Detectors" : ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] - # Log - , "Log" : "" - } - return slots - ################################################################################ # Manage slots properties and input job and data options # @@ -69,10 +27,42 @@ def setSlotValue(slots, name, value): import difflib raise ValueError('Unknown qualifier called %s. Please check the DaVinci database or the job option file!\n ' 'List of possible valid qualifiers: %s' %(name, difflib.get_close_matches(name, slots))) - + +def getDefaultValue(myKey): + """ + Get default value for a specific property + """ + import yaml + optsDefaultName = "$DAVINCIROOT/options/DVconfig-Default.yaml" + with open(os.path.expandvars(optsDefaultName)) as optsDefault: + config = yaml.safe_load(optsDefault) + + for key, args in config.items(): + if key == myKey: + for name, value in args.items(): + if name == 'value': + return value + return 0 + +def initializeSlots(slots, propDct): + """ + Initialize properties with default values. + """ + import yaml + optsDefaultName = "$DAVINCIROOT/options/DVconfig-Default.yaml" + with open(os.path.expandvars(optsDefaultName)) as optsDefault: + config = yaml.safe_load(optsDefault) + + for key, args in config.items(): + for name, value in args.items(): + if name == 'value': + slots[key] = value + else: + propDct[key] = value + def setFileOptions(slots, myKey, dbName, isMC): """ - Set the properties related to the dataset + Set the dataset properties required by the user """ import yaml with open(os.path.expandvars(dbName)) as dbFile: @@ -110,7 +100,7 @@ def setFileOptions(slots, myKey, dbName, isMC): def setJobOptions(slots, configName, myKey, dbName): """ - Set the properties related to the job + Set the job properties required by the user """ log = getSlotValue(slots, "Log") if configName == '': @@ -200,37 +190,3 @@ def checkOptions(slots): if getSlotValue(slots, "MergeGenFSR") and not getSlotValue(slots, "Simulation"): raise TypeError("Cannot MergeGenFSR on real data"); - -def setPropertiesDict(): - """ - Define a dictionary with option details. - """ - propDct = { - "EvtMax" : """ Number of events to analyse """ - , "SkipEvents" : """ Number of events to skip at beginning for file """ - , "PrintFreq" : """ The frequency at which to print event numbers """ - , "DataType" : """ Data type, can be ['Upgrade'] Forwarded to PhysConf, AnalysisConf, DstConf and LHCbApp """ - , "Simulation" : """ set to True to use SimCond. Forwarded to PhysConf """ - , "DDDBtag" : """ Tag for DDDB. Default as set in DDDBConf for DataType """ - , "CondDBtag" : """ Tag for CondDB. Default as set in DDDBConf for DataType """ - , "DQFLAGStag" : """ Tag for DQFLAGS. Default as set in DDDBConf for DataType """ - , "Input" : """ Input data. Can also be passed as a second option file. """ - , "InputType" : """ 'DST', 'DIGI', 'RDST', 'MDST', 'XDST' or 'LDST'. Nothing means the input type is compatible with being a DST. """ - , 'EnableUnpack' : """Explicitly enable/disable unpacking for input data (if specified) """ - , "OverwriteDataOptions" : """ Allow overwriting default data options defined in DaVinci Database. """ - , "HistogramFile" : """ Write name of output Histogram file """ - , "TupleFile" : """ Write name of output Tuple file """ - , 'WriteFSR' : """ Flags whether to write out an FSR """ - , 'MergeGenFSR' : """ Flags whether to merge the generatore level FSRs """ - , "RootCompressionLevel" : """ ROOT Compression level for ntuples """ - , 'IgnoreDQFlags' : """ If False, process only events with good DQ. Default is False """ - , "MainOptions" : """ Main option file to execute """ - , "UserAlgorithms" : """ User algorithms to run. """ - , "RedoMCLinks" : """ On some stripped DST one needs to redo the Track<->MC link table. Set to true if problems with association. """ - , "Lumi" : """ Run event count and Lumi accounting (should normally be True) """ - , "EventPreFilters" : """Set of event filtering algorithms to be run before DaVinci initializaton sequence. Only events passing these filters will be processed.""" - , "VerboseMessages" : """ Enable additional verbose printouts """ - , "RootInTES" : """ RootInTES (for uDst input type) """ - , "Detectors" : """ List of detectors """ - } - return propDct diff --git a/Phys/DaVinci/python/DaVinci/configurations.py b/Phys/DaVinci/python/DaVinci/configurations.py index e9fc7f17..965e6752 100644 --- a/Phys/DaVinci/python/DaVinci/configurations.py +++ b/Phys/DaVinci/python/DaVinci/configurations.py @@ -17,6 +17,9 @@ from DaVinci.configOptions import getSlotValue # def configureAppMgr(appMgr, slots, config): + """ + Configuration of the main application + """ messageSvc = 0 for c in config: if c.name == 'MessageSvc': @@ -35,6 +38,9 @@ def configureAppMgr(appMgr, slots, config): and c.name not in ['ApplicationMgr']] def configureInit(dvInit, slots): + """ + Configuration of + """ dvInit.Increment = getSlotValue(slots, "PrintFreq") #dvInit.addTool(C.MemoryTool()) #dvInit.MemoryTool.HistoSize = 5000 @@ -45,28 +51,40 @@ def configureInit(dvInit, slots): # def configureTiming(auditor, timer, sequencer): + """ + Configuration of timing auditors + """ auditor.Auditors = ['ChronoAuditor', 'TimingAuditor'] sequencer.OutputLevel = 4 #timer.addTool(sequencer, name="TIMER") #if not timer.TIMER.isPropertySrt("NameSize"): # timer.TIMER.NameSize = 50 -def configureNtuplesROOT(svc, slots): +def configureOptions_rootNtuples(svc, slots): + """ + Configuration of output .root ntuples + """ for _line in svc.Output : if 0 <= _line.find ('FILE1') : log = getSlotValue(slots, "Log") log.warning ('Replace NTuple-LUN FILE1: ' + _line ) svc.Output.remove ( _line ) - tupleFile = getSlotValue(slots, "TupleFile") - tupleStr = "FILE1 DATAFILE='%s' TYP='ROOT' OPT='NEW'" % tupleFile - svc.Output += [ tupleStr ] - svc.OutputLevel = 1 + tupleFile = getSlotValue(slots, "TupleFile") + tupleStr = "FILE1 DATAFILE='%s' TYP='ROOT' OPT='NEW'" % tupleFile + svc.Output += [ tupleStr ] + svc.OutputLevel = 1 def configureEventPers(svc): + """ + Configuration of the event persistency service + """ #CnvServices.append(C.Gaudi.RootCnvSvc("RootCnvSvc")) svc.CnvServices.append("Gaudi::RootCnvSvc/RootCnvSvc") def configurePrintOuts(lokiSvc, dataSvc, slots): + """ + Configuration of the printout services + """ verbosePrint = getSlotValue(slots, "VerboseMessages") lokiSvc.Welcome = verbosePrint dataSvc.Dump = verbosePrint @@ -75,9 +93,13 @@ def configurePrintOuts(lokiSvc, dataSvc, slots): # msgSvc.setWarning += ['RFileCnv'] def configureEventSelector(svc, slots): + """ + Configuration of the event selector + """ printFreq = getSlotValue(slots, "PrintFreq") if printFreq == 0: + printFreq = getDefaultValue("PrintFreq") log = getSlotValue(slots, "Log") - log.warning("Print frequence cannot be 0") - printFreq = 1000 + log.warning("Print frequency cannot be 0. Set to %f." %printFreq) svc.PrintFreq = printFreq + -- GitLab From 0e7697c8f6be958649aa19644d96f042e3004638 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Mon, 30 Nov 2020 01:49:19 +0100 Subject: [PATCH 19/28] espand name of configuration function of the event persistency service --- Phys/DaVinci/python/DaVinci/configurations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Phys/DaVinci/python/DaVinci/configurations.py b/Phys/DaVinci/python/DaVinci/configurations.py index 965e6752..bc8b8506 100644 --- a/Phys/DaVinci/python/DaVinci/configurations.py +++ b/Phys/DaVinci/python/DaVinci/configurations.py @@ -74,7 +74,7 @@ def configureOptions_rootNtuples(svc, slots): svc.Output += [ tupleStr ] svc.OutputLevel = 1 -def configureEventPers(svc): +def configureEventPersistecy(svc): """ Configuration of the event persistency service """ -- GitLab From 923743a525afa6366e7d489a4a1d7da2b5cdebfe Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Mon, 30 Nov 2020 09:42:15 +0100 Subject: [PATCH 20/28] cleaning up for merge request --- .../DaVinci/python/DaVinci/ConfigurationUpgrade.py | 1 - Phys/DaVinci/python/DaVinci/algorithms.py | 4 +--- Phys/DaVinci/python/DaVinci/configOptions.py | 10 ---------- Phys/DaVinci/python/DaVinci/configurations.py | 14 ++------------ 4 files changed, 3 insertions(+), 26 deletions(-) diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py index 9f7a0f88..b339afd8 100644 --- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -40,7 +40,6 @@ def main(key, dbFile, jobOptFile, isMC): checkOptions(slots) - #import GaudiConf.options.PreloadUnits.opts defineInput(slots) config = [] diff --git a/Phys/DaVinci/python/DaVinci/algorithms.py b/Phys/DaVinci/python/DaVinci/algorithms.py index 54033823..e608dc11 100644 --- a/Phys/DaVinci/python/DaVinci/algorithms.py +++ b/Phys/DaVinci/python/DaVinci/algorithms.py @@ -74,15 +74,13 @@ def setupExtSvc(config, slots): lokiSvc = C.LoKiSvc() dataSvc = C.DataOnDemandSvc() configurePrintOuts(lokiSvc, dataSvc, slots) - messageSvc = C.MessageSvc(OutputLevel = getSlotValue(slots,"OutputLevel")) evtPers = C.EvtPersistencySvc("EventPersistencySvc") - configureEventPers(evtPers) + configureEventPersistency(evtPers) extSvc = [lokiSvc, dataSvc, evtPers, - #messageSvc, C.ToolSvc(), C.DetDataSvc('DetectorDataSvc'), C.Gaudi.MultiFileCatalog('FileCatalog'), diff --git a/Phys/DaVinci/python/DaVinci/configOptions.py b/Phys/DaVinci/python/DaVinci/configOptions.py index c54975ca..6fb88cf2 100644 --- a/Phys/DaVinci/python/DaVinci/configOptions.py +++ b/Phys/DaVinci/python/DaVinci/configOptions.py @@ -85,16 +85,6 @@ def setFileOptions(slots, myKey, dbName, isMC): setSlotValue(slots, "Input", [obj[idxFile]]) else: raise ValueError('Index file requested exceeds the number of available files related to the given key!') - """ - elif "*" in obj[0]: - import glob - listFiles = [] - #for idx in range[0, 1]: - print(glob.glob(str(obj[0]))) - print(obj[0]) - listFiles = glob.glob(obj[0]) - setSlotValue(slots, "Input", listFiles) - """ else: setSlotValue(slots, "Input", obj) diff --git a/Phys/DaVinci/python/DaVinci/configurations.py b/Phys/DaVinci/python/DaVinci/configurations.py index bc8b8506..4b323db0 100644 --- a/Phys/DaVinci/python/DaVinci/configurations.py +++ b/Phys/DaVinci/python/DaVinci/configurations.py @@ -25,7 +25,6 @@ def configureAppMgr(appMgr, slots, config): if c.name == 'MessageSvc': messageSvc = c - #appMgr.MessageSvcType = messageSvc appMgr.EvtMax = getSlotValue(slots,"EvtMax") appMgr.HistogramPersistency = 'ROOT' appMgr.AuditAlgorithms = True @@ -39,11 +38,9 @@ def configureAppMgr(appMgr, slots, config): def configureInit(dvInit, slots): """ - Configuration of + Configuration of initialisation algorithm """ dvInit.Increment = getSlotValue(slots, "PrintFreq") - #dvInit.addTool(C.MemoryTool()) - #dvInit.MemoryTool.HistoSize = 5000 ################################################################################ @@ -56,9 +53,6 @@ def configureTiming(auditor, timer, sequencer): """ auditor.Auditors = ['ChronoAuditor', 'TimingAuditor'] sequencer.OutputLevel = 4 - #timer.addTool(sequencer, name="TIMER") - #if not timer.TIMER.isPropertySrt("NameSize"): - # timer.TIMER.NameSize = 50 def configureOptions_rootNtuples(svc, slots): """ @@ -74,11 +68,10 @@ def configureOptions_rootNtuples(svc, slots): svc.Output += [ tupleStr ] svc.OutputLevel = 1 -def configureEventPersistecy(svc): +def configureEventPersistency(svc): """ Configuration of the event persistency service """ - #CnvServices.append(C.Gaudi.RootCnvSvc("RootCnvSvc")) svc.CnvServices.append("Gaudi::RootCnvSvc/RootCnvSvc") def configurePrintOuts(lokiSvc, dataSvc, slots): @@ -88,9 +81,6 @@ def configurePrintOuts(lokiSvc, dataSvc, slots): verbosePrint = getSlotValue(slots, "VerboseMessages") lokiSvc.Welcome = verbosePrint dataSvc.Dump = verbosePrint - #if not verbosePrint: - # msgSvc = getConfigurable("MessageSvc") - # msgSvc.setWarning += ['RFileCnv'] def configureEventSelector(svc, slots): """ -- GitLab From 7b64bdca7e65156495ddf9fbf67a4e4b27ec5a15 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Mon, 30 Nov 2020 17:31:43 +0100 Subject: [PATCH 21/28] improving the code --- Phys/DaVinci/options/DaVinciDB-Example.yaml | 8 ++++---- Phys/DaVinci/python/DaVinci/algorithms.py | 9 +++------ Phys/DaVinci/python/DaVinci/configOptions.py | 4 ++++ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Phys/DaVinci/options/DaVinciDB-Example.yaml b/Phys/DaVinci/options/DaVinciDB-Example.yaml index 1d06aaeb..96eef490 100644 --- a/Phys/DaVinci/options/DaVinciDB-Example.yaml +++ b/Phys/DaVinci/options/DaVinciDB-Example.yaml @@ -21,10 +21,10 @@ Upgrade_genFSR_ldst: Simulation: true CondDBtag: sim-20171127-vc-md100 DDDBtag: dddb-20171126 - extra: + metadata: Author: 'dfazzini' Date: '2020-06-17 18:34:43.57756' - comment: 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci' + comment: 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci' Upgrade_Bd2KstarMuMu_ldst: filenames: @@ -38,8 +38,8 @@ Upgrade_Bd2KstarMuMu_ldst: Simulation: true CondDBtag: sim-20171127-vc-md100 DDDBtag: dddb-20171126 - extra: + metadata: Author: 'Patrick Koppenburg' Date: '2020-05-28 11:12:55' - comment: 'K*mumu by Moore Hlt1-filtered with TCK-0x52000000' + comment: 'K*mumu by Moore Hlt1-filtered with TCK-0x52000000' \ No newline at end of file diff --git a/Phys/DaVinci/python/DaVinci/algorithms.py b/Phys/DaVinci/python/DaVinci/algorithms.py index e608dc11..75cf9266 100644 --- a/Phys/DaVinci/python/DaVinci/algorithms.py +++ b/Phys/DaVinci/python/DaVinci/algorithms.py @@ -53,8 +53,7 @@ def defineMonitors(config, slots): evtSel = C.EventSelector() configureEventSelector(evtSel, slots) - for svc in [auditor, timer, sequencer, evtSel]: - config.append(svc) + config.extend([auditor, timer, sequencer, evtSel]) def setupAlgorithms(config, slots): """ @@ -64,8 +63,7 @@ def setupAlgorithms(config, slots): HistoPrint=True, OutputLevel=getSlotValue(slots, "OutputLevel"))] - for alg in algorithms: - config.append(alg) + config.extend(algorithms) def setupExtSvc(config, slots): """ @@ -88,8 +86,7 @@ def setupExtSvc(config, slots): C.Gaudi.RootCnvSvc('FileRecordCnvSvc'), ] - for svc in extSvc: - config.append(svc) + config.extend(extSvc) def rootFiles(config, slots): """ diff --git a/Phys/DaVinci/python/DaVinci/configOptions.py b/Phys/DaVinci/python/DaVinci/configOptions.py index 6fb88cf2..0f1b337b 100644 --- a/Phys/DaVinci/python/DaVinci/configOptions.py +++ b/Phys/DaVinci/python/DaVinci/configOptions.py @@ -99,10 +99,14 @@ def setJobOptions(slots, configName, myKey, dbName): dataOptions = listDataOptions(myKey.split(":")[0], dbName) with open(os.path.expandvars(configName)) as config_file: + """ if configName.endswith('.json'): import json config = json.load(config_file) elif configName.endswith('.yaml') or configName.endswith('.yml'): + """ + _, ext = os.path.splitext(configName) + if ext in (".yaml", ".yml", ".json"): import yaml config = yaml.safe_load(config_file) elif configName.endswith('.py'): -- GitLab From 363c65f5e338daeaaa1b0b0822179a65ed39e7a3 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Tue, 1 Dec 2020 08:40:14 +0100 Subject: [PATCH 22/28] updating reference for initialise upgrade test --- .../qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt | 6 +++--- Phys/DaVinci/options/DaVinciDB-Example.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt index 6a7ad7d7..06c3bd7b 100755 --- a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt +++ b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt @@ -27,9 +27,9 @@ findReferenceBlock(""" TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- -TimingAuditor.T... INFO EVENT LOOP | 1.000 | 1.086 | 0.088 19.2 4.26 | 20 | 0.022 | -TimingAuditor.T... INFO DaVinciInitAlg | 0.000 | 0.129 | 0.035 1.6 0.35 | 20 | 0.003 | -TimingAuditor.T... INFO SimpleHistos | 1.000 | 0.898 | 0.031 17.0 3.80 | 20 | 0.018 | +TimingAuditor.T... INFO EVENT LOOP | 2.000 | 1.952 | 0.357 29.8 6.56 | 20 | 0.039 | +TimingAuditor.T... INFO DaVinciInitAlg | 0.500 | 0.136 | 0.067 0.9 0.18 | 20 | 0.003 | +TimingAuditor.T... INFO SimpleHistos | 1.500 | 1.599 | 0.153 27.9 6.18 | 20 | 0.032 | TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- """, stdout, result, causes, signature_offset = 0, id='rest') countErrorLines({"FATAL":0}) diff --git a/Phys/DaVinci/options/DaVinciDB-Example.yaml b/Phys/DaVinci/options/DaVinciDB-Example.yaml index 96eef490..1b0463e8 100644 --- a/Phys/DaVinci/options/DaVinciDB-Example.yaml +++ b/Phys/DaVinci/options/DaVinciDB-Example.yaml @@ -24,7 +24,7 @@ Upgrade_genFSR_ldst: metadata: Author: 'dfazzini' Date: '2020-06-17 18:34:43.57756' - comment: 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci' + Comment: 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci' Upgrade_Bd2KstarMuMu_ldst: filenames: @@ -41,5 +41,5 @@ Upgrade_Bd2KstarMuMu_ldst: metadata: Author: 'Patrick Koppenburg' Date: '2020-05-28 11:12:55' - comment: 'K*mumu by Moore Hlt1-filtered with TCK-0x52000000' + Comment: 'K*mumu by Moore Hlt1-filtered with TCK-0x52000000' \ No newline at end of file -- GitLab From 0d524082c9e38471f77aa1b7f7531c4c9b4920a9 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Tue, 1 Dec 2020 20:36:52 +0100 Subject: [PATCH 23/28] updating validator of initialise qmtest --- .../test_davinci_initialise_upgrade.qmt | 20 +++++++++++-------- .../refs/test_davinci_initialise_upgrade.ref | 3 +++ 2 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref diff --git a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt index 06c3bd7b..f7e17de0 100755 --- a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt +++ b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt @@ -25,13 +25,17 @@ gaudirun.py DaVinci.ConfigurationUpgrade:mc -findReferenceBlock(""" -TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- -TimingAuditor.T... INFO EVENT LOOP | 2.000 | 1.952 | 0.357 29.8 6.56 | 20 | 0.039 | -TimingAuditor.T... INFO DaVinciInitAlg | 0.500 | 0.136 | 0.067 0.9 0.18 | 20 | 0.003 | -TimingAuditor.T... INFO SimpleHistos | 1.500 | 1.599 | 0.153 27.9 6.18 | 20 | 0.032 | -TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- -""", stdout, result, causes, signature_offset = 0, id='rest') -countErrorLines({"FATAL":0}) +refFileName = "$DAVINCITESTSROOT/tests/refs/test_davinci_initialise_upgrade.ref" +if not os.path.exists(os.path.expandvars(refFileName)): + causes.append('Reference file not found!') +else: + refFile = open(os.path.expandvars(refFileName), "r") + + for line in refFile: + if not stdout.find(line): + _, name = line.split("INFO ") + causes.append("Algorithm %s not run in the job!" %name) + +countErrorLines({"FATAL":0, "ERROR":0}) diff --git a/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref new file mode 100644 index 00000000..cf4380ff --- /dev/null +++ b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref @@ -0,0 +1,3 @@ +TimingAuditor.T... INFO EVENT LOOP +TimingAuditor.T... INFO DaVinciInitAlg +TimingAuditor.T... INFO SimpleHistos \ No newline at end of file -- GitLab From 9254f192290307d62bc4e58b847cb78b6eaf32d8 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Wed, 2 Dec 2020 18:46:39 +0100 Subject: [PATCH 24/28] change validation method --- .../test_davinci_initialise_upgrade.qmt | 16 +-- DaVinciTests/tests/refs/empty.ref | 0 .../refs/test_davinci_initialise_upgrade.ref | 118 +++++++++++++++++- 3 files changed, 119 insertions(+), 15 deletions(-) create mode 100644 DaVinciTests/tests/refs/empty.ref diff --git a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt index f7e17de0..14895aa8 100755 --- a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt +++ b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt @@ -24,18 +24,10 @@ gaudirun.py DaVinci.ConfigurationUpgrade:mc + $DAVINCITESTSROOT/tests/refs/test_initialise_upgrade + $DAVINCITESTSROOT/tests/refs/empty.ref -refFileName = "$DAVINCITESTSROOT/tests/refs/test_davinci_initialise_upgrade.ref" -if not os.path.exists(os.path.expandvars(refFileName)): - causes.append('Reference file not found!') -else: - refFile = open(os.path.expandvars(refFileName), "r") - - for line in refFile: - if not stdout.find(line): - _, name = line.split("INFO ") - causes.append("Algorithm %s not run in the job!" %name) - -countErrorLines({"FATAL":0, "ERROR":0}) +from DaVinciTests.QMTest.DaVinciExclusions import preprocessor +validateWithReference(preproc = preprocessor) diff --git a/DaVinciTests/tests/refs/empty.ref b/DaVinciTests/tests/refs/empty.ref new file mode 100644 index 00000000..e69de29b diff --git a/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref index cf4380ff..e05bf7e0 100644 --- a/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref +++ b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref @@ -1,3 +1,115 @@ -TimingAuditor.T... INFO EVENT LOOP -TimingAuditor.T... INFO DaVinciInitAlg -TimingAuditor.T... INFO SimpleHistos \ No newline at end of file +# setting LC_ALL to "C" +# DaVinci INFO Applying DaVinci configuration +ApplicationMgr SUCCESS +==================================================================================================================================== + Welcome to DaVinciDev version HEAD + running on lxplus760.cern.ch on Mon Nov 30 09:17:00 2020 +==================================================================================================================================== +ApplicationMgr INFO Application Manager Configured successfully +RndmGenSvc.Engine INFO Generator engine type:CLHEP::RanluxEngine +RndmGenSvc.Engine INFO Current Seed:1234567 Luxury:3 +RndmGenSvc INFO Using Random engine:HepRndm::Engine +TimingAuditor.T... INFO This machine has a speed about 2.86 times the speed of a 2.8 GHz Xeon. +DetectorDataSvc INFO Detector description not requested to be loaded +RootHistSvc INFO Writing ROOT histograms to: DVHistos.root +HistogramPersis... INFO Added successfully Conversion service:RootHistSvc +NTupleSvc DEBUG Property update for OutputLevel : new value = 1 +NTupleSvc DEBUG Service base class initialized successfully +NTupleSvc VERBOSE ServiceLocatorHelper::service: found service IncidentSvc +NTupleSvc INFO Added stream file:DVNtuple.root as FILE1 +DaVinciInitAlg SUCCESS ================================================================== +DaVinciInitAlg SUCCESS Requested to process 100 events +DaVinciInitAlg SUCCESS ================================================================== +EventPersistenc... INFO Added successfully Conversion service:RootCnvSvc +EventSelector INFO Stream:EventSelector.DataStreamTool_1 Def:DATAFILE='root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst' SVC='Gaudi::RootEvtSelector' OPT='READ' +ApplicationMgr INFO Application Manager Initialized successfully +ApplicationMgr INFO Application Manager Started successfully +EventSelector SUCCESS Reading Event record 1. Record number within stream 1: 1 +SimpleHistos INFO GaudiHistoAlgorithm:: Filling Histograms...... Please be patient ! +EventSelector INFO Stream:EventSelector.DataStreamTool_2 Def:DATAFILE='root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade1.ldst' SVC='Gaudi::RootEvtSelector' OPT='READ' +EventSelector INFO Stream:EventSelector.DataStreamTool_3 Def:DATAFILE='root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade2.ldst' SVC='Gaudi::RootEvtSelector' OPT='READ' +EventSelector INFO Stream:EventSelector.DataStreamTool_4 Def:DATAFILE='root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade3.ldst' SVC='Gaudi::RootEvtSelector' OPT='READ' +IODataManager INFO Disconnect from dataset root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst [22F2552E-BBB0-EA11-AE30-FA163E0F4653] +RootCnvSvc INFO Removed disconnected IO stream:22F2552E-BBB0-EA11-AE30-FA163E0F4653 [root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst] +EventLoopMgr INFO No more events in event selection +ApplicationMgr INFO Application Manager Stopped successfully +DaVinciInitAlg SUCCESS ================================================================== +DaVinciInitAlg SUCCESS 20 events processed +DaVinciInitAlg SUCCESS ================================================================== +DaVinciInitAlg WARNING Should have processed 100 events +DaVinciInitAlg WARNING ================================================================== +SimpleHistos SUCCESS Booked 30 Histogram(s) : 1D=10 2D=5 3D=3 1DProf=9 2DProf=3 +SimpleHistos SUCCESS 1D histograms in directory "SimpleHistos" : 10 + | ID | Title | # | Mean | RMS | Skewness | Kurtosis | + | 101 | "Exponential" | 20 | 0.86883 | 0.69261 | 0.96379 | 0.55029 | + | 102 | "Breit" | 20 | -0.03827 | 1.064 | 0.069108 | -0.531 | + | 1111 | "Forced Numeric ID time test" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | + | AutoID time test | "AutoID time test" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | + | Gaussian mean=0, sigma=1 | "Gaussian mean=0, sigma=1" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | + | poisson | "Poisson" | 20 | 1.8947 | 1.4471 | 0.49597 | -1.2241 | + | subdir1/bino | "Binominal" | 20 | 2.25 | 1.299 | 0.055654 | -1.0884 | + | subdir2/bino | "Binominal" | 20 | 2.25 | 1.299 | 0.055654 | -1.0884 | + | test1 | "Forced Alpha ID time test" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | + | varBinning/x | "1D Variable Binning" | 20 | 0.93986 | 2.2384 | -0.48081 | -0.51602 | +SimpleHistos SUCCESS 2D histograms in directory "SimpleHistos" : 5 + ID=Exponential V Flat "Exponential V Flat" Ents/All= 18/20 /sX=0.93986/2.2384,/sY=0.7512/0.77771 + ID=Gaussian V Flat "Gaussian V Flat" Ents/All= 20/20 /sX=0.93986/2.2384,/sY=-0.89604/0.96206 + ID=binVpois "Binom V poisson" Ents/All= 20/20 /sX=1.8947/1.4471,/sY=2.3684/1.223 + ID=expoVpois "Expo V poisson" Ents/All= 20/20 /sX=1.8235/1.5043,/sY=0.90267/0.69807 + ID=varBinning/y "2D Variable Binning" Ents/All= 20/20 /sX=0.93986/2.2384,/sY=-0.89604/0.96206 +SimpleHistos SUCCESS 3D histograms in directory "SimpleHistos" : 3 + ID=3D plot AutoID "3D plot AutoID" Ents/All= 20/20 /sX=-0.23747/1.0922,/sY=0.86883/0.69261,/sZ=0.012149/1.0608 + ID=3d "3D plot ForcedID" Ents/All= 20/20 /sX=-0.23747/1.0922,/sY=0.86883/0.69261,/sZ=0.012149/1.0608 + ID=varBinning/z "3D Variable Binning" Ents/All= 20/20 /sX=0.93986/2.2384,/sY=-0.89604/0.96206,/sZ=0.7512/0.77771 +SimpleHistos SUCCESS 1D profile histograms in directory "SimpleHistos" : 9 + | ID | Title | # | Mean | RMS | Skewness | Kurtosis | + | Expo V Gauss 1DProf | "Expo V Gauss 1DProf" | 20 | -0.21376 | 1.0394 | 0.58116 | -0.93335 | + | Expo V Gauss 1DProf s | "Expo V Gauss 1DProf s" | 20 | -0.21376 | 1.0394 | 0.58116 | -0.93335 | + | Gauss V Flat 1DProf | "Gauss V Flat 1DProf" | 20 | 0.87939 | 6.0918 | 0 | -3 | + | Gauss V Flat 1DProf S | "Gauss V Flat 1DProf S" | 20 | 0.87939 | 6.0918 | 0 | -3 | + | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I" | 9 | -0.29997 | 7.2953 | 0.69467 | -1.6474 | + | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I s" | 9 | -0.29997 | 7.2953 | 0.69467 | -1.6474 | + | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II" | 11 | 1.8443 | 4.6754 | 0 | -3 | + | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II s" | 11 | 1.8443 | 4.6754 | 0 | -3 | + | varBinning/a | "1D Profile Variable Binning" | 20 | 0.93986 | 2.2384 | 0 | -3 | +SimpleHistos SUCCESS 2D profile histograms in directory "SimpleHistos" : 3 + ID=2dprof "2D profile1" Ents/All= 20/20 /sX=-0.23747/1.0922,/sY=0.86883/0.69261 + ID=321 "2D profile2" Ents/All= 20/20 /sX=-0.23747/1.0922,/sY=0.86883/0.69261 + ID=varBinning/b "2D Profile Variable Binning" Ents/All= 20/20 /sX=0.93986/2.2384,/sY=-0.89604/0.96206 +EventLoopMgr INFO Histograms converted successfully according to request. +RootCnvSvc INFO Disconnected data IO:0E366139-BBB0-EA11-B989-FA163E0F4653 [root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade2.ldst] +RootCnvSvc INFO Disconnected data IO:184F683C-BBB0-EA11-ACF1-FA163E0F4653 [root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade3.ldst] +RootCnvSvc INFO Disconnected data IO:90CE1434-BBB0-EA11-81E3-FA163E0F4653 [root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade1.ldst] +ToolSvc INFO Removing all tools created by ToolSvc +DaVinciInitAlg....SUCCESS Booked 4 Histogram(s) : 1D=4 +TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- +TimingAuditor.T... INFO This machine has a speed about 2.86 times the speed of a 2.8 GHz Xeon. +TimingAuditor.T... INFO Algorithm (millisec) | | | min max sigma | entries | total (s) | +TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- +TimingAuditor.T... INFO EVENT LOOP | 1.000 | 1.046 | 0.084 18.4 4.08 | 20 | 0.021 | +TimingAuditor.T... INFO DaVinciInitAlg | 0.000 | 0.146 | 0.033 1.9 0.42 | 20 | 0.003 | +TimingAuditor.T... INFO SimpleHistos | 1.000 | 0.835 | 0.029 15.8 3.52 | 20 | 0.017 | +TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- +DataOnDemandSvc INFO Handled "DataFault" incidents: 0/0/0(Alg/Node/Total). +*****Chrono***** INFO **************************************************************************************************** +*****Chrono***** INFO The Final CPU consumption ( Chrono ) Table (ordered) +*****Chrono***** INFO **************************************************************************************************** +SimpleHistos:Stop INFO Time User : Tot= 0 [us] #= 1 +SimpleHistos:Start INFO Time User : Tot= 0 [us] #= 1 +DaVinciInitAlg:... INFO Time User : Tot= 0 [us] #= 1 +DaVinciInitAlg:... INFO Time User : Tot= 0 [us] #= 1 +SimpleHistos:In... INFO Time User : Tot= 0 [us] #= 1 +DaVinciInitAlg:... INFO Time User : Tot= 0 [us] #= 1 +SimpleHistos:Fi... INFO Time User : Tot= 0 [us] #= 1 +1DOldStyle INFO Time User : Tot= 0 [us] Ave/Min/Max= 0(+- 0)/ 0/ 0 [us] #= 19 +1DForcedNumericID INFO Time User : Tot= 0 [us] Ave/Min/Max= 0(+- 0)/ 0/ 0 [us] #= 19 +1DForcedAlphaID INFO Time User : Tot= 0 [us] Ave/Min/Max= 0(+- 0)/ 0/ 0 [us] #= 19 +1DAutoID INFO Time User : Tot= 0 [us] Ave/Min/Max= 0(+- 0)/ 0/ 0 [us] #= 19 +DaVinciInitAlg:... INFO Time User : Tot= 0 [us] Ave/Min/Max= 0(+- 0)/ 0/ 0 [us] #= 20 +SimpleHistos:Ex... INFO Time User : Tot= 10 [ms] Ave/Min/Max= 0.5(+- 2.18)/ 0/ 10 [ms] #= 20 +DaVinciInitAlg:... INFO Time User : Tot= 30 [ms] #= 1 +ChronoStatSvc INFO Time User : Tot= 0.88 [s] #= 1 +*****Chrono***** INFO **************************************************************************************************** +ChronoStatSvc.f... INFO Service finalized successfully +ApplicationMgr INFO Application Manager Finalized successfully +ApplicationMgr INFO Application Manager Terminated successfully -- GitLab From 9d38be310171e4af53ec1a17a853cfce43ca8a22 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Thu, 3 Dec 2020 18:34:36 +0100 Subject: [PATCH 25/28] fixing test --- .../test_davinci_initialise_upgrade.qmt | 2 +- .../refs/test_davinci_initialise_upgrade.ref | 84 ++++--------------- 2 files changed, 19 insertions(+), 67 deletions(-) diff --git a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt index 14895aa8..19d9a3ae 100755 --- a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt +++ b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt @@ -24,7 +24,7 @@ gaudirun.py DaVinci.ConfigurationUpgrade:mc - $DAVINCITESTSROOT/tests/refs/test_initialise_upgrade + $DAVINCITESTSROOT/tests/refs/test_davinci_initialise_upgrade $DAVINCITESTSROOT/tests/refs/empty.ref from DaVinciTests.QMTest.DaVinciExclusions import preprocessor diff --git a/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref index e05bf7e0..6d97b72a 100644 --- a/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref +++ b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref @@ -1,44 +1,40 @@ -# setting LC_ALL to "C" -# DaVinci INFO Applying DaVinci configuration -ApplicationMgr SUCCESS +ApplicationMgr SUCCESS ==================================================================================================================================== - Welcome to DaVinciDev version HEAD - running on lxplus760.cern.ch on Mon Nov 30 09:17:00 2020 ==================================================================================================================================== ApplicationMgr INFO Application Manager Configured successfully RndmGenSvc.Engine INFO Generator engine type:CLHEP::RanluxEngine RndmGenSvc.Engine INFO Current Seed:1234567 Luxury:3 RndmGenSvc INFO Using Random engine:HepRndm::Engine -TimingAuditor.T... INFO This machine has a speed about 2.86 times the speed of a 2.8 GHz Xeon. DetectorDataSvc INFO Detector description not requested to be loaded RootHistSvc INFO Writing ROOT histograms to: DVHistos.root HistogramPersis... INFO Added successfully Conversion service:RootHistSvc -NTupleSvc DEBUG Property update for OutputLevel : new value = 1 -NTupleSvc DEBUG Service base class initialized successfully NTupleSvc VERBOSE ServiceLocatorHelper::service: found service IncidentSvc NTupleSvc INFO Added stream file:DVNtuple.root as FILE1 DaVinciInitAlg SUCCESS ================================================================== DaVinciInitAlg SUCCESS Requested to process 100 events DaVinciInitAlg SUCCESS ================================================================== EventPersistenc... INFO Added successfully Conversion service:RootCnvSvc -EventSelector INFO Stream:EventSelector.DataStreamTool_1 Def:DATAFILE='root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst' SVC='Gaudi::RootEvtSelector' OPT='READ' ApplicationMgr INFO Application Manager Initialized successfully ApplicationMgr INFO Application Manager Started successfully EventSelector SUCCESS Reading Event record 1. Record number within stream 1: 1 SimpleHistos INFO GaudiHistoAlgorithm:: Filling Histograms...... Please be patient ! -EventSelector INFO Stream:EventSelector.DataStreamTool_2 Def:DATAFILE='root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade1.ldst' SVC='Gaudi::RootEvtSelector' OPT='READ' -EventSelector INFO Stream:EventSelector.DataStreamTool_3 Def:DATAFILE='root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade2.ldst' SVC='Gaudi::RootEvtSelector' OPT='READ' -EventSelector INFO Stream:EventSelector.DataStreamTool_4 Def:DATAFILE='root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade3.ldst' SVC='Gaudi::RootEvtSelector' OPT='READ' -IODataManager INFO Disconnect from dataset root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst [22F2552E-BBB0-EA11-AE30-FA163E0F4653] -RootCnvSvc INFO Removed disconnected IO stream:22F2552E-BBB0-EA11-AE30-FA163E0F4653 [root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst] -EventLoopMgr INFO No more events in event selection +RootCnvSvc INFO Removed disconnected IO stream:00000000-0000-0000-0000-000000000000 [root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst] +EventLoopMgr INFO No more events in event selection ApplicationMgr INFO Application Manager Stopped successfully DaVinciInitAlg SUCCESS ================================================================== DaVinciInitAlg SUCCESS 20 events processed DaVinciInitAlg SUCCESS ================================================================== DaVinciInitAlg WARNING Should have processed 100 events DaVinciInitAlg WARNING ================================================================== -SimpleHistos SUCCESS Booked 30 Histogram(s) : 1D=10 2D=5 3D=3 1DProf=9 2DProf=3 +SimpleHistos SUCCESS 2D histograms in directory "SimpleHistos" : 5 +SimpleHistos SUCCESS 3D histograms in directory "SimpleHistos" : 3 +SimpleHistos SUCCESS 2D profile histograms in directory "SimpleHistos" : 3 +EventLoopMgr INFO Histograms converted successfully according to request. +ToolSvc INFO Removing all tools created by ToolSvc +*****Chrono***** INFO The Final CPU consumption ( Chrono ) Table (ordered) +ChronoStatSvc.f... INFO Service finalized successfully +ApplicationMgr INFO Application Manager Finalized successfully +ApplicationMgr INFO Application Manager Terminated successfully SimpleHistos SUCCESS 1D histograms in directory "SimpleHistos" : 10 | ID | Title | # | Mean | RMS | Skewness | Kurtosis | | 101 | "Exponential" | 20 | 0.86883 | 0.69261 | 0.96379 | 0.55029 | @@ -51,16 +47,6 @@ SimpleHistos SUCCESS 1D histograms in directory "SimpleHistos" : 10 | subdir2/bino | "Binominal" | 20 | 2.25 | 1.299 | 0.055654 | -1.0884 | | test1 | "Forced Alpha ID time test" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | | varBinning/x | "1D Variable Binning" | 20 | 0.93986 | 2.2384 | -0.48081 | -0.51602 | -SimpleHistos SUCCESS 2D histograms in directory "SimpleHistos" : 5 - ID=Exponential V Flat "Exponential V Flat" Ents/All= 18/20 /sX=0.93986/2.2384,/sY=0.7512/0.77771 - ID=Gaussian V Flat "Gaussian V Flat" Ents/All= 20/20 /sX=0.93986/2.2384,/sY=-0.89604/0.96206 - ID=binVpois "Binom V poisson" Ents/All= 20/20 /sX=1.8947/1.4471,/sY=2.3684/1.223 - ID=expoVpois "Expo V poisson" Ents/All= 20/20 /sX=1.8235/1.5043,/sY=0.90267/0.69807 - ID=varBinning/y "2D Variable Binning" Ents/All= 20/20 /sX=0.93986/2.2384,/sY=-0.89604/0.96206 -SimpleHistos SUCCESS 3D histograms in directory "SimpleHistos" : 3 - ID=3D plot AutoID "3D plot AutoID" Ents/All= 20/20 /sX=-0.23747/1.0922,/sY=0.86883/0.69261,/sZ=0.012149/1.0608 - ID=3d "3D plot ForcedID" Ents/All= 20/20 /sX=-0.23747/1.0922,/sY=0.86883/0.69261,/sZ=0.012149/1.0608 - ID=varBinning/z "3D Variable Binning" Ents/All= 20/20 /sX=0.93986/2.2384,/sY=-0.89604/0.96206,/sZ=0.7512/0.77771 SimpleHistos SUCCESS 1D profile histograms in directory "SimpleHistos" : 9 | ID | Title | # | Mean | RMS | Skewness | Kurtosis | | Expo V Gauss 1DProf | "Expo V Gauss 1DProf" | 20 | -0.21376 | 1.0394 | 0.58116 | -0.93335 | @@ -72,44 +58,10 @@ SimpleHistos SUCCESS 1D profile histograms in directory "SimpleHistos" : 9 | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II" | 11 | 1.8443 | 4.6754 | 0 | -3 | | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II s" | 11 | 1.8443 | 4.6754 | 0 | -3 | | varBinning/a | "1D Profile Variable Binning" | 20 | 0.93986 | 2.2384 | 0 | -3 | +SimpleHistos SUCCESS Booked 30 Histogram(s) : 1D=10 2D=5 3D=3 1DProf=9 2DProf=3 +SimpleHistos SUCCESS 1D histograms in directory "SimpleHistos" : 10 +SimpleHistos SUCCESS 2D histograms in directory "SimpleHistos" : 5 +SimpleHistos SUCCESS 3D histograms in directory "SimpleHistos" : 3 +SimpleHistos SUCCESS 1D profile histograms in directory "SimpleHistos" : 9 SimpleHistos SUCCESS 2D profile histograms in directory "SimpleHistos" : 3 - ID=2dprof "2D profile1" Ents/All= 20/20 /sX=-0.23747/1.0922,/sY=0.86883/0.69261 - ID=321 "2D profile2" Ents/All= 20/20 /sX=-0.23747/1.0922,/sY=0.86883/0.69261 - ID=varBinning/b "2D Profile Variable Binning" Ents/All= 20/20 /sX=0.93986/2.2384,/sY=-0.89604/0.96206 -EventLoopMgr INFO Histograms converted successfully according to request. -RootCnvSvc INFO Disconnected data IO:0E366139-BBB0-EA11-B989-FA163E0F4653 [root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade2.ldst] -RootCnvSvc INFO Disconnected data IO:184F683C-BBB0-EA11-ACF1-FA163E0F4653 [root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade3.ldst] -RootCnvSvc INFO Disconnected data IO:90CE1434-BBB0-EA11-81E3-FA163E0F4653 [root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade1.ldst] -ToolSvc INFO Removing all tools created by ToolSvc -DaVinciInitAlg....SUCCESS Booked 4 Histogram(s) : 1D=4 -TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- -TimingAuditor.T... INFO This machine has a speed about 2.86 times the speed of a 2.8 GHz Xeon. -TimingAuditor.T... INFO Algorithm (millisec) | | | min max sigma | entries | total (s) | -TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- -TimingAuditor.T... INFO EVENT LOOP | 1.000 | 1.046 | 0.084 18.4 4.08 | 20 | 0.021 | -TimingAuditor.T... INFO DaVinciInitAlg | 0.000 | 0.146 | 0.033 1.9 0.42 | 20 | 0.003 | -TimingAuditor.T... INFO SimpleHistos | 1.000 | 0.835 | 0.029 15.8 3.52 | 20 | 0.017 | -TimingAuditor.T... INFO -------------------------------------------------------------------------------------------------- -DataOnDemandSvc INFO Handled "DataFault" incidents: 0/0/0(Alg/Node/Total). -*****Chrono***** INFO **************************************************************************************************** -*****Chrono***** INFO The Final CPU consumption ( Chrono ) Table (ordered) -*****Chrono***** INFO **************************************************************************************************** -SimpleHistos:Stop INFO Time User : Tot= 0 [us] #= 1 -SimpleHistos:Start INFO Time User : Tot= 0 [us] #= 1 -DaVinciInitAlg:... INFO Time User : Tot= 0 [us] #= 1 -DaVinciInitAlg:... INFO Time User : Tot= 0 [us] #= 1 -SimpleHistos:In... INFO Time User : Tot= 0 [us] #= 1 -DaVinciInitAlg:... INFO Time User : Tot= 0 [us] #= 1 -SimpleHistos:Fi... INFO Time User : Tot= 0 [us] #= 1 -1DOldStyle INFO Time User : Tot= 0 [us] Ave/Min/Max= 0(+- 0)/ 0/ 0 [us] #= 19 -1DForcedNumericID INFO Time User : Tot= 0 [us] Ave/Min/Max= 0(+- 0)/ 0/ 0 [us] #= 19 -1DForcedAlphaID INFO Time User : Tot= 0 [us] Ave/Min/Max= 0(+- 0)/ 0/ 0 [us] #= 19 -1DAutoID INFO Time User : Tot= 0 [us] Ave/Min/Max= 0(+- 0)/ 0/ 0 [us] #= 19 -DaVinciInitAlg:... INFO Time User : Tot= 0 [us] Ave/Min/Max= 0(+- 0)/ 0/ 0 [us] #= 20 -SimpleHistos:Ex... INFO Time User : Tot= 10 [ms] Ave/Min/Max= 0.5(+- 2.18)/ 0/ 10 [ms] #= 20 -DaVinciInitAlg:... INFO Time User : Tot= 30 [ms] #= 1 -ChronoStatSvc INFO Time User : Tot= 0.88 [s] #= 1 -*****Chrono***** INFO **************************************************************************************************** -ChronoStatSvc.f... INFO Service finalized successfully -ApplicationMgr INFO Application Manager Finalized successfully -ApplicationMgr INFO Application Manager Terminated successfully +DaVinciInitAlg....SUCCESS Booked 4 Histogram(s) : 1D=4 -- GitLab From 326a6ef8027642659b69c53dd7707510a9266402 Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Thu, 3 Dec 2020 18:36:53 +0100 Subject: [PATCH 26/28] fixing typo --- .../qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt index 19d9a3ae..520e8c10 100755 --- a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt +++ b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt @@ -24,7 +24,7 @@ gaudirun.py DaVinci.ConfigurationUpgrade:mc - $DAVINCITESTSROOT/tests/refs/test_davinci_initialise_upgrade + $DAVINCITESTSROOT/tests/refs/test_davinci_initialise_upgrade.ref $DAVINCITESTSROOT/tests/refs/empty.ref from DaVinciTests.QMTest.DaVinciExclusions import preprocessor -- GitLab From baee8be437fbb9839cbe243423c24d26c1bc1e8e Mon Sep 17 00:00:00 2001 From: Davide Fazzini Date: Sat, 5 Dec 2020 18:06:21 +0100 Subject: [PATCH 27/28] fixing reference --- .../refs/test_davinci_initialise_upgrade.ref | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref index 6d97b72a..6f473511 100644 --- a/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref +++ b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref @@ -60,8 +60,32 @@ SimpleHistos SUCCESS 1D profile histograms in directory "SimpleHistos" : 9 | varBinning/a | "1D Profile Variable Binning" | 20 | 0.93986 | 2.2384 | 0 | -3 | SimpleHistos SUCCESS Booked 30 Histogram(s) : 1D=10 2D=5 3D=3 1DProf=9 2DProf=3 SimpleHistos SUCCESS 1D histograms in directory "SimpleHistos" : 10 + | ID | Title | # | Mean | RMS | Skewness | Kurtosis | + | 101 | "Exponential" | 20 | 0.86883 | 0.69261 | 0.96379 | 0.55029 | + | 102 | "Breit" | 20 | -0.03827 | 1.064 | 0.069108 | -0.531 | + | 1111 | "Forced Numeric ID time test" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | + | AutoID time test | "AutoID time test" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | + | Gaussian mean=0, sigma=1 | "Gaussian mean=0, sigma=1" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | + | poisson | "Poisson" | 20 | 1.8947 | 1.4471 | 0.49597 | -1.2241 | + | subdir1/bino | "Binominal" | 20 | 2.25 | 1.299 | 0.055654 | -1.0884 | + | subdir2/bino | "Binominal" | 20 | 2.25 | 1.299 | 0.055654 | -1.0884 | + | test1 | "Forced Alpha ID time test" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | + | varBinning/x | "1D Variable Binning" | 20 | 0.93986 | 2.2384 | -0.48081 | -0.51602 | SimpleHistos SUCCESS 2D histograms in directory "SimpleHistos" : 5 + ID=Exponential V Flat "Exponential V Flat" Ents/All= 18/20 /sX=0.93986/2.2384,/sY=0.7512/0.77771 SimpleHistos SUCCESS 3D histograms in directory "SimpleHistos" : 3 + ID=3D plot AutoID "3D plot AutoID" Ents/All= 20/20 /sX=-0.23747/1.0922,/sY=0.86883/0.69261,/sZ=0.012149/1.0608 SimpleHistos SUCCESS 1D profile histograms in directory "SimpleHistos" : 9 + | ID | Title | # | Mean | RMS | Skewness | Kurtosis | + | Expo V Gauss 1DProf | "Expo V Gauss 1DProf" | 20 | -0.21376 | 1.0394 | 0.58116 | -0.93335 | + | Expo V Gauss 1DProf s | "Expo V Gauss 1DProf s" | 20 | -0.21376 | 1.0394 | 0.58116 | -0.93335 | + | Gauss V Flat 1DProf | "Gauss V Flat 1DProf" | 20 | 0.87939 | 6.0918 | 0 | -3 | + | Gauss V Flat 1DProf S | "Gauss V Flat 1DProf S" | 20 | 0.87939 | 6.0918 | 0 | -3 | + | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I" | 9 | -0.29997 | 7.2953 | 0.69467 | -1.6474 | + | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I s" | 9 | -0.29997 | 7.2953 | 0.69467 | -1.6474 | + | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II" | 11 | 1.8443 | 4.6754 | 0 | -3 | + | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II s" | 11 | 1.8443 | 4.6754 | 0 | -3 | + | varBinning/a | "1D Profile Variable Binning" | 20 | 0.93986 | 2.2384 | 0 | -3 | SimpleHistos SUCCESS 2D profile histograms in directory "SimpleHistos" : 3 + ID=2dprof "2D profile1" Ents/All= 20/20 /sX=-0.23747/1.0922,/sY=0.86883/0.69261 DaVinciInitAlg....SUCCESS Booked 4 Histogram(s) : 1D=4 -- GitLab From 285bc057ad8b34a4545d9ae7ab76b8a94ac3978d Mon Sep 17 00:00:00 2001 From: Sebastien Ponce Date: Mon, 7 Dec 2020 14:59:27 +0100 Subject: [PATCH 28/28] Adapted to latest workaround of the handling of histograms in ref files --- .../refs/test_davinci_initialise_upgrade.ref | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref index 6f473511..517b6da8 100644 --- a/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref +++ b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref @@ -26,9 +26,6 @@ DaVinciInitAlg SUCCESS 20 events processed DaVinciInitAlg SUCCESS ================================================================== DaVinciInitAlg WARNING Should have processed 100 events DaVinciInitAlg WARNING ================================================================== -SimpleHistos SUCCESS 2D histograms in directory "SimpleHistos" : 5 -SimpleHistos SUCCESS 3D histograms in directory "SimpleHistos" : 3 -SimpleHistos SUCCESS 2D profile histograms in directory "SimpleHistos" : 3 EventLoopMgr INFO Histograms converted successfully according to request. ToolSvc INFO Removing all tools created by ToolSvc *****Chrono***** INFO The Final CPU consumption ( Chrono ) Table (ordered) @@ -58,34 +55,3 @@ SimpleHistos SUCCESS 1D profile histograms in directory "SimpleHistos" : 9 | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II" | 11 | 1.8443 | 4.6754 | 0 | -3 | | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II s" | 11 | 1.8443 | 4.6754 | 0 | -3 | | varBinning/a | "1D Profile Variable Binning" | 20 | 0.93986 | 2.2384 | 0 | -3 | -SimpleHistos SUCCESS Booked 30 Histogram(s) : 1D=10 2D=5 3D=3 1DProf=9 2DProf=3 -SimpleHistos SUCCESS 1D histograms in directory "SimpleHistos" : 10 - | ID | Title | # | Mean | RMS | Skewness | Kurtosis | - | 101 | "Exponential" | 20 | 0.86883 | 0.69261 | 0.96379 | 0.55029 | - | 102 | "Breit" | 20 | -0.03827 | 1.064 | 0.069108 | -0.531 | - | 1111 | "Forced Numeric ID time test" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | - | AutoID time test | "AutoID time test" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | - | Gaussian mean=0, sigma=1 | "Gaussian mean=0, sigma=1" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | - | poisson | "Poisson" | 20 | 1.8947 | 1.4471 | 0.49597 | -1.2241 | - | subdir1/bino | "Binominal" | 20 | 2.25 | 1.299 | 0.055654 | -1.0884 | - | subdir2/bino | "Binominal" | 20 | 2.25 | 1.299 | 0.055654 | -1.0884 | - | test1 | "Forced Alpha ID time test" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | - | varBinning/x | "1D Variable Binning" | 20 | 0.93986 | 2.2384 | -0.48081 | -0.51602 | -SimpleHistos SUCCESS 2D histograms in directory "SimpleHistos" : 5 - ID=Exponential V Flat "Exponential V Flat" Ents/All= 18/20 /sX=0.93986/2.2384,/sY=0.7512/0.77771 -SimpleHistos SUCCESS 3D histograms in directory "SimpleHistos" : 3 - ID=3D plot AutoID "3D plot AutoID" Ents/All= 20/20 /sX=-0.23747/1.0922,/sY=0.86883/0.69261,/sZ=0.012149/1.0608 -SimpleHistos SUCCESS 1D profile histograms in directory "SimpleHistos" : 9 - | ID | Title | # | Mean | RMS | Skewness | Kurtosis | - | Expo V Gauss 1DProf | "Expo V Gauss 1DProf" | 20 | -0.21376 | 1.0394 | 0.58116 | -0.93335 | - | Expo V Gauss 1DProf s | "Expo V Gauss 1DProf s" | 20 | -0.21376 | 1.0394 | 0.58116 | -0.93335 | - | Gauss V Flat 1DProf | "Gauss V Flat 1DProf" | 20 | 0.87939 | 6.0918 | 0 | -3 | - | Gauss V Flat 1DProf S | "Gauss V Flat 1DProf S" | 20 | 0.87939 | 6.0918 | 0 | -3 | - | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I" | 9 | -0.29997 | 7.2953 | 0.69467 | -1.6474 | - | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I s" | 9 | -0.29997 | 7.2953 | 0.69467 | -1.6474 | - | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II" | 11 | 1.8443 | 4.6754 | 0 | -3 | - | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II s" | 11 | 1.8443 | 4.6754 | 0 | -3 | - | varBinning/a | "1D Profile Variable Binning" | 20 | 0.93986 | 2.2384 | 0 | -3 | -SimpleHistos SUCCESS 2D profile histograms in directory "SimpleHistos" : 3 - ID=2dprof "2D profile1" Ents/All= 20/20 /sX=-0.23747/1.0922,/sY=0.86883/0.69261 -DaVinciInitAlg....SUCCESS Booked 4 Histogram(s) : 1D=4 -- GitLab