diff --git a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt new file mode 100755 index 0000000000000000000000000000000000000000..520e8c10f94be44828ebad4166cee458a605b3cc --- /dev/null +++ b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt @@ -0,0 +1,33 @@ + + + + + + gaudirun.py + DaVinci.ConfigurationUpgrade:mc + $DAVINCITESTSROOT/tests/refs/test_davinci_initialise_upgrade.ref + $DAVINCITESTSROOT/tests/refs/empty.ref + +from DaVinciTests.QMTest.DaVinciExclusions import preprocessor +validateWithReference(preproc = preprocessor) + + diff --git a/DaVinciTests/tests/refs/empty.ref b/DaVinciTests/tests/refs/empty.ref new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref new file mode 100644 index 0000000000000000000000000000000000000000..517b6da830b6673dfc9dd60b8ebb80b15df4fed9 --- /dev/null +++ b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref @@ -0,0 +1,57 @@ +ApplicationMgr SUCCESS +==================================================================================================================================== +==================================================================================================================================== +ApplicationMgr INFO Application Manager Configured successfully +RndmGenSvc.Engine INFO Generator engine type:CLHEP::RanluxEngine +RndmGenSvc.Engine INFO Current Seed:1234567 Luxury:3 +RndmGenSvc INFO Using Random engine:HepRndm::Engine +DetectorDataSvc INFO Detector description not requested to be loaded +RootHistSvc INFO Writing ROOT histograms to: DVHistos.root +HistogramPersis... INFO Added successfully Conversion service:RootHistSvc +NTupleSvc VERBOSE ServiceLocatorHelper::service: found service IncidentSvc +NTupleSvc INFO Added stream file:DVNtuple.root as FILE1 +DaVinciInitAlg SUCCESS ================================================================== +DaVinciInitAlg SUCCESS Requested to process 100 events +DaVinciInitAlg SUCCESS ================================================================== +EventPersistenc... INFO Added successfully Conversion service:RootCnvSvc +ApplicationMgr INFO Application Manager Initialized successfully +ApplicationMgr INFO Application Manager Started successfully +EventSelector SUCCESS Reading Event record 1. Record number within stream 1: 1 +SimpleHistos INFO GaudiHistoAlgorithm:: Filling Histograms...... Please be patient ! +RootCnvSvc INFO Removed disconnected IO stream:00000000-0000-0000-0000-000000000000 [root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst] +EventLoopMgr INFO No more events in event selection +ApplicationMgr INFO Application Manager Stopped successfully +DaVinciInitAlg SUCCESS ================================================================== +DaVinciInitAlg SUCCESS 20 events processed +DaVinciInitAlg SUCCESS ================================================================== +DaVinciInitAlg WARNING Should have processed 100 events +DaVinciInitAlg WARNING ================================================================== +EventLoopMgr INFO Histograms converted successfully according to request. +ToolSvc INFO Removing all tools created by ToolSvc +*****Chrono***** INFO The Final CPU consumption ( Chrono ) Table (ordered) +ChronoStatSvc.f... INFO Service finalized successfully +ApplicationMgr INFO Application Manager Finalized successfully +ApplicationMgr INFO Application Manager Terminated successfully +SimpleHistos SUCCESS 1D histograms in directory "SimpleHistos" : 10 + | ID | Title | # | Mean | RMS | Skewness | Kurtosis | + | 101 | "Exponential" | 20 | 0.86883 | 0.69261 | 0.96379 | 0.55029 | + | 102 | "Breit" | 20 | -0.03827 | 1.064 | 0.069108 | -0.531 | + | 1111 | "Forced Numeric ID time test" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | + | AutoID time test | "AutoID time test" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | + | Gaussian mean=0, sigma=1 | "Gaussian mean=0, sigma=1" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | + | poisson | "Poisson" | 20 | 1.8947 | 1.4471 | 0.49597 | -1.2241 | + | subdir1/bino | "Binominal" | 20 | 2.25 | 1.299 | 0.055654 | -1.0884 | + | subdir2/bino | "Binominal" | 20 | 2.25 | 1.299 | 0.055654 | -1.0884 | + | test1 | "Forced Alpha ID time test" | 20 | -0.21376 | 1.0394 | -0.069985 | -0.90535 | + | varBinning/x | "1D Variable Binning" | 20 | 0.93986 | 2.2384 | -0.48081 | -0.51602 | +SimpleHistos SUCCESS 1D profile histograms in directory "SimpleHistos" : 9 + | ID | Title | # | Mean | RMS | Skewness | Kurtosis | + | Expo V Gauss 1DProf | "Expo V Gauss 1DProf" | 20 | -0.21376 | 1.0394 | 0.58116 | -0.93335 | + | Expo V Gauss 1DProf s | "Expo V Gauss 1DProf s" | 20 | -0.21376 | 1.0394 | 0.58116 | -0.93335 | + | Gauss V Flat 1DProf | "Gauss V Flat 1DProf" | 20 | 0.87939 | 6.0918 | 0 | -3 | + | Gauss V Flat 1DProf S | "Gauss V Flat 1DProf S" | 20 | 0.87939 | 6.0918 | 0 | -3 | + | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I" | 9 | -0.29997 | 7.2953 | 0.69467 | -1.6474 | + | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I s" | 9 | -0.29997 | 7.2953 | 0.69467 | -1.6474 | + | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II" | 11 | 1.8443 | 4.6754 | 0 | -3 | + | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II s" | 11 | 1.8443 | 4.6754 | 0 | -3 | + | varBinning/a | "1D Profile Variable Binning" | 20 | 0.93986 | 2.2384 | 0 | -3 | diff --git a/Phys/DaVinci/options/DVconfig-Default.yaml b/Phys/DaVinci/options/DVconfig-Default.yaml new file mode 100644 index 0000000000000000000000000000000000000000..838039323ff66c0fcb696ca8152941e4ec37068a --- /dev/null +++ b/Phys/DaVinci/options/DVconfig-Default.yaml @@ -0,0 +1,101 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +# Application Configuration +EvtMax: + text: '""" Number of events to analyse """' + value: -1 +SkipEvents: + text: '""" Number of events to skip at beginning for file """' + value: 0 +PrintFreq: + text: '""" The frequency at which to print event numbers """' + value: 1000 +DataType: + text: '""" Data type, can be ["Upgrade"] Forwarded to PhysConf, AnalysisConf, DstConf and LHCbApp """' + value: 'Upgrade' +Simulation: + text: '""" set to True to use SimCond. Forwarded to PhysConf """' + value: False +DDDBtag: + text: '""" Data type, can be ["Upgrade"] Forwarded to PhysConf, AnalysisConf, DstConf and LHCbApp """' + value: 'dddb-20171126' +CondDBtag: + text: '""" Tag for CondDB. Default as set in DDDBConf for DataType """' + value: 'sim-20171127-vc-md100' +DQFLAGStag: + text: '""" Tag for DQFLAGS. Default as set in DDDBConf for DataType """' + value: '' +OutputLevel: + text: '""" Set output level used in the job """' + value: 3 +# Input +Input: + text: '""" Input data. Can also be passed as a second option file. """' + value: [] +InputType: + text: '""" "DST", "DIGI", "RDST", "MDST", "XDST" or "LDST". Nothing means the input type is compatible with being a DST """' + value: 'DST' +EnableUnpack: + text: '""" Explicitly enable/disable unpacking for input data (if specified) """' + value: None +OverwriteDataOptions: + text: '""" Allow overwriting default data options defined in DaVinci Database. """' + value: False +# Output +HistogramFile: + text: '""" Write name of output Histogram file """' + value: '' +TupleFile: + text: '""" Write name of output Tuple file """' + value: '' +WriteFSR: + text: '""" Flags whether to write out an FSR """' + value: True +MergeGenFSR: + text: '""" Flags whether to merge the generatore level FSRs """' + value: False +RootCompressionLevel: + text: '""" ROOT Compression level for ntuples """' + value: 'LZMA:6' +# DQ +IgnoreDQFlags: + text: '""" If False, process only events with good DQ. Default is False """' + value: False +# DaVinci Options +MainOptions: + text: '""" Main option file to execute """' + value: '' +UserAlgorithms: + text: '""" User algorithms to run. """' + value: [] +RedoMCLinks: + text: '""" On some stripped DST one needs to redo the Track<->MC link table. Set to true if problems with association. """' + value: False +Lumi: + text: '""" Run event count and Lumi accounting (should normally be True) """' + value: False +EventPreFilters: + text: '""" Set of event filtering algorithms to be run before DaVinci initializaton sequence. Only events passing these filters will be processed. """' + value: [] +VerboseMessages: + text: '""" Enable additional verbose printouts """' + value: False +RootInTES: + text: '""" RootInTES (for uDst input type) """' + value: '' +Detectors: + text: '""" List of detectors """' + value: ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr'] +# Log +Log: + text: '""" Logger used to print warning and error messages """' + value: '' \ No newline at end of file diff --git a/Phys/DaVinci/options/DaVinciDB-Example.yaml b/Phys/DaVinci/options/DaVinciDB-Example.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b0463e88e7820397abd3bcaa8e79b12fb208229 --- /dev/null +++ b/Phys/DaVinci/options/DaVinciDB-Example.yaml @@ -0,0 +1,45 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +Upgrade_genFSR_ldst: + filenames: + - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade0.ldst' + - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade1.ldst' + - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade2.ldst' + - 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade3.ldst' + qualifiers: + DataType: Upgrade + InputType: LDST + Simulation: true + CondDBtag: sim-20171127-vc-md100 + DDDBtag: dddb-20171126 + metadata: + Author: 'dfazzini' + Date: '2020-06-17 18:34:43.57756' + Comment: 'Files .ldst with Upgrade condition for testing the propagation and merging of genFSR in DaVinci' + +Upgrade_Bd2KstarMuMu_ldst: + filenames: + - 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000002_1.ldst' + - 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000004_1.ldst' + - 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000043_1.ldst' + - 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000068_1.ldst' + qualifiers: + DataType: Upgrade + InputType: LDST + Simulation: true + CondDBtag: sim-20171127-vc-md100 + DDDBtag: dddb-20171126 + metadata: + Author: 'Patrick Koppenburg' + Date: '2020-05-28 11:12:55' + Comment: 'K*mumu by Moore Hlt1-filtered with TCK-0x52000000' + \ No newline at end of file diff --git a/Phys/DaVinci/options/jobOptions-Example.py b/Phys/DaVinci/options/jobOptions-Example.py new file mode 100644 index 0000000000000000000000000000000000000000..6d429177906da08b010530f835626e97d8db8a25 --- /dev/null +++ b/Phys/DaVinci/options/jobOptions-Example.py @@ -0,0 +1,17 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +{ + 'EvtMax': 100, + 'SkipEvents': 2, + 'TupleFile': 'DVNtuple.root', + 'HistogramFile': 'DVHistos.root' +} diff --git a/Phys/DaVinci/options/jobOptions-Example.yaml b/Phys/DaVinci/options/jobOptions-Example.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aaf8e6812256c62280e54454c0335c46c217ef1a --- /dev/null +++ b/Phys/DaVinci/options/jobOptions-Example.yaml @@ -0,0 +1,15 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +EvtMax: 100 +SkipEvents: 2 +TupleFile: 'DVNtuple.root' +HistogramFile: 'DVHistos.root' diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py new file mode 100644 index 0000000000000000000000000000000000000000..b339afd823621a77c7c806d4771c56d7bc24d628 --- /dev/null +++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py @@ -0,0 +1,66 @@ +############################################################################## +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +""" +High level configuration tools for DaVinci +""" +from GaudiConfig2 import Configurables as C +from GaudiConfig2 import mergeConfigs + +from DaVinci.configOptions import initializeSlots, checkOptions, setFileOptions, setJobOptions +from DaVinci.algorithms import dvInit, defineInput, setupAlgorithms, setupExtSvc, rootFiles, defineMonitors, defineLog +from DaVinci.configurations import configureAppMgr + +def mc(key='Upgrade_genFSR_ldst', dbFile='$DAVINCIROOT/options/DaVinciDB-Example.yaml', jobOptFile='$DAVINCIROOT/options/jobOptions-Example.yaml'): + return main(key, dbFile, jobOptFile, True) + +def data(key='', dbFile='', jobOptFile=''): + raise ValueError('Data file with upgrade conditions are not yet available. Please use :mc function instead.') + return main(key, dbFile, jobOptFile, False) + +def main(key, dbFile, jobOptFile, isMC): + """ + DaVinci application main configuration. + """ + slots = {} + propertyDocDct = {} + initializeSlots(slots, propertyDocDct) + + defineLog(slots) + + setFileOptions(slots, key, dbFile, isMC) + setJobOptions(slots, jobOptFile, key, dbFile) + + checkOptions(slots) + + defineInput(slots) + + config = [] + appMgr = C.ApplicationMgr() + config.append(appMgr) + + dvInit(config, slots) + + defineMonitors(config, slots) + + setupAlgorithms(config, slots) + + setupExtSvc(config, slots) + + rootFiles(config, slots) + + configureAppMgr(appMgr, slots, config) + + config = mergeConfigs(config) + return config + + + + diff --git a/Phys/DaVinci/python/DaVinci/algorithms.py b/Phys/DaVinci/python/DaVinci/algorithms.py new file mode 100644 index 0000000000000000000000000000000000000000..75cf926690784ffcbd08661fffb6a28f553868d5 --- /dev/null +++ b/Phys/DaVinci/python/DaVinci/algorithms.py @@ -0,0 +1,116 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +from GaudiConfig2 import Configurables as C +from DaVinci.configOptions import getSlotValue, setSlotValue +from DaVinci.configurations import * + +def dvInit(config, slots): + """ + Set initialisation algorithm + """ + dvInit = C.DaVinciInit('DaVinciInitAlg') + configureInit(dvInit, slots) + config.append(dvInit) + +def defineInput(slots): + """ + Set input files + """ + input = getSlotValue(slots, "Input") + + if ( len(input) > 0 ): + from GaudiConf import IOHelper + + persistency = None + inputType = getSlotValue(slots, "InputType").upper() + if inputType == "MDF" : persistency = "MDF" + #support connection strings and lists of files + input = IOHelper(persistency, persistency).convertConnectionStrings(input, "I") + #clear selector to maintain the same behaviour + IOHelper(persistency, persistency).inputFiles(input, clear = True) + else: + log = getSlotValue(slots, "Log") + log.warning('No input files available along with the selected key. Check the related DaVinciDB.') + +def defineMonitors(config, slots): + """ + Set monitors + """ + auditor = C.AuditorSvc() + timer = C.TimingAuditor() + sequencer = C.SequencerTimerTool() + configureTiming(auditor, timer, sequencer) + + evtSel = C.EventSelector() + configureEventSelector(evtSel, slots) + + config.extend([auditor, timer, sequencer, evtSel]) + +def setupAlgorithms(config, slots): + """ + Set DaVinci algorithms + """ + algorithms = [C.GaudiHistoAlgorithm('SimpleHistos', + HistoPrint=True, + OutputLevel=getSlotValue(slots, "OutputLevel"))] + + config.extend(algorithms) + +def setupExtSvc(config, slots): + """ + Set DaVinci services + """ + lokiSvc = C.LoKiSvc() + dataSvc = C.DataOnDemandSvc() + configurePrintOuts(lokiSvc, dataSvc, slots) + + evtPers = C.EvtPersistencySvc("EventPersistencySvc") + configureEventPersistency(evtPers) + + extSvc = [lokiSvc, + dataSvc, + evtPers, + C.ToolSvc(), + C.DetDataSvc('DetectorDataSvc'), + C.Gaudi.MultiFileCatalog('FileCatalog'), + C.Gaudi.IODataManager('IODataManager'), + C.Gaudi.RootCnvSvc('FileRecordCnvSvc'), + ] + + config.extend(extSvc) + +def rootFiles(config, slots): + """ + Set output .root files + """ + if getSlotValue(slots, "HistogramFile"): + histoSvc = C.HistogramPersistencySvc(OutputFile = getSlotValue(slots, "HistogramFile")) + config.append(histoSvc) + + if getSlotValue(slots, "TupleFile"): + ntSvc = C.NTupleSvc() + configureOptions_rootNtuples(ntSvc, slots) + config.append(ntSvc) + + # Set the compression level for the ROOT tuple file + from GaudiKernel.Configurable import ConfigurableGeneric as RFileCnv + RFileCnv('RFileCnv').GlobalCompression = getSlotValue(slots, "RootCompressionLevel") + +def defineLog(slots): + """ + Set logger + """ + from AnalysisPython.Logger import getLogger + log = getLogger ( 'DaVinci' ) + log.info("Applying DaVinci configuration") + setSlotValue(slots, "Log", log) + diff --git a/Phys/DaVinci/python/DaVinci/configOptions.py b/Phys/DaVinci/python/DaVinci/configOptions.py new file mode 100644 index 0000000000000000000000000000000000000000..0f1b337b5cab8f09f651845f7f2da5741d095e89 --- /dev/null +++ b/Phys/DaVinci/python/DaVinci/configOptions.py @@ -0,0 +1,186 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +import os + +################################################################################ +# Manage slots properties and input job and data options +# +""" +Set and retrieve the value of a specific slot property +""" +def getSlotValue(slots, name): + return slots[name] + +def setSlotValue(slots, name, value): + if name in slots: + slots[name] = value + else: + import difflib + raise ValueError('Unknown qualifier called %s. Please check the DaVinci database or the job option file!\n ' + 'List of possible valid qualifiers: %s' %(name, difflib.get_close_matches(name, slots))) + +def getDefaultValue(myKey): + """ + Get default value for a specific property + """ + import yaml + optsDefaultName = "$DAVINCIROOT/options/DVconfig-Default.yaml" + with open(os.path.expandvars(optsDefaultName)) as optsDefault: + config = yaml.safe_load(optsDefault) + + for key, args in config.items(): + if key == myKey: + for name, value in args.items(): + if name == 'value': + return value + return 0 + +def initializeSlots(slots, propDct): + """ + Initialize properties with default values. + """ + import yaml + optsDefaultName = "$DAVINCIROOT/options/DVconfig-Default.yaml" + with open(os.path.expandvars(optsDefaultName)) as optsDefault: + config = yaml.safe_load(optsDefault) + + for key, args in config.items(): + for name, value in args.items(): + if name == 'value': + slots[key] = value + else: + propDct[key] = value + +def setFileOptions(slots, myKey, dbName, isMC): + """ + Set the dataset properties required by the user + """ + import yaml + with open(os.path.expandvars(dbName)) as dbFile: + dataDV = yaml.safe_load(dbFile) + + idxFile = -1 + if ":" in myKey: + idxFile = int(myKey.split(":")[1]) + myKey = myKey.split(":")[0] + + for key, config in dataDV.items(): + if key == myKey: + for item, obj in config.items(): + if item == 'qualifiers': + for prop, value in obj.items(): + setSlotValue(slots, prop, value) + elif item == 'filenames': + if idxFile > -1: + if idxFile < len(obj): + setSlotValue(slots, "Input", [obj[idxFile]]) + else: + raise ValueError('Index file requested exceeds the number of available files related to the given key!') + else: + setSlotValue(slots, "Input", obj) + +def setJobOptions(slots, configName, myKey, dbName): + """ + Set the job properties required by the user + """ + log = getSlotValue(slots, "Log") + if configName == '': + log.warning('No jobOption file selected, the default values are used.') + else: + dataOptions = listDataOptions(myKey.split(":")[0], dbName) + + with open(os.path.expandvars(configName)) as config_file: + """ + if configName.endswith('.json'): + import json + config = json.load(config_file) + elif configName.endswith('.yaml') or configName.endswith('.yml'): + """ + _, ext = os.path.splitext(configName) + if ext in (".yaml", ".yml", ".json"): + import yaml + config = yaml.safe_load(config_file) + elif configName.endswith('.py'): + import ast + config = ast.literal_eval(config_file.read()) + else: + raise ValueError('JobOption file extension not known! Please use only a .py, .json or .yaml (.yml) file!') + + for key, value in config.items(): + if key in dataOptions: + if getSlotValue(slots, "OverwriteDataOptions"): + log.warning('Default value for option %s is taken from DaVinciDB but a new value is found in option file. ' + 'Since "OverwriteDataOptions" is active the new value is used. Make sure this is the configuration you want to use for this job.' %key) + setSlotValue(slots, key, value) + else: + log.warning('Default value for option %s is taken from DaVinciDB but a new value is found in option file. ' + 'Since "OverwriteDataOptions" is not active the default value is used. Make sure this is the configuration you want to use for this job.' %key) + else: + setSlotValue(slots, key, value) + +def listDataOptions(myKey, dbName): + """ + List of the properties that are set automatically given a dataset + """ + import yaml + + with open(os.path.expandvars(dbName)) as dbFile: + dataDV = yaml.safe_load(dbFile) + + optionList = [] + for key, config in dataDV.items(): + if key == myKey: + for item, obj in config.items(): + if item == 'qualifiers': + for prop, value in obj.items(): + optionList.append(prop) + break + return optionList + + +def checkOptions(slots): + """ + Checks options. Changes a few if needed. + """ + log = getSlotValue(slots, "Log") + known_datatypes = ["Upgrade"] + known_default_detectors = ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ] + + dataType = getSlotValue(slots,"DataType") + if (not dataType): + raise TypeError( "You must set DataType" ) + if dataType not in known_datatypes: + raise TypeError( "Invalid DataType '%s'" %dataType ) + + inputType = getSlotValue(slots, "InputType").upper() + if inputType not in [ "MDF", "DST", "DIGI", "RDST", "MDST", "SDST", "XDST", "LDST" ]: + raise TypeError( "Invalid inputType '%s'"%inputType ) + + if getSlotValue(slots, "Simulation") and not inputType in ( "MDF" , "DIGI" , "MDST" ): + redo = getSlotValue(slots, "RedoMCLinks") + if (inputType == "RDST") and (redo): + log.warning("Re-doing MC links not possible for RDST") + setSlotValue(slots, "RedoMCLinks", False) + + if getSlotValue(slots, "Simulation") and getSlotValue(slots,'Lumi'): + log.warning('Lumi not valid for Simulation. Setting Lumi = False') + setSlotValue(slots, 'Lumi', False) + + ## for simulation, it is very important to specify proper DB-tags: + if getSlotValue(slots, 'Simulation'): + if not getSlotValue(slots,'DDDBtag'): + log.warning("``DDDBtag'' is not specified for simulated data") + if not getSlotValue(slots, 'CondDBtag'): + log.warning("``CondDBtag'' is not specified for simulated data") + + if getSlotValue(slots, "MergeGenFSR") and not getSlotValue(slots, "Simulation"): + raise TypeError("Cannot MergeGenFSR on real data"); diff --git a/Phys/DaVinci/python/DaVinci/configurations.py b/Phys/DaVinci/python/DaVinci/configurations.py new file mode 100644 index 0000000000000000000000000000000000000000..4b323db0a7ba1ffde60c8b5cab3ef1843639dd6e --- /dev/null +++ b/Phys/DaVinci/python/DaVinci/configurations.py @@ -0,0 +1,95 @@ +############################################################################### +# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +from GaudiConfig2 import Configurables as C +from DaVinci.configOptions import getSlotValue + +################################################################################ +# Define configurations for applications and algorithms +# + +def configureAppMgr(appMgr, slots, config): + """ + Configuration of the main application + """ + messageSvc = 0 + for c in config: + if c.name == 'MessageSvc': + messageSvc = c + + appMgr.EvtMax = getSlotValue(slots,"EvtMax") + appMgr.HistogramPersistency = 'ROOT' + appMgr.AuditAlgorithms = True + + appMgr.ExtSvc = [c for c in config + if c.__component_type__ == 'Service' + and c.name not in ['ApplicationMgr']] + appMgr.TopAlg = [c for c in config + if c.__component_type__ == 'Algorithm' + and c.name not in ['ApplicationMgr']] + +def configureInit(dvInit, slots): + """ + Configuration of initialisation algorithm + """ + dvInit.Increment = getSlotValue(slots, "PrintFreq") + + +################################################################################ +# Define configurations for external services +# + +def configureTiming(auditor, timer, sequencer): + """ + Configuration of timing auditors + """ + auditor.Auditors = ['ChronoAuditor', 'TimingAuditor'] + sequencer.OutputLevel = 4 + +def configureOptions_rootNtuples(svc, slots): + """ + Configuration of output .root ntuples + """ + for _line in svc.Output : + if 0 <= _line.find ('FILE1') : + log = getSlotValue(slots, "Log") + log.warning ('Replace NTuple-LUN FILE1: ' + _line ) + svc.Output.remove ( _line ) + tupleFile = getSlotValue(slots, "TupleFile") + tupleStr = "FILE1 DATAFILE='%s' TYP='ROOT' OPT='NEW'" % tupleFile + svc.Output += [ tupleStr ] + svc.OutputLevel = 1 + +def configureEventPersistency(svc): + """ + Configuration of the event persistency service + """ + svc.CnvServices.append("Gaudi::RootCnvSvc/RootCnvSvc") + +def configurePrintOuts(lokiSvc, dataSvc, slots): + """ + Configuration of the printout services + """ + verbosePrint = getSlotValue(slots, "VerboseMessages") + lokiSvc.Welcome = verbosePrint + dataSvc.Dump = verbosePrint + +def configureEventSelector(svc, slots): + """ + Configuration of the event selector + """ + printFreq = getSlotValue(slots, "PrintFreq") + if printFreq == 0: + printFreq = getDefaultValue("PrintFreq") + log = getSlotValue(slots, "Log") + log.warning("Print frequency cannot be 0. Set to %f." %printFreq) + svc.PrintFreq = printFreq +