From 28aa6a135b534e7cc3409990cd8b1862721e39c8 Mon Sep 17 00:00:00 2001
From: Davide Fazzini <davide.fazzini@cern.ch>
Date: Thu, 25 Feb 2021 08:55:37 +0000
Subject: [PATCH] Fixed formatting
patch generated by https://gitlab.cern.ch/lhcb/DaVinci/-/jobs/12217080
---
Phys/DaVinci/options/DVconfig-Default.yaml | 15 +-
.../options/DaVinci-UserAlgs-Example.py | 40 ++++
Phys/DaVinci/options/DaVinciDB-Example.yaml | 2 +-
Phys/DaVinci/options/jobOptions-Example.py | 2 +-
Phys/DaVinci/options/jobOptions-Example.yaml | 2 +-
.../python/DaVinci/ConfigurationUpgrade.py | 108 ++++++++--
Phys/DaVinci/python/DaVinci/algorithms.py | 173 +++++++++++-----
Phys/DaVinci/python/DaVinci/configOptions.py | 187 +++++++++++-------
Phys/DaVinci/python/DaVinci/configurations.py | 135 +++++++++++--
Phys/DaVinci/python/DaVinci/optionChecker.py | 107 ++++++++++
Phys/DaVinci/python/DaVinci/utilities.py | 88 +++++++++
11 files changed, 697 insertions(+), 162 deletions(-)
create mode 100644 Phys/DaVinci/options/DaVinci-UserAlgs-Example.py
create mode 100644 Phys/DaVinci/python/DaVinci/optionChecker.py
create mode 100644 Phys/DaVinci/python/DaVinci/utilities.py
diff --git a/Phys/DaVinci/options/DVconfig-Default.yaml b/Phys/DaVinci/options/DVconfig-Default.yaml
index 838039323..403cfcb9f 100644
--- a/Phys/DaVinci/options/DVconfig-Default.yaml
+++ b/Phys/DaVinci/options/DVconfig-Default.yaml
@@ -1,5 +1,5 @@
###############################################################################
-# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration #
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
@@ -61,8 +61,8 @@ WriteFSR:
text: '""" Flags whether to write out an FSR """'
value: True
MergeGenFSR:
- text: '""" Flags whether to merge the generatore level FSRs """'
- value: False
+ text: '""" Flags whether to merge the generator-level FSRs """'
+ value: True
RootCompressionLevel:
text: '""" ROOT Compression level for ntuples """'
value: 'LZMA:6'
@@ -74,9 +74,12 @@ IgnoreDQFlags:
MainOptions:
text: '""" Main option file to execute """'
value: ''
-UserAlgorithms:
- text: '""" User algorithms to run. """'
- value: []
+UserAlgorithmFile:
+ text: '""" User algorithms file to be processed without .py extension. """'
+ value: '$DAVINCIROOT/options/DaVinci-UserAlgs-Example'
+UserAlgorithm:
+ text: '""" Main user algorithms function to run. """'
+ value: 'myfunc'
RedoMCLinks:
text: '""" On some stripped DST one needs to redo the Track<->MC link table. Set to true if problems with association. """'
value: False
diff --git a/Phys/DaVinci/options/DaVinci-UserAlgs-Example.py b/Phys/DaVinci/options/DaVinci-UserAlgs-Example.py
new file mode 100644
index 000000000..4bab731b3
--- /dev/null
+++ b/Phys/DaVinci/options/DaVinci-UserAlgs-Example.py
@@ -0,0 +1,40 @@
+###############################################################################
+# (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+# #
+###############################################################################
+"""
+ Simple DaVinci example for defining user algorithms
+"""
+__author__ = "Davide Fazzini <davide.fazzini@cern.ch>"
+
+##############################################################################################
+##### SETTINGS
+##############################################################################################
+
+from GaudiConfig2 import Configurables as C
+from PhysConf.Selections import AutomaticData
+
+
+def myfunc():
+ sel = AutomaticData('/Phys/BetaSBd2JpsiKstarDetachedLine/Particles')
+ inputTuple = sel.outputLocation()
+
+ # make a tuple
+ dtt = C.DecayTreeTuple("Bd2JpsiKstar")
+ dtt.Inputs = [inputTuple]
+ dtt.ToolList = ['TupleToolKinematic']
+ dtt.Decay = "[B0 -> ^(J/psi(1S) -> ^mu+ ^mu-) ^(K*(892)0 -> ^K+ ^pi-)]CC"
+
+ evtTuple = C.EventTuple()
+ evtTuple.ToolList += ["TupleToolEventInfo"]
+
+ #listAlg = [evtTuple, dtt]
+ listAlg = []
+ return listAlg
diff --git a/Phys/DaVinci/options/DaVinciDB-Example.yaml b/Phys/DaVinci/options/DaVinciDB-Example.yaml
index 4f360ba5a..85bd9e4d7 100644
--- a/Phys/DaVinci/options/DaVinciDB-Example.yaml
+++ b/Phys/DaVinci/options/DaVinciDB-Example.yaml
@@ -1,5 +1,5 @@
###############################################################################
-# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration #
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
diff --git a/Phys/DaVinci/options/jobOptions-Example.py b/Phys/DaVinci/options/jobOptions-Example.py
index 6d4291779..3d32fecd1 100644
--- a/Phys/DaVinci/options/jobOptions-Example.py
+++ b/Phys/DaVinci/options/jobOptions-Example.py
@@ -1,5 +1,5 @@
###############################################################################
-# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration #
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
diff --git a/Phys/DaVinci/options/jobOptions-Example.yaml b/Phys/DaVinci/options/jobOptions-Example.yaml
index aaf8e6812..0c7b6afa6 100644
--- a/Phys/DaVinci/options/jobOptions-Example.yaml
+++ b/Phys/DaVinci/options/jobOptions-Example.yaml
@@ -1,5 +1,5 @@
###############################################################################
-# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration #
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py
index a0a95bdec..8d3e81f08 100644
--- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py
+++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py
@@ -11,26 +11,93 @@
"""
High level configuration tools for DaVinci.
"""
+
from GaudiConfig2 import Configurables as C
from GaudiConfig2 import mergeConfigs
-from DaVinci.configOptions import initialize_slots, check_options, set_file_options, set_job_options
-from DaVinci.algorithms import define_input, define_monitors, define_log, define_root_files, define_services
-from DaVinci.algorithms import define_dv_init_algo, setupAlgorithms
+from DaVinci.configOptions import initialize_slots, check_options, set_file_options, set_job_options, set_args_options
+from DaVinci.algorithms import (define_input, define_monitors, define_log,
+ define_root_files, define_services,
+ define_dv_init_algo, setup_algorithms,
+ define_genfsr_writer)
from DaVinci.configurations import configure_app_mgr
+from DaVinci.utilities import dump_call, dump_configuration
+
+import click
+#import os
+#os.environ["LC_ALL"] = "en_GB.utf8"
+
+
+@click.option(
+ "--key",
+ default="Upgrade_Bd2KstarMuMu_ldst",
+ help="Key name related to the input files listed in the TestFileDB")
+@click.option(
+ "--dbfile",
+ default="$DAVINCIROOT/options/DaVinciDB-Example.yaml",
+ help="YAML file database containing input file infos")
+@click.option(
+ "--optfile",
+ default="$DAVINCIROOT/options/jobOptions-Example.yaml",
+ help="YAML option file containing the job infos")
+@click.command(
+ context_settings=dict(
+ ignore_unknown_options=True,
+ allow_extra_args=True,
+ ))
+@click.pass_context
+def cli(ctx, key, dbfile, optfile):
+ pass
+
+
+def mc(key="Upgrade_Bd2KstarMuMu_ldst",
+ dbfile="$DAVINCIROOT/options/DaVinciDB-Example.yaml",
+ optfile="$DAVINCIROOT/options/jobOptions-Example.yaml"):
+ """
+ DaVinci function for running job with simulated samples
+ """
+ #dump_call(ctx, key, dbfile, optfile)
+ kwargs = ""
+ #if (len(ctx.args) > 1):
+ # kwargs = ctx.args
+
+ return main(key, dbfile, optfile, True, kwargs)
+
+
+@click.option(
+ "--key",
+ default="Upgrade_Bd2KstarMuMu_ldst",
+ help="Key name related to the input files listed in the TestFileDB")
+@click.option(
+ "--dbfile",
+ default="$DAVINCIROOT/options/DaVinciDB-Example.yaml",
+ help="YAML file database containing input file infos")
+@click.option(
+ "--optfile",
+ default="$DAVINCIROOT/options/jobOptions-Example.yaml",
+ help="YAML option file containing the job infos")
+@click.command(
+ context_settings=dict(
+ ignore_unknown_options=True,
+ allow_extra_args=True,
+ ))
+@click.pass_context
+def data(ctx, key, dbfile, optfile):
+ """
+ DaVinci function for running job with data samples
+ """
+ raise ValueError(
+ 'Data file with upgrade conditions are not yet available. Please use :mc function instead.'
+ )
+ dump_call(ctx, key, dbfile, optfile)
+ kwargs = ""
+ if (len(ctx.args) > 1):
+ kwargs = ctx.args
+ return main(key, dbfile, optfile, False, kwargs)
-def mc(key='Upgrade_genFSR_ldst', dbFile='$DAVINCIROOT/options/DaVinciDB-Example.yaml', jobOptFile='$DAVINCIROOT/options/jobOptions-Example.yaml'):
- return main(key, dbFile, jobOptFile, True)
-
-
-def data(key='', dbFile='', jobOptFile=''):
- raise ValueError('Data file with upgrade conditions are not yet available. Please use :mc function instead.')
- return main(key, dbFile, jobOptFile, False)
-
-
-def main(key, dbFile, jobOptFile, isMC):
- """
+def main(key, dbFile, jobOptFile, isMC, kwargs):
+ """
DaVinci application main configuration.
"""
slots = {}
@@ -41,9 +108,12 @@ def main(key, dbFile, jobOptFile, isMC):
set_file_options(slots, key, dbFile, isMC)
set_job_options(slots, jobOptFile, key, dbFile)
+ if kwargs:
+ set_args_options(slots, kwargs, key, dbFile)
- check_options(slots)
+ dump_configuration(slots)
+ check_options(slots)
define_input(slots)
config = []
@@ -51,16 +121,18 @@ def main(key, dbFile, jobOptFile, isMC):
config.append(appMgr)
define_dv_init_algo(config, slots)
-
+
define_monitors(config, slots)
- setupAlgorithms(config, slots)
+ setup_algorithms(config, slots)
define_services(config, slots)
define_root_files(config, slots)
- configure_app_mgr(appMgr, slots, config)
+ define_genfsr_writer(config, slots)
+
+ configure_app_mgr(config, slots, appMgr)
config = mergeConfigs(config)
return config
diff --git a/Phys/DaVinci/python/DaVinci/algorithms.py b/Phys/DaVinci/python/DaVinci/algorithms.py
index 6c0009ebb..180056e65 100644
--- a/Phys/DaVinci/python/DaVinci/algorithms.py
+++ b/Phys/DaVinci/python/DaVinci/algorithms.py
@@ -1,5 +1,5 @@
###############################################################################
-# (c) Copyright 2020-20201 CERN for the benefit of the LHCb Collaboration #
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
@@ -11,7 +11,8 @@
from GaudiConfig2 import Configurables as C
from DaVinci.configOptions import get_slot_value, set_slot_value
-from DaVinci.configurations import *
+from DaVinci.optionChecker import DVOptionError, DVRuntimeError
+from DaVinci.configurations import *
def define_dv_init_algo(config, slots):
@@ -19,7 +20,7 @@ def define_dv_init_algo(config, slots):
Define the DaVinci initialisation algorithm.
"""
dvInit = C.DaVinciInit("DaVinciInitAlg")
- configure_init(dvInit, slots)
+ configure_init(slots, dvInit)
config.append(dvInit)
@@ -28,44 +29,99 @@ def define_input(slots):
Define the input files via the IOHelper.
"""
from GaudiConf import IOHelper
-
+
input = get_slot_value(slots, "Input")
-
- if ( len(input) > 0 ):
+
+ if (len(input) > 0):
persistency = None
inputType = get_slot_value(slots, "InputType").upper()
- if inputType == "MDF" : persistency = "MDF"
+ if inputType == "MDF": persistency = "MDF"
# Support connection strings and lists of files
- input = IOHelper(persistency, persistency).convertConnectionStrings(input, "I")
+ input = IOHelper(persistency, persistency).convertConnectionStrings(
+ input, "I")
# Clear selector to maintain the same behaviour
- IOHelper(persistency, persistency).inputFiles(input, clear = True)
+ IOHelper(persistency, persistency).inputFiles(input, clear=True)
else:
log = get_slot_value(slots, "Log")
- log.warning("No input files available along with the selected key. Check the related DaVinciDB.")
+ log.warning(
+ "No input files available along with the selected key. Check the related DaVinciDB."
+ )
def define_monitors(config, slots):
"""
- Define the monitors - AuditorSvc, EventSelector, SequencerTimerTool, TimingAuditor.
+ Define the monitors:
+
+ - AuditorSvc
+ - EventSelector
+ - SequencerTimerTool
+ - TimingAuditor
+ - EventClock.
"""
auditor = C.AuditorSvc()
timer = C.TimingAuditor()
sequencer = C.SequencerTimerTool()
- configure_timing(auditor, timer, sequencer)
-
+ configure_timing(slots, auditor, sequencer, timer)
+
evtSel = C.EventSelector()
- configure_event_selector(evtSel, slots)
-
- config.extend([auditor, timer, sequencer, evtSel])
+ configure_event_selector(slots, evtSel)
+ evtTime = C.EventClockSvc(EventTimeDecoder="OdinTimeDecoder")
+ config.extend([auditor, timer, sequencer, evtSel]) #, evtTime])
-def setupAlgorithms(config, slots):
+
+def setup_algorithms(config, slots):
"""
Set DaVinci algorithms
"""
- algorithms = [C.GaudiHistoAlgorithm("SimpleHistos",
- HistoPrint=True,
- OutputLevel=get_slot_value(slots, "OutputLevel"))]
+ import importlib, sys, os
+ stdAlgorithms = [
+ C.GaudiHistoAlgorithm(
+ "SimpleHistos",
+ HistoPrint=True,
+ OutputLevel=get_slot_value(slots, "OutputLevel"))
+ ]
+
+ fileAlg = get_slot_value(slots, 'UserAlgorithmFile')
+ funcName = get_slot_value(slots, 'UserAlgorithm')
+
+ modulePath = fileAlg.rsplit('/', 1)[0]
+ moduleName = fileAlg.rsplit('/', 1)[1]
+
+ modulePath = os.path.expandvars(modulePath)
+ _, ext = os.path.splitext(moduleName)
+ if ext == ".py":
+ moduleName = moduleName.rsplit('.', 1)[0]
+
+ sys.path.append(modulePath)
+ log = get_slot_value(slots, "Log")
+
+ if moduleName and funcName:
+ try:
+ module = importlib.import_module(moduleName)
+ except:
+ raise DVOptionError(
+ "UserAlgorithmFile",
+ "Importing user algorithms failed. Check if the user python module %s defined in %s exists!"
+ % (moduleName, modulePath))
+ algorithms = stdAlgorithms
+ else:
+ try:
+ userAlgorithms = eval("module.%s()" % funcName)
+ log.info("User algorithm %s.%s imported successfully!" %
+ (moduleName, funcName))
+ algorithms = stdAlgorithms + userAlgorithms
+ except:
+ raise DVRuntimeError(
+ "UserAlgorithmFile", "UserAlgorithm",
+ "Run time error when calling the user algorithm. User algorithm %s can not be imported!"
+ % funcName)
+ algorithms = stdAlgorithms
+ else:
+ log.warning(
+ "DV option file or main function not defined. No user algorithms will be used."
+ )
+ algorithms = stdAlgorithms
config.extend(algorithms)
@@ -73,33 +129,49 @@ def setupAlgorithms(config, slots):
def define_services(config, slots):
"""
Define standard services:
-
- - DataOnDemandSvc
- - DetDataSvc
+
+ - AlgContextSvc
+ - EvtDataSvc
- EvtPersistencySvc
+ - FileRecordDataSvc
- IODataManager
- LoKiSvc
- MultiFileCatalog
- RootCnvSvc
- ToolSvc
+ - XmlCnvSvc
+ - XmlParserSvc
"""
- lokiSvc = C.LoKiSvc()
- dataSvc = C.DataOnDemandSvc()
- configure_printouts(lokiSvc, dataSvc, slots)
-
+
+ evtSvc = C.EvtDataSvc()
evtPers = C.EvtPersistencySvc("EventPersistencySvc")
- configure_event_persistency(evtPers)
-
- extSvc = [lokiSvc,
- dataSvc,
- evtPers,
- C.ToolSvc(),
- C.DetDataSvc("DetectorDataSvc"),
- C.Gaudi.MultiFileCatalog("FileCatalog"),
- C.Gaudi.IODataManager("IODataManager"),
- C.Gaudi.RootCnvSvc("FileRecordCnvSvc"),
- ]
-
+ configure_event(evtSvc, evtPers)
+
+ algSvc = C.AlgContextSvc()
+ algSvc.BypassIncidents = True
+
+ rootSvc = C.Gaudi.RootCnvSvc("FileRecordCnvSvc")
+ fileDataSvc = C.FileRecordDataSvc()
+ configure_file_record_data(rootSvc, fileDataSvc)
+
+ xmlSvc = C.XmlCnvSvc()
+ xmlParser = C.XmlParserSvc()
+ configure_xml(xmlSvc, xmlParser)
+
+ extSvc = [
+ evtSvc,
+ evtPers,
+ algSvc,
+ fileDataSvc,
+ rootSvc,
+ xmlSvc,
+ C.LoKiSvc(),
+ C.ToolSvc(),
+ C.Gaudi.Monitoring.MessageSvcSink(),
+ C.Gaudi.MultiFileCatalog("FileCatalog"),
+ C.Gaudi.IODataManager("IODataManager"),
+ ]
+
config.extend(extSvc)
@@ -113,16 +185,18 @@ def define_root_files(config, slots):
from GaudiKernel.Configurable import ConfigurableGeneric as RFileCnv
if get_slot_value(slots, "HistogramFile"):
- histoSvc = C.HistogramPersistencySvc(OutputFile = get_slot_value(slots, "HistogramFile"))
+ histoSvc = C.HistogramPersistencySvc(
+ OutputFile=get_slot_value(slots, "HistogramFile"))
config.append(histoSvc)
if get_slot_value(slots, "TupleFile"):
ntSvc = C.NTupleSvc()
- configure_options_root_ntuples(ntSvc, slots)
+ configure_options_root_ntuples(slots, ntSvc)
config.append(ntSvc)
# Set the compression level for the ROOT tuple file
- RFileCnv("RFileCnv").GlobalCompression = get_slot_value(slots, "RootCompressionLevel")
+ RFileCnv("RFileCnv").GlobalCompression = get_slot_value(
+ slots, "RootCompressionLevel")
def define_log(slots):
@@ -131,6 +205,17 @@ def define_log(slots):
"""
from AnalysisPython.Logger import getLogger
- log = getLogger ("DaVinci")
- log.info("Applying DaVinci configuration")
+ log = getLogger("DaVinci")
set_slot_value(slots, "Log", log)
+
+
+def define_genfsr_writer(config, slots):
+ """
+ Define Generator FSR writer
+ """
+ if get_slot_value(slots, "MergeGenFSR"):
+ config.append(C.GenFSRMerge())
+ config.append(
+ C.RecordStream(
+ "FSROutputStreamDstWriter",
+ OutputLevel=get_slot_value(slots, "OutputLevel")))
diff --git a/Phys/DaVinci/python/DaVinci/configOptions.py b/Phys/DaVinci/python/DaVinci/configOptions.py
index 24a7fd669..bb0a7a9b8 100644
--- a/Phys/DaVinci/python/DaVinci/configOptions.py
+++ b/Phys/DaVinci/python/DaVinci/configOptions.py
@@ -1,5 +1,5 @@
###############################################################################
-# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration #
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
@@ -8,39 +8,49 @@
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
-
"""
Set and retrieve the value of a specific slot property.
"""
-import os
+import os, difflib
+from DaVinci.optionChecker import DVOptionError, DVRuntimeError, option_checker
def get_slot_value(slots, name):
- return slots[name]
+ if if_slot_exists(slots, name):
+ return slots[name]
+ else:
+ return None
def set_slot_value(slots, name, value):
- if name in slots:
+ if if_slot_exists(slots, name):
slots[name] = value
+
+
+def if_slot_exists(slots, name):
+ if name in slots:
+ return True
else:
- import difflib
- raise ValueError("Unknown qualifier called %s. Please check the DaVinci database or the job option file!\n "
- "List of possible valid qualifiers: %s" %(name, difflib.get_close_matches(name, slots)))
+ raise DVOptionError(
+ name,
+ "Unknown qualifier called %s. Please check the DaVinci database or the job option file!\nList of possible valid qualifiers: %s"
+ % (name, difflib.get_close_matches(name, slots)))
+ return False
-def get_default_value(myKey):
+def get_default_value(inputKey):
"""
Get the default value for a specific property.
"""
import yaml
-
+
optsDefaultName = "$DAVINCIROOT/options/DVconfig-Default.yaml"
with open(os.path.expandvars(optsDefaultName)) as optsDefault:
config = yaml.safe_load(optsDefault)
for key, args in config.items():
- if key == myKey:
+ if key == inputKey:
for name, value in args.items():
if name == 'value':
return value
@@ -65,7 +75,7 @@ def initialize_slots(slots, propDct):
propDct[key] = value
-def set_file_options(slots, myKey, dbName, isMC):
+def set_file_options(slots, inputKey, dbName, isMC):
"""
Set the dataset properties required by the user.
"""
@@ -74,12 +84,12 @@ def set_file_options(slots, myKey, dbName, isMC):
dataDV = yaml.safe_load(dbFile)
idxFile = -1
- if ":" in myKey:
- idxFile = int(myKey.split(":")[1])
- myKey = myKey.split(":")[0]
-
+ if ":" in inputKey:
+ idxFile = int(inputKey.split(":")[1])
+ inputKey = inputKey.split(":")[0]
+
for key, config in dataDV.items():
- if key == myKey:
+ if key == inputKey:
for item, obj in config.items():
if item == 'qualifiers':
for prop, value in obj.items():
@@ -89,52 +99,76 @@ def set_file_options(slots, myKey, dbName, isMC):
if idxFile < len(obj):
set_slot_value(slots, "Input", [obj[idxFile]])
else:
- raise ValueError('Index file requested exceeds the number of available files related to the given key!')
+ raise ValueError(
+ 'Index file exceeds the number of available files related to the given key!'
+ )
else:
set_slot_value(slots, "Input", obj)
-def set_job_options(slots, configName, myKey, dbName):
+def set_job_options(slots, configName, inputKey, dbName):
"""
Set the job properties required by the user.
"""
log = get_slot_value(slots, "Log")
- if configName == '':
+ if configName == '':
log.warning('No jobOption file selected, the default values are used.')
else:
- dataOptions = list_data_options(myKey.split(":")[0], dbName)
-
+ dataOptions = list_data_options(inputKey.split(":")[0], dbName)
+
with open(os.path.expandvars(configName)) as config_file:
- """
- if configName.endswith('.json'):
- import json
- config = json.load(config_file)
- elif configName.endswith('.yaml') or configName.endswith('.yml'):
- """
_, ext = os.path.splitext(configName)
if ext in (".yaml", ".yml", ".json"):
import yaml
config = yaml.safe_load(config_file)
- elif configName.endswith('.py'):
+ elif ext == '.py':
import ast
config = ast.literal_eval(config_file.read())
else:
- raise ValueError('JobOption file extension not known! Please use only a .py, .json or .yaml (.yml) file!')
-
+ raise ValueError(
+ 'JobOption file extension not known! Please use only a .py, .json or .yaml (.yml) file!'
+ )
+
for key, value in config.items():
- if key in dataOptions:
- if get_slot_value(slots, "OverwriteDataOptions"):
- log.warning('Default value for option %s is taken from DaVinciDB but a new value is found in option file. '
- 'Since "OverwriteDataOptions" is active the new value is used. Make sure this is the configuration you want to use for this job.' %key)
- set_slot_value(slots, key, value)
- else:
- log.warning('Default value for option %s is taken from DaVinciDB but a new value is found in option file. '
- 'Since "OverwriteDataOptions" is not active the default value is used. Make sure this is the configuration you want to use for this job.' %key)
- else:
+ if is_option_settable(slots, key, dataOptions):
set_slot_value(slots, key, value)
-def list_data_options(myKey, dbName):
+def set_args_options(slots, kwargs, inputKey, dbName):
+ """
+ Set the extra arguments required by the user.
+ """
+ dataOptions = list_data_options(inputKey.split(":")[0], dbName)
+ for i in range(1, len(kwargs), 2):
+ key = kwargs[i][2:]
+ value = kwargs[i + 1]
+
+ if is_option_settable(slots, key, dataOptions):
+ set_slot_value(slots, key, value)
+
+
+def is_option_settable(slots, key, dataOptions):
+ """
+ Check if the current option 'key' can be set corretly
+ """
+ log = get_slot_value(slots, "Log")
+ if key in dataOptions:
+ if get_slot_value(slots, "OverwriteDataOptions"):
+ log.info(
+ "New value found for the option %s in the job option file. 'OverwriteDataOptions' is active so the default value will be overwritten."
+ % key)
+ return True
+ else:
+ raise DVRuntimeError(
+ key, "OverwriteDataOptions",
+ "Default value for option %s is taken from DaVinciDB.\nA new value is found in the job option file but 'OverwriteDataOptions' is not active so the default value can't be overwritten!"
+ % key)
+ return False
+ else:
+ return True
+
+
+def list_data_options(inputKey, dbName):
"""
List of the properties that are set automatically given a dataset.
"""
@@ -145,7 +179,7 @@ def list_data_options(myKey, dbName):
optionList = []
for key, config in dataDV.items():
- if key == myKey:
+ if key == inputKey:
for item, obj in config.items():
if item == 'qualifiers':
for prop, value in obj.items():
@@ -158,36 +192,45 @@ def check_options(slots):
"""
Check the options. Applies changes if needed.
"""
- log = get_slot_value(slots, "Log")
- known_datatypes = ["Upgrade"]
- known_default_detectors = ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ]
-
- dataType = get_slot_value(slots,"DataType")
- if (not dataType):
- raise TypeError( "You must set DataType" )
- if dataType not in known_datatypes:
- raise TypeError( "Invalid DataType '%s'" %dataType )
+ dataType = get_slot_value(slots, "DataType")
+ option_checker("DataType", dataType)
inputType = get_slot_value(slots, "InputType").upper()
- if inputType not in [ "MDF", "DST", "DIGI", "RDST", "MDST", "SDST", "XDST", "LDST" ]:
- raise TypeError( "Invalid inputType '%s'"%inputType )
-
- if get_slot_value(slots, "Simulation") and not inputType in ( "MDF" , "DIGI" , "MDST" ):
- redo = get_slot_value(slots, "RedoMCLinks")
- if (inputType == "RDST") and (redo):
- log.warning("Re-doing MC links not possible for RDST")
- set_slot_value(slots, "RedoMCLinks", False)
-
- if get_slot_value(slots, "Simulation") and get_slot_value(slots,'Lumi'):
- log.warning('Lumi not valid for Simulation. Setting Lumi = False')
- set_slot_value(slots, 'Lumi', False)
-
- ## for simulation, it is very important to specify proper DB-tags:
- if get_slot_value(slots, 'Simulation'):
- if not get_slot_value(slots,'DDDBtag'):
- log.warning("``DDDBtag'' is not specified for simulated data")
+ option_checker("InputType", inputType)
+
+ redoLink = get_slot_value(slots, "RedoMCLinks")
+ isMC = get_slot_value(slots, "Simulation")
+
+ if not isMC:
+ if redoLink:
+ raise DVRuntimeError(
+ "RedoMCLinks", "Simulation",
+ "Re-doing MC links not possible for data! Set RedoMCLinks = False."
+ )
+
+ if get_slot_value(slots, "MergeGenFSR"):
+ raise DVRuntimeError(
+ "MergeGenFSR", "Simulation",
+ "GenFSR are not available in real data so MergeGenFSR cannot be run! Set MergeGenFSR = False."
+ )
+ else:
+ if redoLink and inputType("MDF", "DIGI", "MDST", "RDST"):
+ raise DVRuntimeError(
+ "RedoMCLinks", "InputType",
+ "Re-doing MC links not possible for %s input type! Set RedoMCLinks = False."
+ % inputType)
+
+ if get_slot_value(slots, "Lumi"):
+ raise DVRuntimeError(
+ "Lumi", "Simulation",
+ "Lumi not valid for Simulation! Set Lumi = False.")
+
+ ## for simulation, it is very important to specify proper DB-tags:
+ if not get_slot_value(slots, 'DDDBtag'):
+ raise DVOptionError(
+ "DDDBtag",
+ "``DDDBtag'' is not specified for simulated data!", "")
if not get_slot_value(slots, 'CondDBtag'):
- log.warning("``CondDBtag'' is not specified for simulated data")
-
- if get_slot_value(slots, "MergeGenFSR") and not get_slot_value(slots, "Simulation"):
- raise TypeError("Cannot MergeGenFSR on real data");
+ raise DVOptionError(
+ "CondDBtag",
+ "``CondDBtag'' is not specified for simulated data!", "")
diff --git a/Phys/DaVinci/python/DaVinci/configurations.py b/Phys/DaVinci/python/DaVinci/configurations.py
index 7048daa80..a1b3572b7 100644
--- a/Phys/DaVinci/python/DaVinci/configurations.py
+++ b/Phys/DaVinci/python/DaVinci/configurations.py
@@ -15,7 +15,7 @@ Define configurations for applications, algorithms and external services.
from DaVinci.configOptions import get_slot_value, get_default_value
-def configure_app_mgr(appMgr, slots, config):
+def configure_app_mgr(config, slots, appMgr):
"""
Configuration of the main application.
"""
@@ -28,6 +28,8 @@ def configure_app_mgr(appMgr, slots, config):
appMgr.HistogramPersistency = 'ROOT'
appMgr.AuditAlgorithms = True
+ propagate_properties_to_lhcb(config, slots)
+
appMgr.ExtSvc = [
c for c in config if c.__component_type__ == 'Service'
and c.name not in ['ApplicationMgr']
@@ -38,24 +40,29 @@ def configure_app_mgr(appMgr, slots, config):
]
-def configure_init(dvInit, slots):
+def configure_init(slots, dvInit):
"""
Configuration of initialisation algorithm.
"""
dvInit.Increment = get_slot_value(slots, "PrintFreq")
-def configure_timing(auditor, timer, sequencer):
+################################################################################
+# Define configurations for external services
+#
+
+
+def configure_timing(slots, auditor, sequencer, timer):
"""
Configuration of timing auditors.
"""
auditor.Auditors = ['ChronoAuditor', 'TimingAuditor']
- sequencer.OutputLevel = 4
+ sequencer.OutputLevel = get_slot_value(slots, "OutputLevel")
-def configure_options_root_ntuples(svc, slots):
+def configure_options_root_ntuples(slots, svc):
"""
- Configuration of output .root ntuples
+ Configuration of output .root ntuples.
"""
for _line in svc.Output:
if 0 <= _line.find('FILE1'):
@@ -65,26 +72,20 @@ def configure_options_root_ntuples(svc, slots):
tupleFile = get_slot_value(slots, "TupleFile")
tupleStr = "FILE1 DATAFILE='%s' TYP='ROOT' OPT='NEW'" % tupleFile
svc.Output += [tupleStr]
- svc.OutputLevel = 1
+ svc.OutputLevel = get_slot_value(slots, "OutputLevel")
-def configure_event_persistency(svc):
+def configure_event(evtSvc, svc):
"""
- Configuration of the event persistency service
+ Configuration of the event services.
"""
+ evtSvc.EnableFaultHandler = True
+ evtSvc.ForceLeaves = True
+ evtSvc.RootCLID = 1
svc.CnvServices.append("Gaudi::RootCnvSvc/RootCnvSvc")
-def configure_printouts(lokiSvc, dataSvc, slots):
- """
- Configuration of the printout services.
- """
- verbosePrint = get_slot_value(slots, "VerboseMessages")
- lokiSvc.Welcome = verbosePrint
- dataSvc.Dump = verbosePrint
-
-
-def configure_event_selector(svc, slots):
+def configure_event_selector(slots, svc):
"""
Configuration of the event selector.
"""
@@ -94,3 +95,99 @@ def configure_event_selector(svc, slots):
log = get_slot_value(slots, "Log")
log.warning("Print frequency cannot be 0. Set to %f." % printFreq)
svc.PrintFreq = printFreq
+
+
+#def configure_detector(detSvc, detPers):
+# """
+# Configure Detector services.
+# """
+# detSvc.DetDbLocation = "git:/lhcb.xml"
+# detSvc.DetDbRootName = "dd"
+# detSvc.DetStorageType = 7
+# detSvc.UsePersistency = True
+
+# detPers.CnvServices = ["XmlCnvSvc/XmlCnvSvc"]
+
+
+def configure_file_record_data(rootSvc, fileDataSvc):
+ """
+ Configure File Record Data service.
+ """
+ rootSvc.CacheBranches = []
+ rootSvc.EnableIncident = True
+ rootSvc.VetoBranches = ["*"]
+
+ fileDataSvc.EnableFaultHandler = True
+ fileDataSvc.ForceLeaves = True
+ fileDataSvc.PersistencySvc = "PersistencySvc/FileRecordPersistencySvc"
+ fileDataSvc.RootCLID = 1
+
+
+def configure_xml(xmlSvc, xmlParser):
+ """
+ Configure xml services.
+ """
+ xmlSvc.AllowGenericConversion = True
+
+ xmlParser.CacheBehavior = 3
+ xmlParser.EntityResolver = "EntityResolverDispatcher/EntityResolverDispatcher"
+ xmlParser.MaxDocNbInCache = 15
+
+
+################################################################################
+# Define configurations for MC algorithms
+#
+
+
+def configure_MC_algorithms(slots, dataSvc):
+ """
+ Configure DaVinciAssociators and do MC unpacking.
+ """
+ from CaloKernel.ConfUtils import getAlgo
+
+ root = ""
+ if get_slot_value(slots, "InputType") == "MDST" and get_slot_value(
+ slots, "RootInTES"):
+ root = get_slot_value(slots, "RootInTES")
+ else:
+ root = "/Event"
+
+ configure_MC_unpacking(slots, dataSvc, root)
+
+
+def configure_MC_unpacking(slots, dataSvc, root):
+ """
+ Configure the unpacking of MC particles.
+ """
+ import os
+
+ log = get_slot_value(slots, "Log")
+ mcRoot = os.path.join(root, "MC")
+ log.info("Will unpack MC objects to {0}".format(mcRoot))
+
+ particlesOutput = os.path.join(mcRoot, "Particles")
+ verticesOutput = os.path.join(mcRoot, "Vertices")
+
+ samba = C.UnpackMCParticle("UnpackMCParticle", RootInTES=root)
+ dataSvc.NodeMap[mcRoot] = "DataObject"
+ #dataSvc.AlgMap[particlesOutput] = C.UnpackMCParticle("UnpackMCParticle", RootInTES=root)
+ #dataSvc.AlgMap[verticesOutput] = C.UnpackMCVertex("UnpackMCVertex", RootInTES=root)
+
+
+def propagate_properties_to_lhcb(config, slots):
+ """
+ Define and configure LHCbApp with the relevant DV properties.
+ """
+ from Configurables import LHCbApp
+
+ lhcbApp = LHCbApp()
+ properties = [
+ "EvtMax", "SkipEvents", "DataType", "CondDBtag", "DDDBtag",
+ "DQFLAGStag", "Simulation", "IgnoreDQFlags"
+ ]
+
+ for prop in properties:
+ if get_slot_value(slots, prop):
+ lhcbApp.setProp(prop, get_slot_value(slots, prop))
+
+ #config.append(lhcbApp)
diff --git a/Phys/DaVinci/python/DaVinci/optionChecker.py b/Phys/DaVinci/python/DaVinci/optionChecker.py
new file mode 100644
index 000000000..89dbb4f18
--- /dev/null
+++ b/Phys/DaVinci/python/DaVinci/optionChecker.py
@@ -0,0 +1,107 @@
+###############################################################################
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+"""
+Define value and run-time errors used in DaVinci application.
+"""
+
+
+class DVOptionError(ValueError):
+ """
+ DaVinci exception class for invalid option configurations.
+ """
+
+ def __init__(self, name, value):
+ """
+ Constructor.
+ """
+ self.message = "invalid DaVinci %s option value specified: %s" % (
+ name, value)
+
+ def __str__(self):
+ """
+ String representation.
+ """
+ return (set_color("red") + self.message + set_color("plain"))
+
+
+class DVRuntimeError(RuntimeError):
+ """
+ DaVinci exception class for inconsistent option configurations.
+ """
+
+ def __init__(self, option1, option2, value):
+ """
+ Constructor.
+ """
+ self.message = "Option %s and %s are inconsistent! \n %s" % (
+ option1, option2, value)
+
+ def __str__(self):
+ """
+ String representation.
+ """
+ return (set_color("red") + self.message + set_color("plain"))
+
+
+def option_checker(name, value):
+ """
+ Check the correctness of a DaVinci configurable option.
+ """
+ try:
+ allowedValues = get_allowed_option_values()
+ if value not in allowedValues[name]:
+ print_allowed_option_values(allowedValues, name)
+ raise DVOptionError(name, value)
+ except KeyError:
+ print set_color(
+ "red"
+ ) + "You are trying to check the unknown option %s!" % name + set_color(
+ "plain")
+ print_allowed_option_values()
+
+
+def get_allowed_option_values():
+ allowedValues = {
+ "DataType": ["Upgrade"],
+ "Detectors": [
+ 'VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon',
+ 'Magnet', 'Tr'
+ ],
+ "InputType":
+ ["MDF", "DST", "DIGI", "RDST", "MDST", "SDST", "XDST", "LDST"]
+ }
+
+ return allowedValues
+
+
+def print_allowed_option_values(allowedValues, name=None):
+ """
+ Print the allowed values for the DaVinci option configurations.
+ """
+ if name == None:
+ print set_color(
+ "green") + "Known job option configurations and allowed values:"
+ for name, values in allowedValues.iteritems():
+ print "%s \t : %s" % (name, value), set_color("plain")
+ else:
+ print set_color(
+ "green") + "Allowed values for DaVinci option %s:" % name
+ print allowedValues[name], set_color("plain")
+
+
+def set_color(key):
+ colors = {
+ "plain": "\x1b[00m",
+ "red": "\x1b[01;31m",
+ "green": "\x1b[01;32m"
+ }
+
+ return colors[key]
diff --git a/Phys/DaVinci/python/DaVinci/utilities.py b/Phys/DaVinci/python/DaVinci/utilities.py
new file mode 100644
index 000000000..a0e4e06c2
--- /dev/null
+++ b/Phys/DaVinci/python/DaVinci/utilities.py
@@ -0,0 +1,88 @@
+###############################################################################
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+
+from GaudiConfig2 import Configurables as C
+from DaVinci.configOptions import get_slot_value
+import os, click
+
+
+def dump_call(ctx, key, dbfile, optfile):
+ indentStr = "# "
+ click.echo("%sDaVinci: running using the following arguments:" % indentStr)
+ click.echo("%s - key: {key}" % indentStr)
+ click.echo("%s - dbfile: {dbfile}" % indentStr)
+ click.echo("%s - optfile: {optfile}" % indentStr)
+
+ if len(ctx.args) > 1:
+ for i in range(1, len(ctx.args), 2):
+ click.echo("%s - %s: %s" % (indentStr, ctx.args[i][2:],
+ ctx.args[i + 1]))
+
+
+def dump_configuration(slots):
+ """
+ Dump the DaVinci Configuration.
+ """
+ indentStr = "# "
+ appName = "DaVinci"
+ preLen = 10
+ headerWidth = 100
+
+ log = get_slot_value(slots, "Log")
+
+ header = "Applying %s configuration" % appName
+ dump = indentStr + header + os.linesep
+
+ title = "User %s/%s" % (appName, appName)
+ postLen = get_post_length(title, preLen, headerWidth)
+
+ dump += "%s%s %s %s" % (indentStr, preLen * '*', title,
+ postLen * '*') + os.linesep
+ dump += get_prop_infos(slots, indentStr)
+
+ footer = "(End of User %s/%s)" % (appName, appName)
+ postLen = get_post_length(footer, preLen, headerWidth)
+ dump += "%s%s %s %s" % (indentStr, preLen * '-', footer, postLen * '-')
+ log.info(dump)
+
+
+def get_prop_infos(slots, indentStr):
+ """
+ Dump the DV configuration with the name and value of each property.
+ """
+ nameWidth = get_name_width(slots)
+ config = ""
+
+ for prop in slots:
+ prefix = "%s|-%s" % (indentStr, prop.ljust(nameWidth))
+ value = get_slot_value(slots, prop)
+ line = "%s = %s" % (prefix, repr(value))
+ config += line + os.linesep
+ return config
+
+
+def get_name_width(slots):
+ """
+ Get the maximum length of the property names.
+ """
+ nameWidth = 0
+ for slot in slots:
+ nameWidth = max(nameWidth, len(slot))
+ return nameWidth
+
+
+def get_post_length(name, preLen, width):
+ """
+ Get the number of remaining blank spaces after 'name' given the specific width.
+ """
+ postLen = width - preLen - len(name)
+ postLen = max(preLen, postLen)
+ return postLen
--
GitLab