diff --git a/DaVinciSys/CMakeLists.txt b/DaVinciSys/CMakeLists.txt
index e047419750ca81e3bea6100aed6465f006296d5e..e0bd16fb419553d7f3a642a0e782a43a7d55d504 100644
--- a/DaVinciSys/CMakeLists.txt
+++ b/DaVinciSys/CMakeLists.txt
@@ -17,4 +17,6 @@ gaudi_depends_on_subdirs(DaVinciTests
Phys/DaVinci
)
+gaudi_install_scripts()
+
gaudi_add_test(QMTest QMTEST)
diff --git a/DaVinciSys/scripts/davinci b/DaVinciSys/scripts/davinci
new file mode 100755
index 0000000000000000000000000000000000000000..e19520743f572aecd8dda8fe1f9add5855250bac
--- /dev/null
+++ b/DaVinciSys/scripts/davinci
@@ -0,0 +1,206 @@
+#!/usr/bin/env python
+###############################################################################
+# (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+"""
+This script is meant to implement the Python Click functionality
+when running a DaVinci (Gaudi) application.
+Two possible functions have been developed to automatically handle
+the differences between jobs using data or simulated input files.
+
+A help message with all the details on the possible commands and
+relative options can be obtained with the command line:
+
+./run davinci
+
+A DaVinci job using simulated data can be run using the command line:
+
+./run davinci [davinci_opts] run-mc [job_opts]
+"""
+
+import os, sys, click
+from DaVinci.utilities_script import dump_call, get_configurable_opts
+from DaVinci.ConfigurationUpgrade import run_davinci
+
+if "GAUDIAPPVERSION" in os.environ:
+ APP_VERSION = str(os.environ["GAUDIAPPVERSION"])
+else:
+ APP_VERSION = ""
+
+
+@click.group()
+@click.option(
+ "--export",
+ default=None,
+ help=
+ "export the options in a format suitable for gaudirun.py. Filename has to be provided as argument.",
+)
+@click.option(
+ "--with-defaults",
+ is_flag=True,
+ default=False,
+ help=
+ "explicitly include default values of properties in the final configuration",
+)
+@click.option(
+ "--dry-run",
+ is_flag=True,
+ default=False,
+ help="just process the options but do not run the application",
+)
+@click.version_option(version=APP_VERSION)
+def main(export, with_defaults, dry_run):
+ assert not export or export.endswith(
+ ".opts"), "--export argument must have '.opts' extension"
+
+
+@main.resultcallback()
+def run_job(configurables, export=None, with_defaults=False, dry_run=None):
+ """
+ This function is not invoked directly in Davinci but it is used to override the
+ default click 'result_callback' function by means of the 'resultcallback' decorator.
+ The 'result_callback' method is invoked automatically when returning the values
+ of the main subcommnads: run_mc and run_data.
+ """
+ assert configurables.values() # we only want the values
+ configurables = configurables.values()
+
+ opts = {
+ "ApplicationMgr.AppName": '"DaVinci"',
+ "ApplicationMgr.AppVersion": '"{}"'.format(APP_VERSION),
+ }
+
+ dict_opts = get_configurable_opts(configurables, with_defaults)
+ opts.update(dict_opts)
+
+ if export:
+ click.echo("writing configuration to {}".format(export))
+ with open(export, "w") as f:
+ f.writelines("{} = {};\n".format(*item) for item in opts.items())
+
+ if dry_run:
+ click.echo("dry-run: not starting the application")
+ else:
+ import Gaudi
+ opts["ApplicationMgr.JobOptionsType"] = '"NONE"'
+ app = Gaudi.Application(opts)
+ exit(app.run())
+
+
+@main.command(
+ context_settings=dict(
+ ignore_unknown_options=True,
+ allow_extra_args=True,
+ ))
+@click.option(
+ "--testfiledb",
+ default=("$DAVINCIROOT/options/DaVinciDB-Example.yaml",
+ "Upgrade_Bd2KstarMuMu_ldst"),
+ nargs=2,
+ help=
+ "TestFileDB-like file containing job input and conditions information (.yaml). Takes the pair of values 'filedb-path', 'filedb-key'"
+)
+@click.option(
+ "--joboptfile",
+ default="$DAVINCIROOT/options/jobOptions-Example.yaml",
+ help="Option file containing the job information (.yaml, .py)")
+@click.option(
+ "--simplejob",
+ is_flag=True,
+ default=False,
+ help=
+ "Option for running a simple DaVinci jobs without any specific configuration (.py)."
+)
+@click.pass_context
+def run_mc(ctx, testfiledb, joboptfile, simplejob):
+ """
+ DaVinci function for running jobs on simulated samples on the command line, using Click.
+ Ctx: click.core.Context class (dict). Predefined click object storing information about the invoked command.
+ All the options passed by command line which are not recognised by click are stored into the ctx.args element.
+ Ctx.args is a simple array and each extra option is stored using two values: the first one is the key
+ and the second one is the corresponding value.
+ Eg: --evt_max 100 will be stored as: ctx.args[0] = --evt_max, ctx.args[1] = 100
+
+ Click automatically converts "_" in "-", so this function can be invoked calling run-mc as shown in the help.
+ """
+ assert len(
+ testfiledb
+ ) == 2, "--testfiledb takes two arguments: filedb filename and the relevant key."
+ testfiledb_file = testfiledb[0]
+ testfiledb_key = testfiledb[1]
+
+ ctx_args = (ctx.args if (len(ctx.args) > 1) else [])
+ dump_call(testfiledb_file, testfiledb_key, joboptfile, ctx_args)
+
+ config = run_davinci(
+ testfiledb_file,
+ testfiledb_key, #file and key for job input and conditions
+ joboptfile, # file for job options
+ True, # flag for MC job
+ ctx_args, # list of extra options to be set in the job
+ simplejob # flag for running a simplejob
+ )
+ return config
+
+
+@main.command(
+ context_settings=dict(
+ ignore_unknown_options=True,
+ allow_extra_args=True,
+ ))
+@click.option(
+ "--testfiledb",
+ default=("$DAVINCIROOT/options/DaVinciDB-Example.yaml",
+ "Upgrade_Bd2KstarMuMu_ldst"),
+ nargs=2,
+ help=
+ "TestFileDB-like file containing job input and conditions information (.yaml). Takes the pair of values 'filedb-path', 'filedb-key'"
+)
+@click.option(
+ "--joboption-file",
+ default="$DAVINCIROOT/options/jobOptions-Example.yaml",
+ help="Option file containing the job information (.yaml, .py)")
+@click.pass_context
+def run_data(ctx, testfiledb, optfile):
+ """
+ DaVinci function for running jobs on real data samples on the command line, using Click.
+ Ctx: click.core.Context class (dict). Predefined click object storing information about the invoked command.
+ All options passed by the command line that are not recognised by click are stored into the ctx.args element.
+ Ctx.args is a simple array and each extra option is stored using two values: the first one is the key
+ and the second one is the corresponding value.
+ Eg: --evt_max 100 will be stored as: ctx.args[0] = --evt_max, ctx.args[1] = 100
+
+ Click automatically converts "_" in "-", so this function can be invoked calling run-data as shown in the help.
+ """
+ raise ValueError(
+ 'Data file with upgrade conditions are not yet available. Please use :mc function instead.'
+ )
+
+ assert len(
+ testfiledb
+ ) == 2, "--testfiledb takes two arguments: filedb filename and the relevant key."
+ testfiledb_file = testfiledb[0]
+ testfiledb_key = testfiledb[1]
+ ctx_args = (ctx.args if (len(ctx.args) > 1) else [])
+ dump_call(testfiledb_file, testfiledb_key, joboptfile, ctx_args)
+
+ config = run_davinci(
+ testfiledb_file,
+ testfiledb_key, #file and key for job input and conditions
+ joboptfile, # file for job options
+ False, # flag for MC job
+ ctx_args, # list of extra options to be set in the job
+ simplejob # flag for running a simplejob
+ )
+ return config
+
+
+if __name__ == "__main__":
+ main()
diff --git a/DaVinciTests/python/DaVinciTests/QMTest/DaVinciExclusions.py b/DaVinciTests/python/DaVinciTests/QMTest/DaVinciExclusions.py
index bb5052aa8d0c987033214291dc85ad545ccc921c..15531c69a69d49441e1c9b60d668b613c00d86bf 100755
--- a/DaVinciTests/python/DaVinciTests/QMTest/DaVinciExclusions.py
+++ b/DaVinciTests/python/DaVinciTests/QMTest/DaVinciExclusions.py
@@ -31,4 +31,7 @@ preprocessor = LHCbPreprocessor + \
LineSkipper(["SIMCOND_"]) + \
LineSkipper(["DDDB_"]) + \
LineSkipper(["DataOnDemandSvc INFO Handled "]) + \
- LineSkipper(["SUCCESS Exceptions/Errors/Warnings/Infos Statistics :"])
+ LineSkipper(["SUCCESS Exceptions/Errors/Warnings/Infos Statistics :"]) + \
+ LineSkipper(["DVControlFlowMgr INFO"]) + \
+ LineSkipper(["HLTControlFlowMgr INFO"]) + \
+ LineSkipper(["|-log_file ="])
diff --git a/DaVinciTests/tests/options/option_davinci_initialise_upgrade.yaml b/DaVinciTests/tests/options/option_davinci_initialise_upgrade.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a959d7cc85d4ce7e79cee05d6e40748927c58117
--- /dev/null
+++ b/DaVinciTests/tests/options/option_davinci_initialise_upgrade.yaml
@@ -0,0 +1,15 @@
+###############################################################################
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+
+evt_max: 1000
+skip_events: 0
+ntuple_file: 'DVNtuple.root'
+histo_file: 'DVHistos.root'
\ No newline at end of file
diff --git a/DaVinciTests/tests/options/option_davinci_user_algs.py b/DaVinciTests/tests/options/option_davinci_user_algs.py
new file mode 100644
index 0000000000000000000000000000000000000000..c60a0f119971d4fa3fe57a0796a3cf7c16f31c55
--- /dev/null
+++ b/DaVinciTests/tests/options/option_davinci_user_algs.py
@@ -0,0 +1,31 @@
+###############################################################################
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+"""
+Example of a DaVinci job printing run and event numbers on every read event.
+"""
+
+__author__ = "Davide Fazzini"
+__date__ = "2021-03-31"
+
+from PyConf.control_flow import CompositeNode, NodeLogic
+from DaVinci import options
+from DaVinci.reco_objects import upfront_reconstruction_from_file as upfront_reconstruction
+
+
+def main():
+ # the "upfront_reconstruction" is what unpacks reconstruction objects, particles and primary vertices
+ # from file and creates protoparticles.
+ algs = upfront_reconstruction()
+
+ node = CompositeNode(
+ "PrintJpsiNode", children=algs, combine_logic=NodeLogic.NONLAZY_AND)
+
+ return node
diff --git a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_control_flow.qmt b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_control_flow.qmt
new file mode 100755
index 0000000000000000000000000000000000000000..d13d0eff511a96dc59950b0a0384a673802eb6d2
--- /dev/null
+++ b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_control_flow.qmt
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+<!--
+###############################################################################
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+-->
+<!DOCTYPE extension PUBLIC '-//QM/2.3/Extension//EN' 'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
+<!--
+#######################################################
+# SUMMARY OF THIS TEST
+# ...................
+# Author: dfazzini
+# Purpose: Very simple test of DaVinci configurable for testing the click feature
+# Prerequisites: None
+# joboptfile $DAVINCITESTSROOT/tests/options/option_davinci_initialise_upgrade.yaml
+#######################################################
+-->
+<extension class="GaudiTest.GaudiExeTest" kind="test">
+ <argument name="program"><text>davinci</text></argument>
+ <argument name="args"><set>
+ <text>run-mc</text>
+ <text>--joboptfile</text>
+ <text>$DAVINCITESTSROOT/tests/options/option_davinci_initialise_upgrade.yaml</text>
+ <text>--evt_max</text>
+ <text>150</text>
+ </set></argument>
+ <argument name="reference"><text>$DAVINCITESTSROOT/tests/refs/test_davinci_control_flow.ref</text></argument>
+ <argument name="error_reference"><text>$DAVINCITESTSROOT/tests/refs/empty.ref</text></argument>
+ <argument name="validator"><text>
+from DaVinciTests.QMTest.DaVinciExclusions import preprocessor
+validateWithReference(preproc = preprocessor)
+countErrorLines({"FATAL":0})
+ </text></argument>
+</extension>
diff --git a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt
index 183352a8cd2f619ee2d62d8ceca6ef00c7339d15..ffb98043a761a933a3cd5b99cb124d0fc9bb9bec 100755
--- a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt
+++ b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_initialise_upgrade.qmt
@@ -1,7 +1,7 @@
<?xml version="1.0" ?>
<!--
###############################################################################
-# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration #
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
@@ -17,17 +17,24 @@
# SUMMARY OF THIS TEST
# ...................
# Author: dfazzini
-# Purpose: Very simple test of DaVinci configurable
+# Purpose: Very simple test of DaVinci configurable for testing the click feature
# Prerequisites: None
+# joboptfile $DAVINCITESTSROOT/tests/options/option_davinci_initialise_upgrade.yaml
#######################################################
-->
<extension class="GaudiTest.GaudiExeTest" kind="test">
- <argument name="program"><text>gaudirun.py</text></argument>
- <argument name="args"><set><text>DaVinci:mc</text></set></argument>
+ <argument name="program"><text>davinci</text></argument>
+ <argument name="args"><set>
+ <text>--dry-run</text>
+ <text>run-mc</text>
+ <text>--joboptfile</text>
+ <text>$DAVINCITESTSROOT/tests/options/option_davinci_initialise_upgrade.yaml</text>
+ </set></argument>
<argument name="reference"><text>$DAVINCITESTSROOT/tests/refs/test_davinci_initialise_upgrade.ref</text></argument>
<argument name="error_reference"><text>$DAVINCITESTSROOT/tests/refs/empty.ref</text></argument>
<argument name="validator"><text>
from DaVinciTests.QMTest.DaVinciExclusions import preprocessor
validateWithReference(preproc = preprocessor)
-</text></argument>
+countErrorLines({"FATAL":0})
+ </text></argument>
</extension>
diff --git a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_simplejob.qmt b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_simplejob.qmt
new file mode 100755
index 0000000000000000000000000000000000000000..a7b61fcec817627f8da57894b74f7ae12a058370
--- /dev/null
+++ b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_simplejob.qmt
@@ -0,0 +1,38 @@
+<?xml version="1.0" ?>
+<!--
+###############################################################################
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+-->
+<!DOCTYPE extension PUBLIC '-//QM/2.3/Extension//EN' 'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
+<!--
+#######################################################
+# SUMMARY OF THIS TEST
+# ...................
+# Author: dfazzini
+# Purpose: Very simple test of DaVinci configurable for testing the click feature
+# Prerequisites: None
+# user_algorithms $DAVINCITESTSROOT/tests/options/option_davinci_simplejob:main
+#######################################################
+-->
+<extension class="GaudiTest.GaudiExeTest" kind="test">
+ <argument name="program"><text>davinci</text></argument>
+ <argument name="args"><set>
+ <text>run-mc</text>
+ <text>--simplejob</text>
+ </set></argument>
+ <argument name="reference"><text>$DAVINCITESTSROOT/tests/refs/test_davinci_simplejob.ref</text></argument>
+ <argument name="error_reference"><text>$DAVINCITESTSROOT/tests/refs/empty.ref</text></argument>
+ <argument name="validator"><text>
+from DaVinciTests.QMTest.DaVinciExclusions import preprocessor
+validateWithReference(preproc = preprocessor)
+countErrorLines({"FATAL":0})
+ </text></argument>
+</extension>
diff --git a/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_user_algs.qmt b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_user_algs.qmt
new file mode 100755
index 0000000000000000000000000000000000000000..49e53045ee9ec888c028b7899e909d629e083e43
--- /dev/null
+++ b/DaVinciTests/tests/qmtest/davinci.qms/test_davinci_user_algs.qmt
@@ -0,0 +1,39 @@
+<?xml version="1.0" ?>
+<!--
+###############################################################################
+# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+-->
+<!DOCTYPE extension PUBLIC '-//QM/2.3/Extension//EN' 'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
+<!--
+#######################################################
+# SUMMARY OF THIS TEST
+# ...................
+# Author: dfazzini
+# Purpose: Very simple test of DaVinci configurable for testing the click feature
+# Prerequisites: None
+# user_algorithms $DAVINCITESTSROOT/tests/options/option_davinci_simplejob:main
+#######################################################
+-->
+<extension class="GaudiTest.GaudiExeTest" kind="test">
+ <argument name="program"><text>davinci</text></argument>
+ <argument name="args"><set>
+ <text>run-mc</text>
+ <text>--user_algorithms</text>
+ <text>$DAVINCITESTSROOT/tests/options/option_davinci_user_algs:main</text>
+ </set></argument>
+ <argument name="reference"><text>$DAVINCITESTSROOT/tests/refs/test_davinci_user_algs.ref</text></argument>
+ <argument name="error_reference"><text>$DAVINCITESTSROOT/tests/refs/empty.ref</text></argument>
+ <argument name="validator"><text>
+from DaVinciTests.QMTest.DaVinciExclusions import preprocessor
+validateWithReference(preproc = preprocessor)
+countErrorLines({"FATAL":0})
+ </text></argument>
+</extension>
diff --git a/DaVinciTests/tests/refs/test_davinci_control_flow.ref b/DaVinciTests/tests/refs/test_davinci_control_flow.ref
new file mode 100644
index 0000000000000000000000000000000000000000..4d2f870189ff6727a5d14bbb1e911c4a9b884cea
--- /dev/null
+++ b/DaVinciTests/tests/refs/test_davinci_control_flow.ref
@@ -0,0 +1,104 @@
+/***** User DVAppOptions/DVAppOptions **************************************************************
+|-auditors = [] (default: [])
+|-buffer_events = 20000 (default: 20000)
+|-callgrind_profile = False (default: False)
+|-conddb_tag = 'HEAD' (default: '')
+|-control_flow_file = '' (default: '')
+|-data_flow_file = '' (default: '')
+|-data_type = 'Upgrade' (default: '')
+|-dddb_tag = 'dddb-20200424-2' (default: '')
+|-detectors = ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr']
+| (default: ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr'])
+|-dqflags_tag = '' (default: '')
+|-enable_unpack = None
+|-event_store = 'HiveWhiteBoard' (default: 'HiveWhiteBoard')
+|-evt_max = 150 (default: -1)
+|-first_evt = 0 (default: 0)
+|-histo_file = 'DVHistos.root' (default: '')
+|-ignore_dq_flags = False (default: False)
+|-input_files = ['root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000002_1.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000004_1.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000043_1.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000068_1.ldst']
+| (default: [])
+|-input_raw_format = 0.3 (default: 0.3)
+|-input_type = 'ROOT' (default: 'DST')
+|-lines_maker = None
+|-lumi = False (default: False)
+|-main_options = '' (default: '')
+|-memory_pool_size = 10485760 (default: 10485760)
+|-merge_genfsr = False (default: False)
+|-msg_svc_format = '% F%35W%S %7W%R%T %0W%M' (default: '% F%35W%S %7W%R%T %0W%M')
+|-msg_svc_time_format = '%Y-%m-%d %H:%M:%S UTC' (default: '%Y-%m-%d %H:%M:%S UTC')
+|-n_event_slots = 1 (default: -1)
+|-n_threads = 1 (default: 1)
+|-ntuple_file = 'DVNtuple.root' (default: '')
+|-output_file = '' (default: '')
+|-output_level = 3 (default: 3)
+|-output_type = '' (default: '')
+|-overwrite_data_options = False (default: False)
+|-print_freq = 1000 (default: 1000)
+|-python_logging_level = 30 (default: 30)
+|-root_compression_level = 'LZMA:6' (default: 'LZMA:6')
+|-scheduler_legacy_mode = True (default: True)
+|-simulation = True (default: False)
+|-skip_events = 0 (default: 0)
+|-tck = 0 (default: 0)
+|-use_iosvc = False (default: False)
+|-user_algorithms = '' (default: '')
+|-write_fsr = True (default: True)
+\----- (End of User DVAppOptions/DVAppOptions) -----------------------------------------------------
+ApplicationMgr SUCCESS
+====================================================================================================================================
+====================================================================================================================================
+ApplicationMgr INFO Application Manager Configured successfully
+NTupleSvc INFO Added stream file:DVNtuple.root as FILE1
+RootHistSvc INFO Writing ROOT histograms to: DVHistos.root
+HistogramPersistencySvc INFO Added successfully Conversion service:RootHistSvc
+FSROutputStreamDstWriter INFO Data source: EventDataSvc output: SVC='Gaudi::RootCnvSvc'
+DetectorDataSvc INFO Detector description not requested to be loaded
+EventClockSvc.FakeEventTime INFO Event times generated from 0 with steps of 0
+HiveDataBrokerSvc WARNING non-reentrant algorithm: RecordStream/FSROutputStreamDstWriter
+HiveDataBrokerSvc WARNING non-reentrant algorithm: GaudiHistoAlgorithm/SimpleHistos
+ApplicationMgr INFO Application Manager Initialized successfully
+ApplicationMgr INFO Application Manager Started successfully
+EventPersistencySvc INFO Added successfully Conversion service:RootCnvSvc
+EventSelector INFO Stream:EventSelector.DataStreamTool_1 Def:DATAFILE='root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000002_1.ldst' SVC='Gaudi::RootEvtSelector' OPT='READ' IgnoreChecksum='YES'
+EventSelector SUCCESS Reading Event record 1. Record number within stream 1: 1
+RndmGenSvc.Engine INFO Generator engine type:CLHEP::RanluxEngine
+RndmGenSvc.Engine INFO Current Seed:1234567 Luxury:3
+RndmGenSvc INFO Using Random engine:HepRndm::Engine<CLHEP::RanluxEngine>
+SimpleHistos INFO GaudiHistoAlgorithm:: Filling Histograms...... Please be patient !
+ApplicationMgr INFO Application Manager Stopped successfully
+FSROutputStreamDstWriter INFO Set up File Summary Record
+FSROutputStreamDstWriter INFO Events output: 1
+NONLAZY_OR: DaVinci #=150 Sum=150 Eff=|( 100.0000 +- 0.00000 )%|
+ NONLAZY_OR: WriteFSR #=150 Sum=150 Eff=|( 100.0000 +- 0.00000 )%|
+ RecordStream/FSROutputStreamDstWriter #=150 Sum=150 Eff=|( 100.0000 +- 0.00000 )%|
+ NONLAZY_AND: DVStdAlgs #=150 Sum=150 Eff=|( 100.0000 +- 0.00000 )%|
+ GaudiHistoAlgorithm/SimpleHistos #=150 Sum=150 Eff=|( 100.0000 +- 0.00000 )%|
+ToolSvc INFO Removing all tools created by ToolSvc
+*****Chrono***** INFO The Final CPU consumption ( Chrono ) Table (ordered)
+ChronoStatSvc.finalize() INFO Service finalized successfully
+ApplicationMgr INFO Application Manager Finalized successfully
+ApplicationMgr INFO Application Manager Terminated successfully
+SimpleHistos SUCCESS 1D histograms in directory "SimpleHistos" : 10
+ | ID | Title | # | Mean | RMS | Skewness | Kurtosis |
+ | 101 | "Exponential" | 150 | 1.0544 | 0.96222 | 1.2397 | 1.1807 |
+ | 102 | "Breit" | 150 | -0.17526 | 1.1518 | -0.69906 | 2.2522 |
+ | 1111 | "Forced Numeric ID time test" | 150 | -0.077547 | 0.91944 | -0.29833 | -0.018186 |
+ | AutoID time test | "AutoID time test" | 150 | -0.077547 | 0.91944 | -0.29833 | -0.018186 |
+ | Gaussian mean=0, sigma=1 | "Gaussian mean=0, sigma=1" | 150 | -0.077547 | 0.91944 | -0.29833 | -0.018186 |
+ | poisson | "Poisson" | 150 | 1.838 | 1.1606 | 0.39305 | -0.75933 |
+ | subdir1/bino | "Binominal" | 150 | 1.9732 | 1.0488 | 0.26648 | -0.57997 |
+ | subdir2/bino | "Binominal" | 150 | 1.9732 | 1.0488 | 0.26648 | -0.57997 |
+ | test1 | "Forced Alpha ID time test" | 150 | -0.077547 | 0.91944 | -0.29833 | -0.018186 |
+ | varBinning/x | "1D Variable Binning" | 150 | -0.0098433 | 2.7426 | -0.027901 | -1.0865 |
+SimpleHistos SUCCESS 1D profile histograms in directory "SimpleHistos" : 9
+ | ID | Title | # | Mean | RMS | Skewness | Kurtosis |
+ | Expo V Gauss 1DProf | "Expo V Gauss 1DProf" | 150 | -0.077547 | 0.91944 | -3.0691 | 9.4945 |
+ | Expo V Gauss 1DProf s | "Expo V Gauss 1DProf s" | 150 | -0.077547 | 0.91944 | -3.0691 | 9.4945 |
+ | Gauss V Flat 1DProf | "Gauss V Flat 1DProf" | 150 | 0.38454 | 5.7038 | 0 | -3 |
+ | Gauss V Flat 1DProf S | "Gauss V Flat 1DProf S" | 150 | 0.38454 | 5.7038 | 0 | -3 |
+ | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I" | 68 | 0.38536 | 5.5289 | -0.45136 | -0.94643 |
+ | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I s" | 68 | 0.38536 | 5.5289 | -0.45136 | -0.94643 |
+ | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II" | 82 | 0.38385 | 5.8449 | 0 | -3 |
+ | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II s" | 82 | 0.38385 | 5.8449 | 0 | -3 |
+ | varBinning/a | "1D Profile Variable Binning" | 150 | -0.0098433 | 2.7426 | -4.458 | 50.149 |
diff --git a/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref
index 5fdd5a154603c82c583da1583e16bfd854153238..909e122b9275c6f4b99b014cc62ed14eacd785c4 100644
--- a/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref
+++ b/DaVinciTests/tests/refs/test_davinci_initialise_upgrade.ref
@@ -1,55 +1,48 @@
-ApplicationMgr SUCCESS
-====================================================================================================================================
-====================================================================================================================================
-ApplicationMgr INFO Application Manager Configured successfully
-RndmGenSvc.Engine INFO Generator engine type:CLHEP::RanluxEngine
-RndmGenSvc.Engine INFO Current Seed:1234567 Luxury:3
-RndmGenSvc INFO Using Random engine:HepRndm::Engine<CLHEP::RanluxEngine>
-RootHistSvc INFO Writing ROOT histograms to: DVHistos.root
-HistogramPersis... INFO Added successfully Conversion service:RootHistSvc
-NTupleSvc INFO Added stream file:DVNtuple.root as FILE1
-DaVinciInitAlg SUCCESS ==================================================================
-DaVinciInitAlg SUCCESS Requested to process 100 events
-DaVinciInitAlg SUCCESS ==================================================================
-FSROutputStream... INFO Data source: EventDataSvc output:
-EventPersistenc... INFO Added successfully Conversion service:RootCnvSvc
-ApplicationMgr INFO Application Manager Initialized successfully
-ApplicationMgr INFO Application Manager Started successfully
-EventSelector SUCCESS Reading Event record 1. Record number within stream 1: 1
-SimpleHistos INFO GaudiHistoAlgorithm:: Filling Histograms...... Please be patient !
-ApplicationMgr INFO Application Manager Stopped successfully
-DaVinciInitAlg SUCCESS ==================================================================
-DaVinciInitAlg SUCCESS 100 events processed
-DaVinciInitAlg SUCCESS ==================================================================
-GenFSRMerge INFO ========== Merging GenFSR ==========
-FSROutputStream... INFO Set up File Summary Record
-FSROutputStream... INFO Events output: 1
-EventLoopMgr INFO Histograms converted successfully according to request.
-ToolSvc INFO Removing all tools created by ToolSvc
-*****Chrono***** INFO The Final CPU consumption ( Chrono ) Table (ordered)
-ChronoStatSvc.f... INFO Service finalized successfully
-ApplicationMgr INFO Application Manager Finalized successfully
-ApplicationMgr INFO Application Manager Terminated successfully
-SimpleHistos SUCCESS 1D histograms in directory "SimpleHistos" : 10
- | ID | Title | # | Mean | RMS | Skewness | Kurtosis |
- | 101 | "Exponential" | 100 | 0.86944 | 0.83526 | 1.3231 | 1.3673 |
- | 102 | "Breit" | 100 | -0.050344 | 1.116 | -0.36005 | 1.6486 |
- | 1111 | "Forced Numeric ID time test" | 100 | 0.04085 | 1.1075 | -0.12242 | -0.32017 |
- | AutoID time test | "AutoID time test" | 100 | 0.04085 | 1.1075 | -0.12242 | -0.32017 |
- | Gaussian mean=0, sigma=1 | "Gaussian mean=0, sigma=1" | 100 | 0.04085 | 1.1075 | -0.12242 | -0.32017 |
- | poisson | "Poisson" | 100 | 1.8085 | 1.1694 | 0.26477 | -0.78547 |
- | subdir1/bino | "Binominal" | 100 | 2.0306 | 1.138 | 0.27968 | -0.82258 |
- | subdir2/bino | "Binominal" | 100 | 2.0306 | 1.138 | 0.27968 | -0.82258 |
- | test1 | "Forced Alpha ID time test" | 100 | 0.04085 | 1.1075 | -0.12242 | -0.32017 |
- | varBinning/x | "1D Variable Binning" | 100 | -0.20883 | 2.6896 | 0.18717 | -1.2165 |
-SimpleHistos SUCCESS 1D profile histograms in directory "SimpleHistos" : 9
- | ID | Title | # | Mean | RMS | Skewness | Kurtosis |
- | Expo V Gauss 1DProf | "Expo V Gauss 1DProf" | 100 | 0.04085 | 1.1075 | -1.2689 | 2.4065 |
- | Expo V Gauss 1DProf s | "Expo V Gauss 1DProf s" | 100 | 0.04085 | 1.1075 | -1.2689 | 2.4065 |
- | Gauss V Flat 1DProf | "Gauss V Flat 1DProf" | 100 | -0.53201 | 5.8217 | 11.029 | 12.795 |
- | Gauss V Flat 1DProf S | "Gauss V Flat 1DProf S" | 100 | -0.53201 | 5.8217 | 11.029 | 12.795 |
- | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I" | 52 | 0.1216 | 5.9301 | 0.32131 | -1.1125 |
- | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I s" | 52 | 0.1216 | 5.9301 | 0.32131 | -1.1125 |
- | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II" | 48 | -1.2401 | 5.6166 | 0 | -3 |
- | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II s" | 48 | -1.2401 | 5.6166 | 0 | -3 |
- | varBinning/a | "1D Profile Variable Binning" | 100 | -0.20883 | 2.6896 | 0 | -3 |
+/***** User DVAppOptions/DVAppOptions **************************************************************
+|-auditors = [] (default: [])
+|-buffer_events = 20000 (default: 20000)
+|-callgrind_profile = False (default: False)
+|-conddb_tag = 'HEAD' (default: '')
+|-control_flow_file = '' (default: '')
+|-data_flow_file = '' (default: '')
+|-data_type = 'Upgrade' (default: '')
+|-dddb_tag = 'dddb-20200424-2' (default: '')
+|-detectors = ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr']
+| (default: ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr'])
+|-dqflags_tag = '' (default: '')
+|-enable_unpack = None
+|-event_store = 'HiveWhiteBoard' (default: 'HiveWhiteBoard')
+|-evt_max = 1000 (default: -1)
+|-first_evt = 0 (default: 0)
+|-histo_file = 'DVHistos.root' (default: '')
+|-ignore_dq_flags = False (default: False)
+|-input_files = ['root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000002_1.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000004_1.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000043_1.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000068_1.ldst']
+| (default: [])
+|-input_raw_format = 0.3 (default: 0.3)
+|-input_type = 'ROOT' (default: 'DST')
+|-lines_maker = None
+|-lumi = False (default: False)
+|-main_options = '' (default: '')
+|-memory_pool_size = 10485760 (default: 10485760)
+|-merge_genfsr = False (default: False)
+|-msg_svc_format = '% F%35W%S %7W%R%T %0W%M' (default: '% F%35W%S %7W%R%T %0W%M')
+|-msg_svc_time_format = '%Y-%m-%d %H:%M:%S UTC' (default: '%Y-%m-%d %H:%M:%S UTC')
+|-n_event_slots = 1 (default: -1)
+|-n_threads = 1 (default: 1)
+|-ntuple_file = 'DVNtuple.root' (default: '')
+|-output_file = '' (default: '')
+|-output_level = 3 (default: 3)
+|-output_type = '' (default: '')
+|-overwrite_data_options = False (default: False)
+|-print_freq = 1000 (default: 1000)
+|-python_logging_level = 30 (default: 30)
+|-root_compression_level = 'LZMA:6' (default: 'LZMA:6')
+|-scheduler_legacy_mode = True (default: True)
+|-simulation = True (default: False)
+|-skip_events = 0 (default: 0)
+|-tck = 0 (default: 0)
+|-use_iosvc = False (default: False)
+|-user_algorithms = '' (default: '')
+|-write_fsr = True (default: True)
+\----- (End of User DVAppOptions/DVAppOptions) -----------------------------------------------------
+dry-run: not starting the application
diff --git a/DaVinciTests/tests/refs/test_davinci_simplejob.ref b/DaVinciTests/tests/refs/test_davinci_simplejob.ref
new file mode 100644
index 0000000000000000000000000000000000000000..9dc87929da36f2736b65b867c3fc92aeea65d506
--- /dev/null
+++ b/DaVinciTests/tests/refs/test_davinci_simplejob.ref
@@ -0,0 +1,167 @@
+/***** User DVAppOptions/DVAppOptions **************************************************************
+|-auditors = [] (default: [])
+|-buffer_events = 20000 (default: 20000)
+|-callgrind_profile = False (default: False)
+|-conddb_tag = 'HEAD' (default: '')
+|-control_flow_file = '' (default: '')
+|-data_flow_file = '' (default: '')
+|-data_type = 'Upgrade' (default: '')
+|-dddb_tag = 'dddb-20200424-2' (default: '')
+|-detectors = ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr']
+| (default: ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr'])
+|-dqflags_tag = '' (default: '')
+|-enable_unpack = None
+|-event_store = 'HiveWhiteBoard' (default: 'HiveWhiteBoard')
+|-evt_max = 100 (default: -1)
+|-first_evt = 0 (default: 0)
+|-histo_file = 'DVHistos.root' (default: '')
+|-ignore_dq_flags = False (default: False)
+|-input_files = ['root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000002_1.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000004_1.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000043_1.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000068_1.ldst']
+| (default: [])
+|-input_raw_format = 0.3 (default: 0.3)
+|-input_type = 'ROOT' (default: 'DST')
+|-lines_maker = None
+|-lumi = False (default: False)
+|-main_options = '' (default: '')
+|-memory_pool_size = 10485760 (default: 10485760)
+|-merge_genfsr = False (default: False)
+|-msg_svc_format = '% F%35W%S %7W%R%T %0W%M' (default: '% F%35W%S %7W%R%T %0W%M')
+|-msg_svc_time_format = '%Y-%m-%d %H:%M:%S UTC' (default: '%Y-%m-%d %H:%M:%S UTC')
+|-n_event_slots = 1 (default: -1)
+|-n_threads = 1 (default: 1)
+|-ntuple_file = 'DVNtuple.root' (default: '')
+|-output_file = '' (default: '')
+|-output_level = 3 (default: 3)
+|-output_type = '' (default: '')
+|-overwrite_data_options = False (default: False)
+|-print_freq = 1000 (default: 1000)
+|-python_logging_level = 30 (default: 30)
+|-root_compression_level = 'LZMA:6' (default: 'LZMA:6')
+|-scheduler_legacy_mode = True (default: True)
+|-simulation = True (default: False)
+|-skip_events = 2 (default: 0)
+|-tck = 0 (default: 0)
+|-use_iosvc = False (default: False)
+|-user_algorithms = '' (default: '')
+|-write_fsr = True (default: True)
+\----- (End of User DVAppOptions/DVAppOptions) -----------------------------------------------------
+ApplicationMgr SUCCESS
+====================================================================================================================================
+====================================================================================================================================
+ApplicationMgr INFO Application Manager Configured successfully
+NTupleSvc INFO Added stream file:DVNtuple.root as FILE1
+RootHistSvc INFO Writing ROOT histograms to: DVHistos.root
+HistogramPersistencySvc INFO Added successfully Conversion service:RootHistSvc
+DetectorDataSvc INFO Detector description not requested to be loaded
+EventClockSvc.FakeEventTime INFO Event times generated from 0 with steps of 0
+ApplicationMgr INFO Application Manager Initialized successfully
+ApplicationMgr INFO Application Manager Started successfully
+EventPersistencySvc INFO Added successfully Conversion service:RootCnvSvc
+EventSelector INFO Stream:EventSelector.DataStreamTool_1 Def:DATAFILE='root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000002_1.ldst' SVC='Gaudi::RootEvtSelector' OPT='READ' IgnoreChecksum='YES'
+EventSelector SUCCESS Reading Event record 1. Record number within stream 1: 1
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+Gaudi__Examples__VoidConsumer INFO executing VoidConsumer
+ApplicationMgr INFO Application Manager Stopped successfully
+LAZY_AND: DummyNode #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ Gaudi__Examples__VoidConsumer/Gaudi__Examples__VoidConsumer #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ToolSvc INFO Removing all tools created by ToolSvc
+ApplicationMgr INFO Application Manager Finalized successfully
+ApplicationMgr INFO Application Manager Terminated successfully
diff --git a/DaVinciTests/tests/refs/test_davinci_user_algs.ref b/DaVinciTests/tests/refs/test_davinci_user_algs.ref
new file mode 100644
index 0000000000000000000000000000000000000000..34dd13078b2f28ab33503c5a81462ab6240cc554
--- /dev/null
+++ b/DaVinciTests/tests/refs/test_davinci_user_algs.ref
@@ -0,0 +1,133 @@
+/***** User DVAppOptions/DVAppOptions **************************************************************
+|-auditors = [] (default: [])
+|-buffer_events = 20000 (default: 20000)
+|-callgrind_profile = False (default: False)
+|-conddb_tag = 'HEAD' (default: '')
+|-control_flow_file = '' (default: '')
+|-data_flow_file = '' (default: '')
+|-data_type = 'Upgrade' (default: '')
+|-dddb_tag = 'dddb-20200424-2' (default: '')
+|-detectors = ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr']
+| (default: ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr'])
+|-dqflags_tag = '' (default: '')
+|-enable_unpack = None
+|-event_store = 'HiveWhiteBoard' (default: 'HiveWhiteBoard')
+|-evt_max = 100 (default: -1)
+|-first_evt = 0 (default: 0)
+|-histo_file = 'DVHistos.root' (default: '')
+|-ignore_dq_flags = False (default: False)
+|-input_files = ['root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000002_1.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000004_1.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000043_1.ldst', 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000068_1.ldst']
+| (default: [])
+|-input_raw_format = 0.3 (default: 0.3)
+|-input_type = 'ROOT' (default: 'DST')
+|-lines_maker = None
+|-lumi = False (default: False)
+|-main_options = '' (default: '')
+|-memory_pool_size = 10485760 (default: 10485760)
+|-merge_genfsr = False (default: False)
+|-msg_svc_format = '% F%35W%S %7W%R%T %0W%M' (default: '% F%35W%S %7W%R%T %0W%M')
+|-msg_svc_time_format = '%Y-%m-%d %H:%M:%S UTC' (default: '%Y-%m-%d %H:%M:%S UTC')
+|-n_event_slots = 1 (default: -1)
+|-n_threads = 1 (default: 1)
+|-ntuple_file = 'DVNtuple.root' (default: '')
+|-output_file = '' (default: '')
+|-output_level = 3 (default: 3)
+|-output_type = '' (default: '')
+|-overwrite_data_options = False (default: False)
+|-print_freq = 1000 (default: 1000)
+|-python_logging_level = 30 (default: 30)
+|-root_compression_level = 'LZMA:6' (default: 'LZMA:6')
+|-scheduler_legacy_mode = True (default: True)
+|-simulation = True (default: False)
+|-skip_events = 2 (default: 0)
+|-tck = 0 (default: 0)
+|-use_iosvc = False (default: False)
+|-user_algorithms = '$DAVINCITESTSROOT/tests/options/option_davinci_user_algs:main' (default: '')
+|-write_fsr = True (default: True)
+\----- (End of User DVAppOptions/DVAppOptions) -----------------------------------------------------
+<module 'option_davinci_user_algs' from '/workspace/build/DaVinci/DaVinciTests/tests/options/option_davinci_user_algs.py'>
+ApplicationMgr SUCCESS
+====================================================================================================================================
+====================================================================================================================================
+ApplicationMgr INFO Application Manager Configured successfully
+NTupleSvc INFO Added stream file:DVNtuple.root as FILE1
+RootHistSvc INFO Writing ROOT histograms to: DVHistos.root
+HistogramPersistencySvc INFO Added successfully Conversion service:RootHistSvc
+FSROutputStreamDstWriter INFO Data source: EventDataSvc output: SVC='Gaudi::RootCnvSvc'
+UnpackChargedProtos.ChargedProto... INFO Using retuned RICH el and mu DLL values in combined DLLs
+DetectorDataSvc INFO Detector description not requested to be loaded
+EventClockSvc.FakeEventTime INFO Event times generated from 0 with steps of 0
+HiveDataBrokerSvc WARNING non-reentrant algorithm: RecordStream/FSROutputStreamDstWriter
+HiveDataBrokerSvc WARNING non-reentrant algorithm: GaudiHistoAlgorithm/SimpleHistos
+HiveDataBrokerSvc WARNING non-reentrant algorithm: UnpackRecVertex/UnpackRecVertices
+HiveDataBrokerSvc WARNING non-reentrant algorithm: DataPacking::Unpack<LHCb::MuonPIDPacker>/UnpackMuonPIDs
+HiveDataBrokerSvc WARNING non-reentrant algorithm: DataPacking::Unpack<LHCb::RichPIDPacker>/UnpackRichPIDs
+HiveDataBrokerSvc WARNING non-reentrant algorithm: UnpackProtoParticle/UnpackNeutralProtos
+HiveDataBrokerSvc WARNING non-reentrant algorithm: UnpackProtoParticle/UnpackChargedProtos
+ApplicationMgr INFO Application Manager Initialized successfully
+ApplicationMgr INFO Application Manager Started successfully
+EventPersistencySvc INFO Added successfully Conversion service:RootCnvSvc
+EventSelector INFO Stream:EventSelector.DataStreamTool_1 Def:DATAFILE='root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000002_1.ldst' SVC='Gaudi::RootEvtSelector' OPT='READ' IgnoreChecksum='YES'
+EventSelector SUCCESS Reading Event record 1. Record number within stream 1: 1
+RndmGenSvc.Engine INFO Generator engine type:CLHEP::RanluxEngine
+RndmGenSvc.Engine INFO Current Seed:1234567 Luxury:3
+RndmGenSvc INFO Using Random engine:HepRndm::Engine<CLHEP::RanluxEngine>
+SimpleHistos INFO GaudiHistoAlgorithm:: Filling Histograms...... Please be patient !
+ApplicationMgr INFO Application Manager Stopped successfully
+FSROutputStreamDstWriter INFO Set up File Summary Record
+FSROutputStreamDstWriter INFO Events output: 1
+UnpackRichPIDs SUCCESS #WARNINGS = 100 Message = 'Incorrect data version 0 for packing version > 3. Correcting data to version 2.'
+NONLAZY_OR: DaVinci #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ NONLAZY_OR: WriteFSR #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ RecordStream/FSROutputStreamDstWriter #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ NONLAZY_AND: DVStdAlgs #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ GaudiHistoAlgorithm/SimpleHistos #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ NONLAZY_AND: PrintJpsiNode #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ UnpackRecVertex/UnpackRecVertices #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ UnpackCaloHypo/UnpackCaloElectrons #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ UnpackCaloHypo/UnpackCaloPhotons #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ UnpackCaloHypo/UnpackCaloMergedPi0s #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ UnpackCaloHypo/UnpackCaloSplitPhotons #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ DataPacking__Unpack<LHCb__MuonPIDPacker>/UnpackMuonPIDs #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ DataPacking__Unpack<LHCb__RichPIDPacker>/UnpackRichPIDs #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ UnpackTrackFunctional/UnpackBestTracks #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ UnpackTrackFunctional/UnpackMuonTracks #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ UnpackProtoParticle/UnpackNeutralProtos #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ UnpackProtoParticle/UnpackChargedProtos #=100 Sum=100 Eff=|( 100.0000 +- 0.00000 )%|
+ToolSvc INFO Removing all tools created by ToolSvc
+*****Chrono***** INFO The Final CPU consumption ( Chrono ) Table (ordered)
+ChronoStatSvc.finalize() INFO Service finalized successfully
+ApplicationMgr INFO Application Manager Finalized successfully
+ApplicationMgr INFO Application Manager Terminated successfully
+UnpackBestTracks INFO Number of counters : 1
+ | Counter | # | sum | mean/eff^* | rms/err^* | min | max |
+ | "# Unpacked Tracks" | 100 | 41403 | 414.03 |
+UnpackMuonPIDs INFO Number of counters : 1
+ | Counter | # | sum | mean/eff^* | rms/err^* | min | max |
+ | "# UnPackedData" | 100 | 9896 | 98.960 | 47.355 | 25.000 | 264.00 |
+UnpackMuonTracks INFO Number of counters : 1
+ | Counter | # | sum | mean/eff^* | rms/err^* | min | max |
+ | "# Unpacked Tracks" | 100 | 1684 | 16.840 |
+SimpleHistos SUCCESS 1D histograms in directory "SimpleHistos" : 10
+ | ID | Title | # | Mean | RMS | Skewness | Kurtosis |
+ | 101 | "Exponential" | 100 | 1.141 | 1.0488 | 1.1417 | 0.71718 |
+ | 102 | "Breit" | 100 | -0.15117 | 1.1275 | -0.32351 | 2.0377 |
+ | 1111 | "Forced Numeric ID time test" | 100 | -0.047428 | 0.89797 | -0.25222 | 0.14476 |
+ | AutoID time test | "AutoID time test" | 100 | -0.047428 | 0.89797 | -0.25222 | 0.14476 |
+ | Gaussian mean=0, sigma=1 | "Gaussian mean=0, sigma=1" | 100 | -0.047428 | 0.89797 | -0.25222 | 0.14476 |
+ | poisson | "Poisson" | 100 | 1.8947 | 1.1376 | 0.42462 | -0.74327 |
+ | subdir1/bino | "Binominal" | 100 | 1.8788 | 1.0567 | 0.38496 | -0.44547 |
+ | subdir2/bino | "Binominal" | 100 | 1.8788 | 1.0567 | 0.38496 | -0.44547 |
+ | test1 | "Forced Alpha ID time test" | 100 | -0.047428 | 0.89797 | -0.25222 | 0.14476 |
+ | varBinning/x | "1D Variable Binning" | 100 | -0.22236 | 2.6506 | 0.026186 | -0.97273 |
+SimpleHistos SUCCESS 1D profile histograms in directory "SimpleHistos" : 9
+ | ID | Title | # | Mean | RMS | Skewness | Kurtosis |
+ | Expo V Gauss 1DProf | "Expo V Gauss 1DProf" | 100 | -0.047428 | 0.89797 | -2.3629 | 9.4136 |
+ | Expo V Gauss 1DProf s | "Expo V Gauss 1DProf s" | 100 | -0.047428 | 0.89797 | -2.3629 | 9.4136 |
+ | Gauss V Flat 1DProf | "Gauss V Flat 1DProf" | 100 | 0.18402 | 5.688 | 0 | -3 |
+ | Gauss V Flat 1DProf S | "Gauss V Flat 1DProf S" | 100 | 0.18402 | 5.688 | 0 | -3 |
+ | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I" | 45 | 0.24192 | 5.5416 | -0.26447 | -1.0149 |
+ | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-I s" | 45 | 0.24192 | 5.5416 | -0.26447 | -1.0149 |
+ | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II" | 55 | 0.13664 | 5.8046 | 0 | -3 |
+ | Gauss V Flat 1DProf, with | "Gauss V Flat 1DProf, with limits-II s" | 55 | 0.13664 | 5.8046 | 0 | -3 |
+ | varBinning/a | "1D Profile Variable Binning" | 100 | -0.22236 | 2.6506 | 7.4264 | 29.421 |
diff --git a/Phys/DaVinci/options/DaVinci-UserAlgs-Example.py b/Phys/DaVinci/options/DaVinci-UserAlgs-Example.py
deleted file mode 100644
index 4bab731b36864728a5645dfa7ad03a3f61c26884..0000000000000000000000000000000000000000
--- a/Phys/DaVinci/options/DaVinci-UserAlgs-Example.py
+++ /dev/null
@@ -1,40 +0,0 @@
-###############################################################################
-# (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration #
-# #
-# This software is distributed under the terms of the GNU General Public #
-# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
-# #
-# In applying this licence, CERN does not waive the privileges and immunities #
-# granted to it by virtue of its status as an Intergovernmental Organization #
-# or submit itself to any jurisdiction. #
-# #
-###############################################################################
-"""
- Simple DaVinci example for defining user algorithms
-"""
-__author__ = "Davide Fazzini <davide.fazzini@cern.ch>"
-
-##############################################################################################
-##### SETTINGS
-##############################################################################################
-
-from GaudiConfig2 import Configurables as C
-from PhysConf.Selections import AutomaticData
-
-
-def myfunc():
- sel = AutomaticData('/Phys/BetaSBd2JpsiKstarDetachedLine/Particles')
- inputTuple = sel.outputLocation()
-
- # make a tuple
- dtt = C.DecayTreeTuple("Bd2JpsiKstar")
- dtt.Inputs = [inputTuple]
- dtt.ToolList = ['TupleToolKinematic']
- dtt.Decay = "[B0 -> ^(J/psi(1S) -> ^mu+ ^mu-) ^(K*(892)0 -> ^K+ ^pi-)]CC"
-
- evtTuple = C.EventTuple()
- evtTuple.ToolList += ["TupleToolEventInfo"]
-
- #listAlg = [evtTuple, dtt]
- listAlg = []
- return listAlg
diff --git a/Phys/DaVinci/options/DaVinciDB-Example.yaml b/Phys/DaVinci/options/DaVinciDB-Example.yaml
index 85bd9e4d73adb95abd2d6204a024101bbb57b2b8..0c64652f196a2d79f28d717093c461a99e6f2e03 100644
--- a/Phys/DaVinci/options/DaVinciDB-Example.yaml
+++ b/Phys/DaVinci/options/DaVinciDB-Example.yaml
@@ -16,11 +16,11 @@ Upgrade_genFSR_ldst:
- 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade2.ldst'
- 'root://eoslhcb.cern.ch//eos/lhcb/cern-swtest/lhcb/swtest/genFSR/genfsr_upgrade3.ldst'
qualifiers:
- DataType: Upgrade
- InputType: LDST
- Simulation: true
- CondDBtag: sim-20171127-vc-md100
- DDDBtag: dddb-20171126
+ data_type: Upgrade
+ input_type: LDST
+ simulation: true
+ conddb_tag: sim-20171127-vc-md100
+ dddb_tag: dddb-20171126
metadata:
Author: 'dfazzini'
Date: '2020-06-17 18:34:43.57756'
@@ -33,11 +33,11 @@ Upgrade_Bd2KstarMuMu_ldst:
- 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000043_1.ldst'
- 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00076720/0000/00076720_00000068_1.ldst'
qualifiers:
- DataType: Upgrade
- InputType: LDST
- Simulation: true
- CondDBtag: sim-20171127-vc-md100
- DDDBtag: dddb-20171126
+ data_type: Upgrade
+ input_type: LDST
+ simulation: true
+ conddb_tag: HEAD
+ dddb_tag: dddb-20200424-2
metadata:
Author: 'Patrick Koppenburg'
Date: '2020-05-28 11:12:55'
diff --git a/Phys/DaVinci/options/application-option-defaults.yaml b/Phys/DaVinci/options/application-option-defaults.yaml
index 1f3917813aaa0065feca849cc005b6f9142cfb94..18fc29bed28180ef10211484d2859d060418768e 100644
--- a/Phys/DaVinci/options/application-option-defaults.yaml
+++ b/Phys/DaVinci/options/application-option-defaults.yaml
@@ -31,13 +31,19 @@ data_flow_file:
value: ''
data_type:
text: '"""Data type, can be ["Upgrade"]."""'
- value: 'Upgrade'
+ value: ''
dddb_tag:
- text: '""" Data type, can be ["Upgrade"] Forwarded to PhysConf, AnalysisConf, DstConf and LHCbApp"""'
- value: 'dddb-20171126'
+ text: '""" Data type, can be ["Upgrade"] Forwarded to PhysConf, AnalysisConf, DstConf and LHCbApp."""'
+ value: ''
+detectors:
+ text: '"""List of detectors."""'
+ value: ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr']
dqflags_tag:
- text: '"""Tag for DQFLAGS. Default as set in DDDBConf for DataType """'
+ text: '"""Tag for DQFLAGS. Default as set in DDDBConf for DataType."""'
value: ''
+enable_unpack:
+ text: '"""Explicitly enable/disable unpacking for input data (if specified)"""'
+ value:
evt_max:
text: '"""Number of events to analyse. Default = -1 to run over all events."""'
value: -1
@@ -48,13 +54,22 @@ ignore_dq_flags:
text: '"""If False, process only events with good DQ. Default = False."""'
value: False
input_files:
- text: '"""Input data. Default = []. """'
+ text: '"""Input data. Default = []."""'
value: []
input_type:
text: '"""Type of input files, e.g. "DST", "DIGI", "RDST", "MDST", "XDST" or "LDST". Default = DST."""'
value: 'DST'
+log_file:
+ text: '"""Logger used to print warning and error messages."""'
+ value:
lumi:
- text: '"""Luminosity accounting. Default = False."""'
+ text: '"""Luminosity accounting. Default = False."""'
+ value: False
+main_options:
+ text: '"""Main option file to execute."""'
+ value: ''
+merge_genfsr:
+ text: '"""Flags whether to merge the generator-level FSRs."""'
value: False
msg_svc_format:
text: '"""MessageSvc output format.Default = "% F%35W%S %7W%R%T %0W%M"."""'
@@ -68,12 +83,18 @@ ntuple_file:
output_level:
text: '"""Set the output level used in the job. Default = INFO=3."""'
value: 3
+overwrite_data_options:
+ text: '"""Allow overwriting default data options defined in DaVinci Database."""'
+ value: False
print_freq:
text: '"""Frequency at which to print event numbers. Default = 1000."""'
value: 1000
python_logging_level:
text: '"""Python logger level. Default = logging.WARNING=30."""'
value: 30
+root_compression_level:
+ text: '"""ROOT Compression level for ntuples."""'
+ value: 'LZMA:6'
skip_events:
text: '"""Number of events to skip at the beginning. Default = 0."""'
value: 0
@@ -81,8 +102,11 @@ simulation:
text: '"""Boolean to specify simulated samples. Default = False."""'
value: False
user_algorithms:
- text: '"""List of user algorithms to run. Default = []."""'
- value: []
+ text: '"""User algorithm to run. Default = ''."""'
+ value: ''
use_iosvc:
text: '"""Use an alternative, faster, IIOSvc implementation for MDFs. Default = False."""'
value: False
+write_fsr:
+ text: '"""Flags whether to write out an FSR."""'
+ value: True
\ No newline at end of file
diff --git a/Phys/DaVinci/options/jobOptions-Example.py b/Phys/DaVinci/options/jobOptions-Example.py
index 3d32fecd157d4ccd20c11381f7ce6c99c4b64fc0..807397eb2dedeec167e84c5cc26324153f5719ba 100644
--- a/Phys/DaVinci/options/jobOptions-Example.py
+++ b/Phys/DaVinci/options/jobOptions-Example.py
@@ -10,8 +10,8 @@
###############################################################################
{
- 'EvtMax': 100,
- 'SkipEvents': 2,
- 'TupleFile': 'DVNtuple.root',
- 'HistogramFile': 'DVHistos.root'
+ 'evt_max': 100,
+ 'skip_events': 2,
+ 'ntuple_file': 'DVNtuple.root',
+ 'histo_file': 'DVHistos.root'
}
diff --git a/Phys/DaVinci/options/jobOptions-Example.yaml b/Phys/DaVinci/options/jobOptions-Example.yaml
index 0c7b6afa6c165cc259f74b1903d97a4c3fb180ea..ae0f47415103fbabfe491644dd1d26c4c49527f1 100644
--- a/Phys/DaVinci/options/jobOptions-Example.yaml
+++ b/Phys/DaVinci/options/jobOptions-Example.yaml
@@ -9,7 +9,7 @@
# or submit itself to any jurisdiction. #
###############################################################################
-EvtMax: 100
-SkipEvents: 2
-TupleFile: 'DVNtuple.root'
-HistogramFile: 'DVHistos.root'
+evt_max: 100
+skip_events: 2
+ntuple_file: 'DVNtuple.root'
+histo_file: 'DVHistos.root'
\ No newline at end of file
diff --git a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py
index 8d3e81f0881c8c0ec89ee95a581a436cc202b835..817f93626044d41497d21db666b78cd9258ca3cf 100644
--- a/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py
+++ b/Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py
@@ -11,128 +11,64 @@
"""
High level configuration tools for DaVinci.
"""
+from PyConf.application import ComponentConfig, configure, configure_input
+from PyConf.control_flow import CompositeNode, NodeLogic
+from DaVinci.configOptions import (check_options, set_file_options,
+ set_option_value, set_job_options,
+ set_args_options)
+from DaVinci.algorithms import (define_log, setup_algorithms,
+ define_fsr_writer, setup_user_algorithms,
+ add_missing_configurables)
-from GaudiConfig2 import Configurables as C
-from GaudiConfig2 import mergeConfigs
-from DaVinci.configOptions import initialize_slots, check_options, set_file_options, set_job_options, set_args_options
-from DaVinci.algorithms import (define_input, define_monitors, define_log,
- define_root_files, define_services,
- define_dv_init_algo, setup_algorithms,
- define_genfsr_writer)
-from DaVinci.configurations import configure_app_mgr
-from DaVinci.utilities import dump_call, dump_configuration
-
-import click
-#import os
-#os.environ["LC_ALL"] = "en_GB.utf8"
-
-
-@click.option(
- "--key",
- default="Upgrade_Bd2KstarMuMu_ldst",
- help="Key name related to the input files listed in the TestFileDB")
-@click.option(
- "--dbfile",
- default="$DAVINCIROOT/options/DaVinciDB-Example.yaml",
- help="YAML file database containing input file infos")
-@click.option(
- "--optfile",
- default="$DAVINCIROOT/options/jobOptions-Example.yaml",
- help="YAML option file containing the job infos")
-@click.command(
- context_settings=dict(
- ignore_unknown_options=True,
- allow_extra_args=True,
- ))
-@click.pass_context
-def cli(ctx, key, dbfile, optfile):
- pass
-
-
-def mc(key="Upgrade_Bd2KstarMuMu_ldst",
- dbfile="$DAVINCIROOT/options/DaVinciDB-Example.yaml",
- optfile="$DAVINCIROOT/options/jobOptions-Example.yaml"):
+def add_davinci_configurables(options):
"""
- DaVinci function for running job with simulated samples
+ Add dedicated Davinci configurables to the main job configuration.
"""
- #dump_call(ctx, key, dbfile, optfile)
- kwargs = ""
- #if (len(ctx.args) > 1):
- # kwargs = ctx.args
+ dvNodes = []
+ if options.simulation and options.write_fsr:
+ fsrNode = define_fsr_writer(options)
+ dvNodes.append(fsrNode)
+ algsNodes = setup_algorithms(options)
+ dvNodes.extend(algsNodes)
+
+ topNode = CompositeNode(
+ "DaVinci",
+ combine_logic=NodeLogic.NONLAZY_OR,
+ children=dvNodes,
+ force_order=True)
+
+ config = ComponentConfig()
+ config.update(configure(options, topNode))
+ config.update(add_missing_configurables(options, topNode))
- return main(key, dbfile, optfile, True, kwargs)
-
-
-@click.option(
- "--key",
- default="Upgrade_Bd2KstarMuMu_ldst",
- help="Key name related to the input files listed in the TestFileDB")
-@click.option(
- "--dbfile",
- default="$DAVINCIROOT/options/DaVinciDB-Example.yaml",
- help="YAML file database containing input file infos")
-@click.option(
- "--optfile",
- default="$DAVINCIROOT/options/jobOptions-Example.yaml",
- help="YAML option file containing the job infos")
-@click.command(
- context_settings=dict(
- ignore_unknown_options=True,
- allow_extra_args=True,
- ))
-@click.pass_context
-def data(ctx, key, dbfile, optfile):
- """
- DaVinci function for running job with data samples
- """
- raise ValueError(
- 'Data file with upgrade conditions are not yet available. Please use :mc function instead.'
- )
- dump_call(ctx, key, dbfile, optfile)
- kwargs = ""
- if (len(ctx.args) > 1):
- kwargs = ctx.args
- return main(key, dbfile, optfile, False, kwargs)
+ return config
-def main(key, dbFile, jobOptFile, isMC, kwargs):
+def run_davinci(fileDB_file, fileDB_key, jobOptFile, flagMC, ctx_args,
+ simplejob):
"""
- DaVinci application main configuration.
+ Run DaVinci application.
"""
- slots = {}
- propertyDocDct = {}
- initialize_slots(slots, propertyDocDct)
-
- define_log(slots)
-
- set_file_options(slots, key, dbFile, isMC)
- set_job_options(slots, jobOptFile, key, dbFile)
- if kwargs:
- set_args_options(slots, kwargs, key, dbFile)
-
- dump_configuration(slots)
-
- check_options(slots)
- define_input(slots)
-
- config = []
- appMgr = C.ApplicationMgr()
- config.append(appMgr)
-
- define_dv_init_algo(config, slots)
-
- define_monitors(config, slots)
+ from DaVinci import options
- setup_algorithms(config, slots)
+ define_log(options)
+ set_file_options(options, fileDB_key, fileDB_file)
+ set_job_options(options, jobOptFile, fileDB_key, fileDB_file)
- define_services(config, slots)
+ if ctx_args:
+ set_args_options(options, ctx_args, fileDB_key, fileDB_file)
- define_root_files(config, slots)
+ set_option_value(options, "input_type", "ROOT")
+ check_options(options)
- define_genfsr_writer(config, slots)
+ config = configure_input(options)
- configure_app_mgr(config, slots, appMgr)
+ if simplejob:
+ node = setup_user_algorithms(options)
+ config.update(configure(options, node))
+ config.update(add_missing_configurables(options, node))
+ else:
+ config.update(add_davinci_configurables(options))
- config = mergeConfigs(config)
return config
diff --git a/Phys/DaVinci/python/DaVinci/__init__.py b/Phys/DaVinci/python/DaVinci/__init__.py
index 73ced637546a5ee4724025aad881dab36e402269..d44ee4dc5806a51bf450b1729e6dacd4d09c30ac 100644
--- a/Phys/DaVinci/python/DaVinci/__init__.py
+++ b/Phys/DaVinci/python/DaVinci/__init__.py
@@ -10,9 +10,6 @@
###############################################################################
from __future__ import absolute_import
-
-from .ConfigurationUpgrade import data, mc, main
from .config import options, run_davinci, DVNode, DVHelper
-__all__ = ('data', 'mc', 'main', 'options', 'run_davinci', 'DVNode',
- 'DVHelper')
+__all__ = ('options', 'run_davinci', 'DVNode', 'DVHelper')
diff --git a/Phys/DaVinci/python/DaVinci/algorithms.py b/Phys/DaVinci/python/DaVinci/algorithms.py
index c31fb651706c74283de4b0a91aadd70db1d92878..fdaaa12cf584e8c981f7c994256bd416a8addbb2 100644
--- a/Phys/DaVinci/python/DaVinci/algorithms.py
+++ b/Phys/DaVinci/python/DaVinci/algorithms.py
@@ -9,213 +9,222 @@
# or submit itself to any jurisdiction. #
###############################################################################
-from GaudiConfig2 import Configurables as C
-from DaVinci.configOptions import get_slot_value, set_slot_value
-from DaVinci.optionChecker import DVOptionError, DVRuntimeError
+import os, sys, importlib
+from PyConf.components import setup_component
+from PyConf.application import ComponentConfig, all_nodes_and_algs
+from PyConf.dataflow import dataflow_config
+from PyConf.control_flow import CompositeNode, NodeLogic
+from DaVinci.configOptions import get_option_value, set_option_value
+from DaVinci.optionChecker import DVOptionError
from DaVinci.configurations import *
-def define_dv_init_algo(config, slots):
+def setup_algorithms(options):
"""
- Define the DaVinci initialisation algorithm.
+ Set DaVinci algorithms
"""
- dvInit = C.DaVinciInit("DaVinciInitAlg")
- configure_init(slots, dvInit)
- config.append(dvInit)
-
+ from PyConf.Algorithms import GaudiHistoAlgorithm
-def define_input(slots):
- """
- Define the input files via the IOHelper.
- """
- from GaudiConf import IOHelper
-
- input = get_slot_value(slots, "Input")
-
- if (len(input) > 0):
- persistency = None
- inputType = get_slot_value(slots, "InputType").upper()
- if inputType == "MDF": persistency = "MDF"
- # Support connection strings and lists of files
- input = IOHelper(persistency, persistency).convertConnectionStrings(
- input, "I")
- # Clear selector to maintain the same behaviour
- IOHelper(persistency, persistency).inputFiles(input, clear=True)
+ log = get_option_value(options, "log_file")
+ opts = get_option_value(options, "main_options")
+ if not opts == "":
+ importOptions(opts)
else:
- log = get_slot_value(slots, "Log")
+ log.info(
+ "No MainOptions specified. DaVinci() will import no options file!")
+
+ stdAlgs = [
+ GaudiHistoAlgorithm(
+ name="SimpleHistos",
+ HistoPrint=True,
+ OutputLevel=get_option_value(options, "output_level"))
+ ]
+
+ stdAlgsNode = CompositeNode(
+ "DVStdAlgs", children=stdAlgs, combine_logic=NodeLogic.NONLAZY_AND)
+
+ userAlgs = get_option_value(options, 'user_algorithms')
+ if not userAlgs:
log.warning(
- "No input files available along with the selected key. Check the related DaVinciDB."
+ "DV option file or main function not defined. No user algorithms will be used."
)
+ return [stdAlgsNode]
+ else:
+ userAlgsNode = setup_user_algorithms(options)
+ return [stdAlgsNode, userAlgsNode]
-def define_monitors(config, slots):
+def define_log(options):
+ """
+ Define the logger.
"""
- Define the monitors:
+ from AnalysisPython.Logger import getLogger
+
+ log = getLogger("DaVinci")
+ set_option_value(options, "log_file", log)
+
- - AuditorSvc
- - EventSelector
- - SequencerTimerTool
- - TimingAuditor
- - EventClock.
+def define_fsr_writer(options):
"""
- auditor = C.AuditorSvc()
- timer = C.TimingAuditor()
- sequencer = C.SequencerTimerTool()
- configure_timing(slots, auditor, sequencer, timer)
+ Define Generator FSR writer
+ """
+ from PyConf.Algorithms import GenFSRMerge, RecordStream
+
+ algs = []
+ if get_option_value(options, "merge_genfsr"):
+ mergeGenfsr = GenFSRMerge(name="GenFSRMerge")
+ algs.append(mergeGenfsr)
+
+ outputLevel = get_option_value(options, "output_level")
+ if outputLevel == '':
+ outputLevel = INFO
- evtSel = C.EventSelector()
- configure_event_selector(slots, evtSel)
+ recStream = RecordStream(
+ name="FSROutputStreamDstWriter",
+ OutputLevel=outputLevel,
+ Output="SVC='Gaudi::RootCnvSvc'")
+ algs.append(recStream)
+ fsrNode = CompositeNode(
+ "WriteFSR",
+ combine_logic=NodeLogic.NONLAZY_OR,
+ children=algs,
+ force_order=True)
- #evtTime = C.EventClockSvc(EventTimeDecoder="OdinTimeDecoder")
- config.extend([auditor, timer, sequencer, evtSel]) #, evtTime])
+ return fsrNode
-def setup_algorithms(config, slots):
+def setup_user_algorithms(options):
"""
- Set DaVinci algorithms
+ Set user algorithms and return a PyConf node
"""
- import importlib, sys, os
- stdAlgorithms = [
- C.GaudiHistoAlgorithm(
- "SimpleHistos",
- HistoPrint=True,
- OutputLevel=get_slot_value(slots, "OutputLevel"))
- ]
+ userAlgs = get_option_value(options, 'user_algorithms')
+ log = get_option_value(options, "log_file")
- fileAlg = get_slot_value(slots, 'UserAlgorithmFile')
- funcName = get_slot_value(slots, 'UserAlgorithm')
+ if userAlgs == "":
+ log.warning(
+ "DV option file or main function not defined. No user algorithms will be used."
+ )
+ # Add a dummy algorithm in order to avoid errors from empty nodes
+ from PyConf.Algorithms import Gaudi__Examples__VoidConsumer as VoidConsumer
+ alg = VoidConsumer()
+ userAlgsNode = CompositeNode("DummyNode", children=[alg])
+ else:
+ modulePath = userAlgs.rsplit('/', 1)[0]
+ algName = userAlgs.rsplit('/', 1)[1]
- modulePath = fileAlg.rsplit('/', 1)[0]
- moduleName = fileAlg.rsplit('/', 1)[1]
+ modulePath = os.path.expandvars(modulePath)
+ sys.path.append(modulePath)
- modulePath = os.path.expandvars(modulePath)
- _, ext = os.path.splitext(moduleName)
- if ext == ".py":
- moduleName = moduleName.rsplit('.', 1)[0]
+ _, ext = os.path.splitext(algName)
+ if ext == ".py":
+ algName = algName.rsplit('.', 1)[0]
- sys.path.append(modulePath)
- log = get_slot_value(slots, "Log")
+ if ":" in algName:
+ moduleName = algName.rsplit(":", 1)[0]
+ funcName = ".%s" % algName.rsplit(":", 1)[1]
+ else:
+ moduleName = algName
+ funcName = ""
- if moduleName and funcName:
try:
module = importlib.import_module(moduleName)
except:
raise DVOptionError(
- "UserAlgorithmFile",
+ "user_algorithms",
"Importing user algorithms failed. Check if the user python module %s defined in %s exists!"
% (moduleName, modulePath))
- algorithms = stdAlgorithms
else:
+ print(module)
try:
- userAlgorithms = eval("module.%s()" % funcName)
- log.info("User algorithm %s.%s imported successfully!" %
- (moduleName, funcName))
- algorithms = stdAlgorithms + userAlgorithms
+ userAlgsNode = eval("module%s()" % funcName)
except:
- raise DVRuntimeError(
- "UserAlgorithmFile", "UserAlgorithm",
+ raise DVOptionError(
+ "user_algorithms",
"Run time error when calling the user algorithm. User algorithm %s can not be imported!"
% funcName)
- algorithms = stdAlgorithms
- else:
- log.warning(
- "DV option file or main function not defined. No user algorithms will be used."
- )
- algorithms = stdAlgorithms
-
- config.extend(algorithms)
-
-
-def define_services(config, slots):
- """
- Define standard services:
-
- - AlgContextSvc
- - EvtDataSvc
- - EvtPersistencySvc
- - FileRecordDataSvc
- - IODataManager
- - LoKiSvc
- - MultiFileCatalog
- - RootCnvSvc
- - ToolSvc
- - XmlCnvSvc
- - XmlParserSvc
- """
-
- evtSvc = C.EvtDataSvc()
- evtPers = C.EvtPersistencySvc("EventPersistencySvc")
- configure_event(evtSvc, evtPers)
-
- algSvc = C.AlgContextSvc()
- algSvc.BypassIncidents = True
+ else:
+ log.info("User algorithm %s%s imported successfully!" %
+ (moduleName, funcName))
- rootSvc = C.Gaudi.RootCnvSvc("FileRecordCnvSvc")
- fileDataSvc = C.FileRecordDataSvc()
+ return userAlgsNode
+
+
+def add_missing_configurables(options,
+ topNode,
+ public_tools=[],
+ barrier_algorithms=[]):
+ """
+ Temporary function used for including missing configurables in the DaVinci job.
+ The reason of these missing configurables is probably due to the different Gaudi call
+ ('./run davinci' instead of './run gaudirun.py') and from the 'configure' method
+ implemented in PyConf: indeed the method doesn't return all the configurables stored
+ in 'configurable_algs' and 'configurable_tools'.
+ TO BE FIXED.
+ """
+ from Configurables import (
+ Gaudi__RootCnvSvc as RootCnvSvc, XmlCnvSvc, FileRecordDataSvc,
+ XmlParserSvc, Gaudi__MultiFileCatalog as MultiFileCatalog,
+ LHCb__DetDesc__ReserveDetDescForEvent as reserveIOV,
+ LHCb__Tests__FakeEventTimeProducer as DummyEventTime)
+ from DDDB.CheckDD4Hep import UseDD4Hep
+ if UseDD4Hep:
+ from Configurables import LHCb__Det__LbDD4hep__IOVProducer as IOVProducer
+
+ options.finalize()
+ config = ComponentConfig()
+
+ nodes, algs = all_nodes_and_algs(topNode)
+ configuration = dataflow_config()
+ for alg in algs:
+ configuration.update(alg.configuration())
+ for tool in public_tools:
+ configuration.update(tool.configuration())
+ configurable_algs, configurable_tools = configuration.apply()
+
+ INITIAL_TIME = 1433509200
+ odin_loc = "/Event/DAQ/DummyODIN"
+ configurable_algs += [
+ setup_component(
+ DummyEventTime,
+ "DummyEventTime",
+ Start=INITIAL_TIME,
+ Step=0,
+ ODIN=odin_loc,
+ require_IOVLock=False),
+ setup_component(
+ reserveIOV, "reserveIOV", require_IOVLock=False, ODIN=odin_loc)
+ ]
+ if UseDD4Hep:
+ configurable_algs += [
+ setup_component(
+ IOVProducer,
+ "ReserveIOVDD4hep",
+ require_IOVLock=False,
+ SliceLocation="IOVLockDD4hep",
+ ODIN=odin_loc)
+ ]
+
+ rootSvc = RootCnvSvc("RootCnvSvc")
+ fileDataSvc = FileRecordDataSvc()
configure_file_record_data(rootSvc, fileDataSvc)
- xmlSvc = C.XmlCnvSvc()
- xmlParser = C.XmlParserSvc()
+ xmlSvc = XmlCnvSvc()
+ xmlParser = XmlParserSvc()
configure_xml(xmlSvc, xmlParser)
- extSvc = [
- evtSvc,
- evtPers,
- algSvc,
- fileDataSvc,
- rootSvc,
- xmlSvc,
- C.LoKiSvc(),
- C.ToolSvc(),
- C.Gaudi.Monitoring.MessageSvcSink(),
- C.Gaudi.MultiFileCatalog("FileCatalog"),
- C.Gaudi.IODataManager("IODataManager"),
- ]
-
- config.extend(extSvc)
-
-
-def define_root_files(config, slots):
- """
- Set the output ROOT file names for ntuples and histograms.
-
- Note:
- Ntuples and histograms can be output to different files, and are so typically.
- """
- from GaudiKernel.Configurable import ConfigurableGeneric as RFileCnv
-
- if get_slot_value(slots, "HistogramFile"):
- histoSvc = C.HistogramPersistencySvc(
- OutputFile=get_slot_value(slots, "HistogramFile"))
- config.append(histoSvc)
-
- if get_slot_value(slots, "TupleFile"):
- ntSvc = C.NTupleSvc()
- configure_options_root_ntuples(slots, ntSvc)
- config.append(ntSvc)
-
- # Set the compression level for the ROOT tuple file
- RFileCnv("RFileCnv").GlobalCompression = get_slot_value(
- slots, "RootCompressionLevel")
-
-
-def define_log(slots):
- """
- Define the logger.
- """
- from AnalysisPython.Logger import getLogger
-
- log = getLogger("DaVinci")
- set_slot_value(slots, "Log", log)
-
-
-def define_genfsr_writer(config, slots):
- """
- Define Generator FSR writer
- """
- if get_slot_value(slots, "MergeGenFSR"):
- config.append(C.GenFSRMerge())
- config.append(
- C.RecordStream(
- "FSROutputStreamDstWriter",
- OutputLevel=get_slot_value(slots, "OutputLevel")))
+ config.add(setup_component("ToolSvc"))
+ config.add(
+ setup_component(
+ "EventPersistencySvc",
+ CnvServices=["Gaudi::RootCnvSvc/RootCnvSvc"]))
+ config.add(fileDataSvc)
+ config.add(rootSvc)
+ config.add(xmlSvc)
+ config.add(xmlParser)
+
+ for alg in configurable_algs:
+ config.add(alg)
+ for tool in configurable_tools:
+ config.add(tool)
+
+ return config
diff --git a/Phys/DaVinci/python/DaVinci/configOptions.py b/Phys/DaVinci/python/DaVinci/configOptions.py
index bb0a7a9b84c21a4920a9ae24c74982375d7186e2..90a02b3f9a1b9dbc60159c740dbc09c1e9cf3206 100644
--- a/Phys/DaVinci/python/DaVinci/configOptions.py
+++ b/Phys/DaVinci/python/DaVinci/configOptions.py
@@ -9,115 +9,65 @@
# or submit itself to any jurisdiction. #
###############################################################################
"""
-Set and retrieve the value of a specific slot property.
+Set and retrieve the value of a specific option property.
"""
import os, difflib
from DaVinci.optionChecker import DVOptionError, DVRuntimeError, option_checker
-def get_slot_value(slots, name):
- if if_slot_exists(slots, name):
- return slots[name]
- else:
- return None
-
+def get_option_value(options, name):
+ return options.getProp(name)
-def set_slot_value(slots, name, value):
- if if_slot_exists(slots, name):
- slots[name] = value
+def set_option_value(options, name, value):
+ options.setProp(name, value)
-def if_slot_exists(slots, name):
- if name in slots:
- return True
- else:
- raise DVOptionError(
- name,
- "Unknown qualifier called %s. Please check the DaVinci database or the job option file!\nList of possible valid qualifiers: %s"
- % (name, difflib.get_close_matches(name, slots)))
- return False
-
-def get_default_value(inputKey):
- """
- Get the default value for a specific property.
- """
- import yaml
-
- optsDefaultName = "$DAVINCIROOT/options/DVconfig-Default.yaml"
- with open(os.path.expandvars(optsDefaultName)) as optsDefault:
- config = yaml.safe_load(optsDefault)
-
- for key, args in config.items():
- if key == inputKey:
- for name, value in args.items():
- if name == 'value':
- return value
- return 0
-
-
-def initialize_slots(slots, propDct):
- """
- Initialize the properties with default values.
- """
- import yaml
-
- optsDefaultName = "$DAVINCIROOT/options/DVconfig-Default.yaml"
- with open(os.path.expandvars(optsDefaultName)) as optsDefault:
- config = yaml.safe_load(optsDefault)
-
- for key, args in config.items():
- for name, value in args.items():
- if name == 'value':
- slots[key] = value
- else:
- propDct[key] = value
-
-
-def set_file_options(slots, inputKey, dbName, isMC):
+def set_file_options(options, fileDB_key, fileDB_file):
"""
Set the dataset properties required by the user.
"""
import yaml
- with open(os.path.expandvars(dbName)) as dbFile:
+ with open(os.path.expandvars(fileDB_file)) as dbFile:
dataDV = yaml.safe_load(dbFile)
idxFile = -1
- if ":" in inputKey:
- idxFile = int(inputKey.split(":")[1])
- inputKey = inputKey.split(":")[0]
+ if ":" in fileDB_key:
+ idxFile = int(fileDB_key.split(":")[1])
+ fileDB_key = fileDB_key.split(":")[0]
for key, config in dataDV.items():
- if key == inputKey:
+ if key == fileDB_key:
for item, obj in config.items():
if item == 'qualifiers':
for prop, value in obj.items():
- set_slot_value(slots, prop, value)
+ set_option_value(options, prop, value)
elif item == 'filenames':
if idxFile > -1:
if idxFile < len(obj):
- set_slot_value(slots, "Input", [obj[idxFile]])
+ set_option_value(options, "input_files",
+ [obj[idxFile]])
else:
raise ValueError(
'Index file exceeds the number of available files related to the given key!'
)
else:
- set_slot_value(slots, "Input", obj)
+ set_option_value(options, "input_files", obj)
-def set_job_options(slots, configName, inputKey, dbName):
+def set_job_options(options, jobOptFile, fileDB_key, fileDB_file):
"""
Set the job properties required by the user.
"""
- log = get_slot_value(slots, "Log")
- if configName == '':
+ log = get_option_value(options, "log_file")
+ if jobOptFile == '':
log.warning('No jobOption file selected, the default values are used.')
else:
- dataOptions = list_data_options(inputKey.split(":")[0], dbName)
+ dataOptions = list_data_options(fileDB_key.split(":")[0], fileDB_file)
- with open(os.path.expandvars(configName)) as config_file:
- _, ext = os.path.splitext(configName)
+ with open(os.path.expandvars(jobOptFile)) as config_file:
+ _, ext = os.path.splitext(jobOptFile)
if ext in (".yaml", ".yml", ".json"):
import yaml
config = yaml.safe_load(config_file)
@@ -130,37 +80,37 @@ def set_job_options(slots, configName, inputKey, dbName):
)
for key, value in config.items():
- if is_option_settable(slots, key, dataOptions):
- set_slot_value(slots, key, value)
+ if is_option_settable(options, key, dataOptions):
+ set_option_value(options, key, value)
-def set_args_options(slots, kwargs, inputKey, dbName):
+def set_args_options(options, ctx_args, fileDB_key, fileDB_file):
"""
Set the extra arguments required by the user.
"""
- dataOptions = list_data_options(inputKey.split(":")[0], dbName)
- for i in range(1, len(kwargs), 2):
- key = kwargs[i][2:]
- value = kwargs[i + 1]
+ dataOptions = list_data_options(fileDB_key.split(":")[0], fileDB_file)
+ for i in range(0, len(ctx_args), 2):
+ key = ctx_args[i][2:]
+ value = ctx_args[i + 1]
- if is_option_settable(slots, key, dataOptions):
- set_slot_value(slots, key, value)
+ if is_option_settable(options, key, dataOptions):
+ set_option_value(options, key, value)
-def is_option_settable(slots, key, dataOptions):
+def is_option_settable(options, key, dataOptions):
"""
Check if the current option 'key' can be set corretly
"""
- log = get_slot_value(slots, "Log")
+ log = get_option_value(options, "log_file")
if key in dataOptions:
- if get_slot_value(slots, "OverwriteDataOptions"):
+ if get_option_value(options, "overwrite_data_options"):
log.info(
"New value found for the option %s in the job option file. 'OverwriteDataOptions' is active so the default value will be overwritten."
% key)
return True
else:
raise DVRuntimeError(
- key, "OverwriteDataOptions",
+ key, "overwrite_data_options",
"Default value for option %s is taken from DaVinciDB.\nA new value is found in the job option file but 'OverwriteDataOptions' is not active so the default value can't be overwritten!"
% key)
return False
@@ -168,18 +118,18 @@ def is_option_settable(slots, key, dataOptions):
return True
-def list_data_options(inputKey, dbName):
+def list_data_options(fileDB_key, fileDB_file):
"""
List of the properties that are set automatically given a dataset.
"""
import yaml
- with open(os.path.expandvars(dbName)) as dbFile:
+ with open(os.path.expandvars(fileDB_file)) as dbFile:
dataDV = yaml.safe_load(dbFile)
optionList = []
for key, config in dataDV.items():
- if key == inputKey:
+ if key == fileDB_key:
for item, obj in config.items():
if item == 'qualifiers':
for prop, value in obj.items():
@@ -188,49 +138,36 @@ def list_data_options(inputKey, dbName):
return optionList
-def check_options(slots):
+def check_options(options):
"""
Check the options. Applies changes if needed.
"""
- dataType = get_slot_value(slots, "DataType")
- option_checker("DataType", dataType)
-
- inputType = get_slot_value(slots, "InputType").upper()
- option_checker("InputType", inputType)
+ dataType = get_option_value(options, "data_type")
+ option_checker("data_type", dataType)
- redoLink = get_slot_value(slots, "RedoMCLinks")
- isMC = get_slot_value(slots, "Simulation")
+ inputType = get_option_value(options, "input_type").upper()
+ option_checker("input_type", inputType)
- if not isMC:
- if redoLink:
- raise DVRuntimeError(
- "RedoMCLinks", "Simulation",
- "Re-doing MC links not possible for data! Set RedoMCLinks = False."
- )
+ flagMC = get_option_value(options, "simulation")
- if get_slot_value(slots, "MergeGenFSR"):
+ if not flagMC:
+ if get_option_value(options, "merge_genfsr"):
raise DVRuntimeError(
- "MergeGenFSR", "Simulation",
- "GenFSR are not available in real data so MergeGenFSR cannot be run! Set MergeGenFSR = False."
+ "merge_genfsr", "simulation",
+ "GenFSR are not available in real data so merge_genfsr cannot be run! Set merge_genfsr = False."
)
else:
- if redoLink and inputType("MDF", "DIGI", "MDST", "RDST"):
- raise DVRuntimeError(
- "RedoMCLinks", "InputType",
- "Re-doing MC links not possible for %s input type! Set RedoMCLinks = False."
- % inputType)
-
- if get_slot_value(slots, "Lumi"):
+ if get_option_value(options, "lumi"):
raise DVRuntimeError(
- "Lumi", "Simulation",
- "Lumi not valid for Simulation! Set Lumi = False.")
+ "Lumi", "simulation",
+ "Lumi not valid for simulation! Set lumi = False.")
## for simulation, it is very important to specify proper DB-tags:
- if not get_slot_value(slots, 'DDDBtag'):
+ if not get_option_value(options, 'dddb_tag'):
raise DVOptionError(
- "DDDBtag",
- "``DDDBtag'' is not specified for simulated data!", "")
- if not get_slot_value(slots, 'CondDBtag'):
+ "dddb_tag",
+ "``dddb_tag'' is not specified for simulated data!", "")
+ if not get_option_value(options, 'conddb_tag'):
raise DVOptionError(
- "CondDBtag",
- "``CondDBtag'' is not specified for simulated data!", "")
+ "conddb_tag",
+ "``conddb_tag'' is not specified for simulated data!", "")
diff --git a/Phys/DaVinci/python/DaVinci/configurations.py b/Phys/DaVinci/python/DaVinci/configurations.py
index 43aed758699d638e2e2bfdd84bfb6e7eb5d1b132..8c59a5d177251ed51e1c48080a201ed906f954c5 100644
--- a/Phys/DaVinci/python/DaVinci/configurations.py
+++ b/Phys/DaVinci/python/DaVinci/configurations.py
@@ -12,103 +12,11 @@
Define configurations for applications, algorithms and external services.
"""
-from DaVinci.configOptions import get_slot_value, get_default_value
-
-
-def configure_app_mgr(config, slots, appMgr):
- """
- Configuration of the main application.
- """
- messageSvc = 0
- for c in config:
- if c.name == 'MessageSvc':
- messageSvc = c
-
- appMgr.EvtMax = get_slot_value(slots, "EvtMax")
- appMgr.HistogramPersistency = 'ROOT'
- appMgr.AuditAlgorithms = True
-
- propagate_properties_to_lhcb(config, slots)
-
- appMgr.ExtSvc = [
- c for c in config if c.__component_type__ == 'Service'
- and c.name not in ['ApplicationMgr']
- ]
- appMgr.TopAlg = [
- c for c in config if c.__component_type__ == 'Algorithm'
- and c.name not in ['ApplicationMgr']
- ]
-
-
-def configure_init(slots, dvInit):
- """
- Configuration of initialisation algorithm.
- """
- dvInit.Increment = get_slot_value(slots, "PrintFreq")
-
-
################################################################################
# Define configurations for external services
#
-def configure_timing(slots, auditor, sequencer, timer):
- """
- Configuration of timing auditors.
- """
- auditor.Auditors = ['ChronoAuditor', 'TimingAuditor']
- sequencer.OutputLevel = get_slot_value(slots, "OutputLevel")
-
-
-def configure_options_root_ntuples(slots, svc):
- """
- Configuration of output .root ntuples.
- """
- for _line in svc.Output:
- if 0 <= _line.find('FILE1'):
- log = get_slot_value(slots, "Log")
- log.warning('Replace NTuple-LUN FILE1: ' + _line)
- svc.Output.remove(_line)
- tupleFile = get_slot_value(slots, "TupleFile")
- tupleStr = "FILE1 DATAFILE='%s' TYP='ROOT' OPT='NEW'" % tupleFile
- svc.Output += [tupleStr]
- svc.OutputLevel = get_slot_value(slots, "OutputLevel")
-
-
-def configure_event(evtSvc, svc):
- """
- Configuration of the event services.
- """
- evtSvc.EnableFaultHandler = True
- evtSvc.ForceLeaves = True
- evtSvc.RootCLID = 1
- svc.CnvServices.append("Gaudi::RootCnvSvc/RootCnvSvc")
-
-
-def configure_event_selector(slots, svc):
- """
- Configuration of the event selector.
- """
- printFreq = get_slot_value(slots, "PrintFreq")
- if printFreq == 0:
- printFreq = get_default_value("PrintFreq")
- log = get_slot_value(slots, "Log")
- log.warning("Print frequency cannot be 0. Set to %f." % printFreq)
- svc.PrintFreq = printFreq
-
-
-#def configure_detector(detSvc, detPers):
-# """
-# Configure Detector services.
-# """
-# detSvc.DetDbLocation = "git:/lhcb.xml"
-# detSvc.DetDbRootName = "dd"
-# detSvc.DetStorageType = 7
-# detSvc.UsePersistency = True
-
-# detPers.CnvServices = ["XmlCnvSvc/XmlCnvSvc"]
-
-
def configure_file_record_data(rootSvc, fileDataSvc):
"""
Configure File Record Data service.
@@ -132,60 +40,3 @@ def configure_xml(xmlSvc, xmlParser):
xmlParser.CacheBehavior = 3
xmlParser.EntityResolver = "EntityResolverDispatcher/EntityResolverDispatcher"
xmlParser.MaxDocNbInCache = 15
-
-
-################################################################################
-# Define configurations for MC algorithms
-#
-
-
-def configure_MC_algorithms(slots, dataSvc):
- """
- Configure DaVinciAssociators and do MC unpacking.
- """
- root = ""
- if get_slot_value(slots, "InputType") == "MDST" and get_slot_value(
- slots, "RootInTES"):
- root = get_slot_value(slots, "RootInTES")
- else:
- root = "/Event"
-
- configure_MC_unpacking(slots, dataSvc, root)
-
-
-def configure_MC_unpacking(slots, dataSvc, root):
- """
- Configure the unpacking of MC particles.
- """
- import os
-
- log = get_slot_value(slots, "Log")
- mcRoot = os.path.join(root, "MC")
- log.info("Will unpack MC objects to {0}".format(mcRoot))
-
- #particlesOutput = os.path.join(mcRoot, "Particles")
- #verticesOutput = os.path.join(mcRoot, "Vertices")
-
- #samba = C.UnpackMCParticle("UnpackMCParticle", RootInTES=root)
- dataSvc.NodeMap[mcRoot] = "DataObject"
- #dataSvc.AlgMap[particlesOutput] = C.UnpackMCParticle("UnpackMCParticle", RootInTES=root)
- #dataSvc.AlgMap[verticesOutput] = C.UnpackMCVertex("UnpackMCVertex", RootInTES=root)
-
-
-def propagate_properties_to_lhcb(config, slots):
- """
- Define and configure LHCbApp with the relevant DV properties.
- """
- from Configurables import LHCbApp
-
- lhcbApp = LHCbApp()
- properties = [
- "EvtMax", "SkipEvents", "DataType", "CondDBtag", "DDDBtag",
- "DQFLAGStag", "Simulation", "IgnoreDQFlags"
- ]
-
- for prop in properties:
- if get_slot_value(slots, prop):
- lhcbApp.setProp(prop, get_slot_value(slots, prop))
-
- #config.append(lhcbApp)
diff --git a/Phys/DaVinci/python/DaVinci/optionChecker.py b/Phys/DaVinci/python/DaVinci/optionChecker.py
index 9b21a3b31efe95a81a1e4155a0d5acd0c9ef8414..6ebd36c1d9ae87280edcf33f9f7b53d9f9454804 100644
--- a/Phys/DaVinci/python/DaVinci/optionChecker.py
+++ b/Phys/DaVinci/python/DaVinci/optionChecker.py
@@ -69,13 +69,13 @@ def option_checker(name, value):
def get_allowed_option_values():
allowedValues = {
- "DataType": ["Upgrade"],
- "Detectors": [
+ "data_type": ["Upgrade"],
+ "detectors": [
'VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon',
'Magnet', 'Tr'
],
- "InputType":
- ["MDF", "DST", "DIGI", "RDST", "MDST", "SDST", "XDST", "LDST"]
+ "input_type":
+ ["MDF", "DST", "DIGI", "RDST", "MDST", "SDST", "XDST", "LDST", "ROOT"]
}
return allowedValues
diff --git a/Phys/DaVinci/python/DaVinci/utilities.py b/Phys/DaVinci/python/DaVinci/utilities.py
deleted file mode 100644
index 61c9d608c5432d9d84e1c327478da66a6adc90dc..0000000000000000000000000000000000000000
--- a/Phys/DaVinci/python/DaVinci/utilities.py
+++ /dev/null
@@ -1,87 +0,0 @@
-###############################################################################
-# (c) Copyright 2020-2021 CERN for the benefit of the LHCb Collaboration #
-# #
-# This software is distributed under the terms of the GNU General Public #
-# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
-# #
-# In applying this licence, CERN does not waive the privileges and immunities #
-# granted to it by virtue of its status as an Intergovernmental Organization #
-# or submit itself to any jurisdiction. #
-###############################################################################
-
-from DaVinci.configOptions import get_slot_value
-import os, click
-
-
-def dump_call(ctx, key, dbfile, optfile):
- indentStr = "# "
- click.echo("%sDaVinci: running using the following arguments:" % indentStr)
- click.echo("%s - key: {key}" % indentStr)
- click.echo("%s - dbfile: {dbfile}" % indentStr)
- click.echo("%s - optfile: {optfile}" % indentStr)
-
- if len(ctx.args) > 1:
- for i in range(1, len(ctx.args), 2):
- click.echo("%s - %s: %s" % (indentStr, ctx.args[i][2:],
- ctx.args[i + 1]))
-
-
-def dump_configuration(slots):
- """
- Dump the DaVinci Configuration.
- """
- indentStr = "# "
- appName = "DaVinci"
- preLen = 10
- headerWidth = 100
-
- log = get_slot_value(slots, "Log")
-
- header = "Applying %s configuration" % appName
- dump = indentStr + header + os.linesep
-
- title = "User %s/%s" % (appName, appName)
- postLen = get_post_length(title, preLen, headerWidth)
-
- dump += "%s%s %s %s" % (indentStr, preLen * '*', title,
- postLen * '*') + os.linesep
- dump += get_prop_infos(slots, indentStr)
-
- footer = "(End of User %s/%s)" % (appName, appName)
- postLen = get_post_length(footer, preLen, headerWidth)
- dump += "%s%s %s %s" % (indentStr, preLen * '-', footer, postLen * '-')
- log.info(dump)
-
-
-def get_prop_infos(slots, indentStr):
- """
- Dump the DV configuration with the name and value of each property.
- """
- nameWidth = get_name_width(slots)
- config = ""
-
- for prop in slots:
- prefix = "%s|-%s" % (indentStr, prop.ljust(nameWidth))
- value = get_slot_value(slots, prop)
- line = "%s = %s" % (prefix, repr(value))
- config += line + os.linesep
- return config
-
-
-def get_name_width(slots):
- """
- Get the maximum length of the property names.
- """
- nameWidth = 0
- for slot in slots:
- nameWidth = max(nameWidth, len(slot))
- return nameWidth
-
-
-def get_post_length(name, preLen, width):
- """
- Get the number of remaining blank spaces after 'name' given the specific width.
- """
- postLen = width - preLen - len(name)
- postLen = max(preLen, postLen)
- return postLen
diff --git a/Phys/DaVinci/python/DaVinci/utilities_script.py b/Phys/DaVinci/python/DaVinci/utilities_script.py
new file mode 100644
index 0000000000000000000000000000000000000000..d5d8b4c8678bf8332c5d73b094450336f060535e
--- /dev/null
+++ b/Phys/DaVinci/python/DaVinci/utilities_script.py
@@ -0,0 +1,79 @@
+###############################################################################
+# (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+"""
+This module contains some functions useful while running the davinci script.
+"""
+
+
+def dump_call(testfiledb_file, testfiledb_key, joboptfile, ctx_args):
+ """
+ Print out all the davinci options used to run the job by means of the
+ 'davinci' script.
+
+ Args:
+ - testfiledb_file: TestFileDB-like file with job input information (1st davinci argument, 1st slot)
+ - testfiledb_key: relevant key to be looked for in the TestFileDB-like file (1st davinci argument, 2nd slot)
+ - joboptfile: file containing the job option information (2nd davinci argument)
+ - ctx_args: array with the remnant options passed to davinci by command line
+ """
+ import os, click
+
+ indentStr = "# "
+ click.echo("%sDaVinci: running using the following arguments:" % indentStr)
+ click.echo(
+ "%s - testfiledb-file: %s" % (indentStr, testfiledb_file))
+ click.echo(
+ "%s - testfiledb-key: %s" % (indentStr, testfiledb_key))
+ click.echo("%s - joboption-file: %s" % (indentStr, joboptfile))
+
+ if len(ctx_args) > 1:
+ for i in range(0, len(ctx_args), 2):
+ click.echo("%s - %s: %s" % (indentStr, ctx_args[i][2:],
+ ctx_args[i + 1]))
+
+
+def get_configurable_opts(configurables, with_defaults):
+ """
+ Temporary method to be used until GaudiConfig2 will be not implemented.
+ Loop over the list of configurables to be passed to Gaudi and convert them
+ in a dictionary.
+
+ Args:
+ - configurables: list of configurables to be passed to Gaudi
+ - with_defaults: flag to set the default values
+
+ Ouput:
+ - opts: dictionary containing all the configurables
+ """
+ import sys
+ from itertools import chain
+ from GaudiKernel.Proxy.Configurable import Configurable
+
+ opts = {}
+ for c in configurables:
+ items = (chain(c.getDefaultProperties().items(),
+ c.getValuedProperties().items())
+ if with_defaults else c.getValuedProperties().items())
+
+ for p, v in items:
+ if hasattr(Configurable, "PropertyReference") and isinstance(
+ v, Configurable.PropertyReference):
+ v = v.__resolve__()
+ if isinstance(v, str):
+ v = '"%s"' % v.replace('"', '\\"')
+ elif sys.version_info < (3, ) and isinstance(v, long):
+ v = '%d' % v
+ elif hasattr(v, '__opt_value__'):
+ v = v.__opt_value__()
+
+ opts['.'.join((c.name(), p))] = str(v)
+
+ return opts