Merge branch 'rm-additional-algs' into 'master'

Generalize profiling algorithms configuration with HLTControlFlowMgr

See merge request !3919

Det/VPDet/tests/data/velo_motion_from_yaml_pyconf.ref

 /***** User ApplicationOptions/ApplicationOptions **************************************************
+|-preamble_algs                           = []  (default: [])
 |-append_decoding_keys_to_output_manifest = True  (default: True)
 |-auditors                                = []  (default: [])
 |-buffer_events                           = 20000  (default: 20000)
-|-callgrind_profile                       = False  (default: False)
 |-conddb_tag                              = 'upgrade/sim-20220705-vc-mu100'  (default: '')
 |-control_flow_file                       = ''  (default: '')
 |-data_flow_file                          = ''  (default: '')

GaudiConf/python/GaudiConf/LbExec/options.py

@@ -105,7 +105,7 @@ class Options(BaseModel):
     control_flow_file: Optional[str] = None
     data_flow_file: Optional[str] = None
     phoenix_filename: Optional[str] = None
-    callgrind_profile: bool = False
+    preamble_algs: list = []
     # Define list of auditors to run. Possible common choices include
     # "NameAuditor", "MemoryAuditor" or "ChronoAuditor".
     # For a full list see Gaudi documentation.

Hlt/HLTScheduler/options/scheduler_testCF.py

@@ -46,6 +46,9 @@ a5.inpKeys = ['/Event/a1']
 a5.outKeys = ['/Event/a5']
 a5.CFD = 1
 
+b1 = ConfigurableDummy("B1")
+b2 = ConfigurableDummy("B2")
+
 exerep = ExecutionReportsWriter(
     "ExecReportsWriter",
     OutputLevel=DEBUG,
@@ -61,7 +64,7 @@ HLTControlFlowMgr().CompositeCFNodes = [
     ('notA1', 'NOT', ['A1'], True),
 ]
 HLTControlFlowMgr().AdditionalCFEdges = [['A5', 'line1']]
-
+HLTControlFlowMgr().PreambleAlgs = [b1, b2]
 HLTControlFlowMgr().ThreadPoolSize = threads
 HLTControlFlowMgr().OutputLevel = VERBOSE
 

Hlt/HLTScheduler/src/HLTControlFlowMgr.cpp

@@ -158,7 +158,7 @@ private:
       this, "AdditionalCFEdges", {}, "Additional Control Flow Edges defined by the User \
                                       (format: [ [before, after], [before2, after2] ])"};
   Gaudi::Property<std::set<std::string>> m_definitelyRunThese{
-      this, "AdditionalAlgs", {}, "Add algs that do not participate in the control flow but should\
+      this, "PreambleAlgs", {}, "Add algs that do not participate in the control flow but should\
                                    definitely run, like e.g. a callgrindprofile"};
 
   Gaudi::Property<int> m_startTimeAtEvt{this, "StartTimeAtEvt", -1, "start timing at this event. Counting from 0. \

Hlt/HLTScheduler/tests/qmtest/scheduler_testCF.qmt

@@ -32,6 +32,8 @@ NONLAZY_AND: moore     0|0
  ExecReportsWriter     0|1
 """,
 """HLTControlFlowMgr VERBOSE AlgsWithStates: Algorithm   isExecuted|filterPassed
+B1                  1|1
+B2                  1|1
 A1                  1|1
 A2                  0|0
 A3                  1|0

PyConf/python/PyConf/application.py

@@ -41,7 +41,6 @@ from .Algorithms import (
     LHCb__MDFWriter,
     OutputStream,
     CopyInputStream,
-    CallgrindProfile,
     LHCb__MDF__IOAlg,
     LHCb__UnpackRawEvent,
     Gaudi__Hive__FetchDataFromFile,
@@ -361,8 +360,9 @@ class ApplicationOptions(ConfigurableUser):
         'require_specific_decoding_keys': [],
         # input manifest of TES contents -- only needed when running passthrough to produce the output manifest, and/or as source of decoding keys...
         'input_manifest_file': '',
-        # Enables callgrind profiling
-        'callgrind_profile': False,
+        # Algorithms to be executed before the main control flow.
+        # Could be used for controlling profiling, e.g. CallgrindProfile, PerfProfile.
+        'preamble_algs': [],
         'msg_svc_format': '% F%35W%S %7W%R%T %0W%M',
         'msg_svc_time_format': '%Y-%m-%d %H:%M:%S UTC',
         # Moore-specific options
@@ -700,6 +700,9 @@ def make_callgrind_profile(start=10,
                            dumpName='CALLGRIND-OUT'):
     """Return algorithm that allows to use callgrind profiling.
 
+    Pass it to the application options with
+    ``preamble_algs=[make_callgrind_profile()]``.
+
     For more info see CodeAnalysisTools_.
 
     .. _CodeAnalysisTools: https://twiki.cern.ch/twiki/bin/view/LHCb/CodeAnalysisTools
@@ -714,6 +717,7 @@ def make_callgrind_profile(start=10,
         dumpName: Name of dump.
             Defaults to 'CALLGRIND-OUT'.
     """
+    from PyConf.Algorithms import CallgrindProfile
     return CallgrindProfile(
         StartFromEventN=start,
         StopAtEventN=stop,
@@ -840,15 +844,13 @@ def assert_empty_dataondemand_service():
 
 def configure(options, control_flow_node, public_tools=[],
               make_odin=make_odin):
+
     # TODO get rid of magic initial time (must go to configure_input)
     INITIAL_TIME = 1433509200
 
     options.finalize()
     config = ComponentConfig()
-    if options.callgrind_profile:
-        control_flow_node = CompositeNode(
-            'profile_' + control_flow_node.name,
-            [make_callgrind_profile(), control_flow_node])
+
     nodes, algs = all_nodes_and_algs(control_flow_node)
 
     # The following components are setup outside of the normal control and data flow. Thus, the automatic dependence evaluation in python does not work.
@@ -938,7 +940,9 @@ def configure(options, control_flow_node, public_tools=[],
             ThreadPoolSize=options.n_threads,
             EndEventIncident=end_event_incident,
             EnableLegacyMode=options.scheduler_legacy_mode,
-            BarrierAlgNames=barriers))
+            BarrierAlgNames=barriers,
+            PreambleAlgs=options.preamble_algs,
+        ))
     appMgr = config.add(
         ApplicationMgr(OutputLevel=options.output_level, EventLoop=scheduler))
     appMgr.ExtSvc.insert(