Fixed formatting

patch generated by https://gitlab.cern.ch/lhcb/DaVinci/-/jobs/12686596

DaVinciExamples/python/DaVinciExamples/tupling/example-tupling-basic.py

+###############################################################################
+# (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+"""
+Example of a typical DaVinci job:
+ - reconstruction and selection of two detached muons
+ - user algorithm printing decay trees via `PrintDecayTree`
+ - tuple of the selected candidates
+"""
+
+__author__ = "Maurizio Martinelli"
+__date__ = "2021-03-16"
+
+from PyConf.Algorithms import PrintDecayTree
+from PyConf.Algorithms import FunTuple_Particles as FunTuple
+from DaVinci import options, run_davinci, DVNode, DVHelper
+from DaVinci.standard_particles import make_detached_mumu
+from DaVinci.reco_objects import upfront_reconstruction_from_file as upfront_reconstruction
+
+# print control flow and data flow graphs
+options.control_flow_file = 'control_flow.gv'
+options.data_flow_file = 'data_flow.gv'
+
+options.ntuple_file = 'DV-example-tupling-basic-ntp.root'
+options.histo_file = 'DV-example-tupling-basic-his.root'
+
+# Setup dataset
+options.evt_max = 10
+options.set_input_and_conds_from_testfiledb('Upgrade_Bd2KstarMuMu')
+options.input_raw_format = 4.3
+options.lumi = True
+
+print(options)
+
+# Prepare the node with the selection
+dimuons = make_detached_mumu()
+
+# Prepare the node with the user algorithms
+pdt = PrintDecayTree(name="PrintDimuons", Input=dimuons)
+
+# Add a FUNTuple
+decay_descriptors = [
+    '[J/psi(1S) -> mu+ mu-]CC', '[J/psi(1S) -> ^mu+ mu-]CC',
+    '[J/psi(1S) -> mu+ ^mu-]CC'
+]
+branch_names = ['Jpsi', 'MuPlus', 'MuMinus']
+#for Jpsi store p, pt and mu+ store p and mu- store pt
+functors = [['P', 'PT'], ['P'], ['PT']]
+functor_branch_names = [['P', 'PT'], ['P'], ['PT']]
+ftup = FunTuple(
+    name="DimuonsTuple",
+    tree_name="DecayTree",
+    decay_descriptors=decay_descriptors,
+    branch_names=branch_names,
+    functors=functors,
+    functor_branch_names=functor_branch_names,
+    input_location=dimuons)
+
+dvh = DVHelper()
+dvh.add_selections_node("Selection1", upfront_reconstruction() + [dimuons])
+dvh.add_user_node('User1', [pdt])
+dvh.add_tuples_node('Tuple1', [ftup])
+dvh.run(options)

DaVinciExamples/tests/qmtest/tupling.qms/test_example-tupling-basic.qmt

+<?xml version="1.0" ?>
+<!--
+    (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration
+
+    This software is distributed under the terms of the GNU General Public
+    Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
+
+    In applying this licence, CERN does not waive the privileges and immunities
+    granted to it by virtue of its status as an Intergovernmental Organization
+    or submit itself to any jurisdiction.
+-->
+<!DOCTYPE extension  PUBLIC '-//QM/2.3/Extension//EN'  'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
+<extension class="GaudiTest.GaudiExeTest" kind="test">
+<argument name="program"><text>gaudirun.py</text></argument>
+<argument name="timeout"><integer>3600</integer></argument>
+<argument name="args"><set>
+  <text>$DAVINCIEXAMPLESROOT/python/DaVinciExamples/tupling/example-tupling-basic.py</text>
+</set></argument>
+<argument name="validator"><text>
+findReferenceBlock("""
+PrintDimuons.PrintDecayTreeTool        INFO                 Name         E         M         P        Pt       phi        Vz    P(C/K)   PP(C/K)
+PrintDimuons.PrintDecayTreeTool        INFO                            MeV       MeV       MeV       MeV      mrad        mm
+PrintDimuons.PrintDecayTreeTool        INFO J/psi(1S)            30155.22   4062.62  29880.30   3993.47  -3007.05     45.68       0/0       N/A
+PrintDimuons.PrintDecayTreeTool        INFO +-->mu+              12777.17    105.66  12776.73   3572.06  -2747.17     45.41      1/99      2/99
+PrintDimuons.PrintDecayTreeTool        INFO +-->mu-              17377.81    105.66  17377.49   1065.10   2238.10     43.82      1/54      2/54
+PrintDimuons.PrintDecayTreeTool        INFO
+PrintDimuons.PrintDecayTreeTool        INFO Used TES locations :-
+PrintDimuons.PrintDecayTreeTool        INFO       0 = '/Event/CombineParticles/Particles'
+PrintDimuons.PrintDecayTreeTool        INFO       1 = '/Event/FunctionalParticleMaker/Particles'
+PrintDimuons.PrintDecayTreeTool        INFO       2 = '/Event/Rec/ProtoP/Charged'
+""", stdout, result, causes, signature_offset = 0)
+countErrorLines({"FATAL":0, "ERROR":0})
+
+import sys, os, glob
+import ROOT as r
+from ROOT import TFile
+
+try:
+    ntuple = 'DV-example-tupling-basic-ntp.root'
+    f = TFile.Open(ntuple)
+    if f.IsZombie():
+        raise AttributeError('Output ROOT file is Zombie')
+    print('-- DV-example-tupling-basic-ntp.root QMTest -- : found output NTuple')
+    f.Close()
+    os.system('rm %s' %ntuple)
+except IOError as err:
+    raise IOError('Impossible to open file: ', err)
+except:
+    print('Unexpected error while opening file: ', sys.exc_info()[0])
+<argument name="exit_code"><integer>1</integer></argument>
+</text></argument>
+</extension>

Phys/DaVinci/options/application-option-defaults.yaml

@@ -53,6 +53,9 @@ input_files:
 input_type:
    text: '"""Type of input files, e.g. "DST", "DIGI", "RDST", "MDST", "XDST" or "LDST". Default = DST."""'
    value: 'DST'
+lumi:
+   text: '"""Luminosity accounting. Default = False."""'
+   value: False
 msg_svc_format:
    text: '"""MessageSvc output format.Default = "% F%35W%S %7W%R%T %0W%M"."""'
    value: '% F%35W%S %7W%R%T %0W%M'
@@ -68,7 +71,7 @@ output_level:
 print_freq:
    text: '"""Frequency at which to print event numbers. Default = 1000."""'
    value: 1000
-python_logging_level: 
+python_logging_level:
    text: '"""Python logger level. Default = logging.WARNING=30."""'
    value: 30
 skip_events:

Phys/DaVinci/python/DaVinci/__init__.py

@@ -12,7 +12,7 @@
 from __future__ import absolute_import
 
 from .ConfigurationUpgrade import data, mc, main
-from .config import options, run_davinci, dv_node, DVSelection
+from .config import options, run_davinci, DVNode, DVHelper
 
-__all__ = ('data', 'mc', 'main', 'options', 'run_davinci', 'dv_node',
-           'DVSelection')
+__all__ = ('data', 'mc', 'main', 'options', 'run_davinci', 'DVNode',
+           'DVHelper')

Phys/DaVinci/python/DaVinci/config.py

@@ -13,10 +13,11 @@
 
 from __future__ import absolute_import
 import logging
-from collections import namedtuple
+from collections import namedtuple, OrderedDict
 from PyConf.application import configure_input, configure
 from PyConf.components import Algorithm
 from PyConf.control_flow import CompositeNode, NodeLogic
+from PyConf.Algorithms import EventAccounting
 
 from DaVinci.application import DVAppOptions
 
@@ -25,10 +26,9 @@ log = logging.getLogger(__name__)
 options = DVAppOptions(_enabled=False)
 
 
-class DVSelection(namedtuple('DVSelection',
-                             ['node', 'extra_outputs'])):  # noqa
-    """Immutable object fully qualifiying an DaVinci selection.
-    Copied from HltLine without the prescaler
+class DVNode(namedtuple('DVNode', ['node', 'extra_outputs'])):  # noqa
+    """Immutable object fully qualifying a DaVinci node.
+    Copied from the `HltLine` class without the prescaler argument.
 
     Attributes:
         node (CompositeNode): the control flow node of the line
@@ -37,7 +37,7 @@ class DVSelection(namedtuple('DVSelection',
     __slots__ = ()  # do not add __dict__ (preserve immutability)
 
     def __new__(cls, name, algs, extra_outputs=None):
-        """Initialize a HltLine from name, algs and prescale.
+        """Initialize a DaVinci node from name and a set of algorithms.
 
         Creates a control flow `CompositeNode` with the given `algs`
         combined with `LAZY_AND` logic.
@@ -54,8 +54,7 @@ class DVSelection(namedtuple('DVSelection',
             force_order=True)
         if extra_outputs is None:
             extra_outputs = []
-        return super(DVSelection, cls).__new__(cls, node,
-                                               frozenset(extra_outputs))
+        return super(DVNode, cls).__new__(cls, node, frozenset(extra_outputs))
 
     @property
     def name(self):
@@ -85,41 +84,154 @@ class DVSelection(namedtuple('DVSelection',
         return self.output_producer is not None
 
 
-def dv_node(name, algs, logic=NodeLogic.NONLAZY_OR, force_order=False):
-    return CompositeNode(
-        name, combine_logic=logic, children=algs, force_order=force_order)
-
-
-def davinci_control_flow(options, dvsels=[]):
+class DVHelper():
+    """Class for collecting DV algorithms."""
+
+    def __init__(self):
+        self.selections = []
+        self.user = []
+        self.tuples = []
+        self.writers = []
+
+    def add_node(self, node_name, algs, kind):
+        if hasattr(self, kind):
+            setattr(self, kind,
+                    getattr(self, kind) + [DVNode(node_name, algs)])
+        else:
+            print('the specified note type ({}) is not valid'.format(kind))
+        return
+
+    def add_selections_node(self, node_name, algs):
+        self.add_node(node_name, algs, 'selections')
+        return
+
+    def add_user_node(self, node_name, algs):
+        self.add_node(node_name, algs, 'user')
+        return
+
+    def add_tuples_node(self, node_name, algs):
+        self.add_node(node_name, algs, 'tuples')
+        return
+
+    def add_writers_node(self, node_name, algs):
+        self.add_node(node_name, algs, 'writers')
+        return
+
+    def run(self, options, public_tools=[]):
+        run_davinci(
+            options,
+            selections=self.selections,
+            user=self.user,
+            tuples=self.tuples,
+            writers=self.writers,
+            public_tools=public_tools)
+
+
+def davinci_control_flow(options,
+                         SelectionNodes=[],
+                         UserAlgorithms=[],
+                         TuplesNodes=[],
+                         WritersNodes=[]):
+    '''
+    DaVinci control flow is split in a few sections as described in DaVinci/issue#2
+
+       DaVinci (LAZY_AND)
+       *-- LuminosityNode (NONLAZY_OR)
+       |   *-- EventAccounting/EventAccount
+       *-- DVSelectionsNode (NONLAZY_OR)
+       |   +-- DVCandidate1Node (LAZY_AND)
+       |   |   *-- PVFilter
+       |   |   *-- ParticlesFilter
+       |   |   *-- Candidate1Combiner
+       |   +-- DVCandidate2Node (LAZY_AND)
+       |   |   *-- PVFilter
+       |   |   *-- ParticlesFilter
+       |   |   *-- Candidate2Combiner
+       *-- DVUserAnalysisNode (NONLAZY_OR)
+       |   *-- PrintHeader
+       |   *-- PrintDecayTree/PrintMyB
+       *-- DVTuplesNode (NONLAZY_OR)
+       |   +-- DVTuple1Node (LAZY_AND)
+       |   |   *-- AlgorithmsForTuple1
+       |   |   *-- Tuple1
+       |   +-- DVTuple2Node (LAZY_AND)
+       |   |   *-- AlgorithmsForTuple2
+       |   |   *-- Tuple2
+       |   +-- DVTuple3Node (LAZY_AND)
+       |   |   *-- AlgorithmsForTuple3
+       |   |   *-- Tuple3
+       *-- DVWriterNode (LAZY_AND)
+           *-- DSTWriter?
+           *-- TuplesWriter
+
+    The main parts are
+        . LuminosityNode
+        . SelectionsNode
+        . UserAnalysisNode
+        . TuplesNode
+        . WritersNode
+    who are in AND among each other and can accomodate nodes (defined with the class DVNode)
+    in OR among themselves.
+
+    To prepare the control flow there are a few options:
+        1. take a dictionary as input where all the nodes are listed in their categories
+        2. take a various lists as input each related to a specific category of nodes
+    This function is then used to fill the control flow
+    '''
     options.finalize()
-    dec = CompositeNode(
-        'dv_decision',
-        combine_logic=NodeLogic.NONLAZY_OR,
-        children=[dvsel.node for dvsel in dvsels],
-        force_order=False)
+    dv_top_children = []
+    # Setup luminosity
+    LumiNodes = []
+    if options.lumi:
+        LumiNodes += [
+            DVNode('Lumi', [EventAccounting(name='EventAccount')])
+        ]  # this should be modified to reflect LumiAlgsConf (configured separately?)
+    ordered_nodes = OrderedDict()
+    ordered_nodes['Luminosity'] = LumiNodes
+    ordered_nodes['Selections'] = SelectionNodes
+    ordered_nodes['UserAlgorithms'] = UserAlgorithms
+    ordered_nodes['Tuples'] = TuplesNodes
+    ordered_nodes['Writers'] = WritersNodes
+    for k, v in ordered_nodes.items():
+        if len(v):
+            cnode = CompositeNode(
+                k,
+                combine_logic=NodeLogic.NONLAZY_OR,
+                children=[dv_node.node for dv_node in v],
+                force_order=False)
+            dv_top_children += [cnode]
 
     return CompositeNode(
-        'davinci',
-        combine_logic=NodeLogic.NONLAZY_OR,
-        children=[dec],
+        'DaVinci',
+        combine_logic=NodeLogic.LAZY_AND,
+        children=dv_top_children,
         force_order=True)
 
 
-def run_davinci(options, dvsels=[], public_tools=[]):
+def run_davinci(options,
+                selections=[],
+                user=[],
+                tuples=[],
+                writers=[],
+                public_tools=[]):
     '''
     DaVinci application control flow
 
     Args:
         options (ApplicationOptions): holder of application options
+        selections  : list of selections nodes
+        user        : list of user nodes
+        tuples      : list of tuples nodes
+        writers     : list of writers nodes
         public_tools (list): list of public `Tool` instances to configure
     '''
-    if not dvsels:
+    if not len(selections + user + tuples + writers):
         dummy = CompositeNode("EmptyNode", children=[])
         config = configure(options, dummy, public_tools=public_tools)
     else:
         config = configure_input(options)
-        top_dv_node = davinci_control_flow(options, dvsels)
+        top_dv_node = davinci_control_flow(options, selections, user, tuples,
+                                           writers)
         config.update(
             configure(options, top_dv_node, public_tools=public_tools))
-
     return config

Phys/DaVinci/tests/selection.py

@@ -8,7 +8,9 @@
 # granted to it by virtue of its status as an Intergovernmental Organization  #
 # or submit itself to any jurisdiction.                                       #
 ###############################################################################
-from DaVinci import options, run_davinci, DVSelection
+from DaVinci import options, run_davinci, DVNode
+from DaVinci.standard_particles import make_detached_mumu
+from DaVinci.reco_objects import upfront_reconstruction_from_file as upfront_reconstruction
 
 # print control flow and data flow graphs
 options.control_flow_file = 'control_flow.gv'
@@ -21,17 +23,13 @@ options.histo_file = 'DVU_test-his.root'
 options.evt_max = 10
 options.set_input_and_conds_from_testfiledb('Upgrade_Bd2KstarMuMu')
 #options.set_input_and_conds_from_testfiledb('upgrade-magdown-sim09c-up02-reco-up01-minbias-ldst')
-
 options.input_raw_format = 4.3
 
 print(options)
 
-from DaVinci.standard_particles import make_detached_mumu
-from DaVinci.reco_objects import upfront_reconstruction_from_file as upfront_reconstruction
-sel = DVSelection(
-    name="DVselection", algs=upfront_reconstruction() + [make_detached_mumu()])
+sel = DVNode(
+    name="Selection1", algs=upfront_reconstruction() + [make_detached_mumu()])
 
 # run davinci
 #public_tools = [stateProvider_with_simplified_geom()]
-#run_davinci(options, [sel, sel2], public_tools)
-run_davinci(options, [sel])  #, public_tools)
+run_davinci(options, selections=[sel])  #, public_tools)

Phys/DaVinci/tests/simple_histos.py

@@ -9,7 +9,7 @@
 # or submit itself to any jurisdiction.                                       #
 ###############################################################################
 from PyConf.Algorithms import GaudiHistoAlgorithm
-from DaVinci import options, run_davinci, DVSelection
+from DaVinci import options, run_davinci, DVNode
 
 # print control flow and data flow graphs
 options.control_flow_file = 'control_flow.gv'
@@ -26,5 +26,5 @@ options.set_input_and_conds_from_testfiledb('Upgrade_Bd2KstarMuMu')
 simple_histos = GaudiHistoAlgorithm(name="SimpleHistos", HistoPrint=True)
 
 # run davinci
-algs = DVSelection(name="DVselection", algs=[simple_histos])
-run_davinci(options, [algs])
+algs = DVNode(name="DVselection", algs=[simple_histos])
+run_davinci(options, user=[algs])