Merge branch 'olupton_improve_functors' into 'master'

Add a 2-track line using new combiner/functors

See merge request lhcb/Moore!204

CMakeLists.txt

@@ -8,7 +8,8 @@
 # granted to it by virtue of its status as an Intergovernmental Organization  #
 # or submit itself to any jurisdiction.                                       #
 ###############################################################################
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8.5)
+CMAKE_MINIMUM_REQUIRED(VERSION 3.14)
+project(Moore LANGUAGES CXX)
 
 #---------------------------------------------------------------
 # Load macros and functions for Gaudi-based projects

Hlt/Hlt1Conf/options/hlt1_example.py

+###############################################################################
+# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+from PyConf.environment import EverythingHandler
+from PyConf.Algorithms import FTRawBankDecoder
+from RecoConf.hlt1_tracking import require_gec
+from Hlt1Conf.lines.track_mva import (one_track_mva_line, two_track_mva_line,
+                                      debug_two_track_mva_line)
+
+ftdec_v = 4
+env = EverythingHandler(
+    threadPoolSize=1, nEventSlots=1, evtMax=1000, debug=True)
+
+with FTRawBankDecoder.bind(DecodingVersion=ftdec_v), \
+     require_gec.bind(FTDecodingVersion=ftdec_v):
+
+    builders = {
+        'Hlt1TrackMVALine': one_track_mva_line,
+        'Hlt1TwoTrackMVALine': two_track_mva_line,
+        'Hlt1DebugTwoTrackMVALine': debug_two_track_mva_line,
+    }
+    for name, builder in builders.items():
+        env.registerLine(name, builder())
+
+env.setupInputFromTestFileDB('MiniBrunel_2018_MinBias_FTv4_DIGI')
+env.configure()
+# env.plotDataFlow()

Hlt/Hlt1Conf/python/Hlt1Conf/lines.py → Hlt/Hlt1Conf/python/Hlt1Conf/algorithms.py

@@ -12,7 +12,10 @@ from __future__ import absolute_import, division, print_function
 from PyConf import configurable
 from Configurables import (
     PrFilter__Track_v1,
+    PrFilter__Track_v2,
     TrackCombiner,
+    CombineTracks__2Body__Track_v2,
+    CombineTracks__2Body__Track_v2__Dumper,
 )
 
 from PyConf.components import Algorithm
@@ -69,3 +72,48 @@ def TrackCombinerWithPVs(Preamble=[], **kwargs):
         }
 
     return Algorithm(TrackCombiner, input_transform=transform_inputs, **kwargs)
+
+
+@configurable
+def PrFilterV2Tracks(functor, **kwargs):
+    return Algorithm(
+        PrFilter__Track_v2,
+        Code=functor.code(),
+        Headers=functor.headers(),
+        Factory='FunctorFactory',
+        **kwargs)
+
+
+@configurable
+def CombineTracks(NBodies=2,
+                  CombinationCut=None,
+                  VertexCut=None,
+                  VoidDump=None,
+                  ChildDump=None,
+                  CombinationDump=None,
+                  VertexDump=None,
+                  **kwargs):
+    assert NBodies == 2
+
+    def parse(input_dict):
+        if input_dict is None: return {}
+        return {k: [v.code()] + v.headers() for k, v in input_dict.items()}
+
+    VoidDump_dict = parse(VoidDump)
+    ChildDump_dict = parse(ChildDump)
+    CombinationDump_dict = parse(CombinationDump)
+    VertexDump_dict = parse(VertexDump)
+    enable_dumper = len(ChildDump_dict) or len(CombinationDump_dict) or len(
+        VertexDump_dict)
+    return Algorithm(
+        CombineTracks__2Body__Track_v2__Dumper
+        if enable_dumper else CombineTracks__2Body__Track_v2,
+        VertexCut_Code=VertexCut.code(),
+        VertexCut_Headers=VertexCut.headers(),
+        CombinationCut_Code=CombinationCut.code(),
+        CombinationCut_Headers=CombinationCut.headers(),
+        VoidDump_Functors=VoidDump_dict,
+        ChildDump_Functors=ChildDump_dict,
+        CombinationDump_Functors=CombinationDump_dict,
+        VertexDump_Functors=VertexDump_dict,
+        **kwargs)

Hlt/Hlt1Conf/python/Hlt1Conf/lines/track_mva.py

+###############################################################################
+# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+from __future__ import absolute_import, division, print_function
+import math
+from GaudiKernel.SystemOfUnits import GeV
+from PyConf import configurable
+from RecoConf.hlt1_tracking import (require_gec, require_pvs, make_pvs,
+                                    make_odin, make_hlt1_tracks,
+                                    make_velokalman_fitted_tracks)
+from ..algorithms import CombineTracks, PrFilterV2Tracks
+from PyConf.Algorithms import FTRawBankDecoder
+
+
+def make_tracks_mva_tracks():
+    return make_velokalman_fitted_tracks(make_hlt1_tracks())
+
+
+@configurable
+def track_mva_prefilters(make_pvs=make_pvs):
+    return [require_gec(), require_pvs(make_pvs())]
+
+
+@configurable
+def one_track_mva_line(
+        make_input_tracks=make_tracks_mva_tracks,
+        make_pvs=make_pvs,
+        # TrackMVALoose cuts from ZombieMoore
+        max_chi2dof=2.5,
+        min_pt=2.0 * GeV,
+        max_pt=26 * GeV,
+        min_ipchi2=7.4,
+        param1=1.0,
+        param2=2.0,
+        param3=1.248):
+    pvs = make_pvs().location
+    from Functors import PT, CHI2DOF, MINIPCHI2, MINIPCHI2CUT
+    from Functors.math import in_range, log
+    pre_sel = (CHI2DOF < max_chi2dof)
+    hard_sel = (PT > max_pt) & MINIPCHI2CUT(IPChi2Cut=min_ipchi2, Vertices=pvs)
+    bulk_sel = in_range(min_pt, PT, max_pt) & (
+        log(MINIPCHI2(pvs)) >
+        (param1 / ((PT / GeV - param2)**2) +
+         (param3 / max_pt) * (max_pt - PT) + math.log(min_ipchi2)))
+    full_sel = pre_sel & (hard_sel | bulk_sel)
+    track_filter = PrFilterV2Tracks(
+        full_sel, Input=make_input_tracks()['v2Sel'])
+    return track_mva_prefilters() + [track_filter]
+
+
+@configurable
+def two_track_mva_line(make_input_tracks=make_tracks_mva_tracks,
+                       make_pvs=make_pvs):
+    # Compared to TwoTrackMVALoose in ZombieMoore this is missing:
+    # in_range( 2, BPVETA, 5 )
+    # BPVCORRM > 1. * GeV
+    # (ignoring a very loose upper cut on BPVCORRM)
+    # and, of course, the actual MVA....
+    from Functors import P, PT, CHI2DOF, DOCACHI2, BPVDIRA, MINIPCHI2CUT
+    from GaudiKernel.SystemOfUnits import MeV
+    pvs = make_pvs().location
+    ChildCut = ((PT > 800. * MeV) & (P > 5. * GeV) &
+                (CHI2DOF < 2.5) & MINIPCHI2CUT(IPChi2Cut=4., Vertices=pvs))
+    CombinationCut = (PT > 2. * GeV) & (DOCACHI2 < 10.)
+    VertexCut = (CHI2DOF < 10.) & (BPVDIRA(pvs) > 0)
+    children = PrFilterV2Tracks(
+        ChildCut, Input=make_input_tracks()['v2Sel']).Output
+    combination_filter = CombineTracks(
+        NBodies=2,
+        VertexCut=VertexCut,
+        InputTracks=children,
+        CombinationCut=CombinationCut)
+    return track_mva_prefilters() + [combination_filter]
+
+
+@configurable
+def debug_two_track_mva_line(make_input_tracks=make_tracks_mva_tracks,
+                             make_pvs=make_pvs,
+                             VertexCut=None,
+                             ChildCut=None,
+                             CombinationCut=None,
+                             VoidDump=None,
+                             ChildDump=None,
+                             CombinationDump=None,
+                             VertexDump=None,
+                             dump_filename='PrCombineTracks.root'):
+
+    from Functors import (ALL, ETA, P, PT, CHI2DOF, SUM, DOCA, DOCACHI2,
+                          BPVDIRA, BPVIPCHI2, MINIP, PHI, RUNNUMBER,
+                          EVENTNUMBER, EVENTTYPE, BPVIPCHI2STATE)
+    from GaudiKernel.SystemOfUnits import MeV, GeV, mm, mrad
+    pv_loc, odin_loc = make_pvs().location, make_odin().location
+    if VoidDump is None:
+        VoidDump = {
+            'runNumber': RUNNUMBER(odin_loc),
+            'eventType': EVENTTYPE(odin_loc),
+            'eventNumber': EVENTNUMBER(odin_loc),
+        }
+    if ChildCut is None:
+        ChildCut = (PT > 250. * MeV)
+    if ChildDump is None:
+        ChildDump = {
+            'P': P,
+            'PT': PT,
+            'ETA': ETA,
+            'PHI': PHI,
+            'IP': MINIP(pv_loc),
+            'IPCHI2': BPVIPCHI2(pv_loc),
+            'CHI2DOF': CHI2DOF,
+        }
+    if VertexCut is None:
+        VertexCut = (CHI2DOF < 1e3)
+    if VertexDump is None:
+        VertexDump = {
+            'PT': PT,
+            'ETA': ETA,
+            'PHI': PHI,
+            'CHI2DOF': CHI2DOF,
+            'DIRA': BPVDIRA(pv_loc),
+            'IPCHI2': BPVIPCHI2(pv_loc),
+            'IPCHI2_STATE': BPVIPCHI2STATE(pv_loc),
+        }
+    if CombinationCut is None:
+        CombinationCut = ALL
+    if CombinationDump is None:
+        CombinationDump = {
+            'PT': PT,
+            'DOCA': DOCA,
+            'SUMPT': SUM(PT),
+            'DOCACHI2': DOCACHI2,
+        }
+    children = PrFilterV2Tracks(
+        ChildCut, Input=make_input_tracks()['v2Sel']).Output
+    combiner = CombineTracks(
+        NBodies=2,
+        VertexCut=VertexCut,
+        InputTracks=children,
+        CombinationCut=CombinationCut,
+        VoidDump=VoidDump,
+        ChildDump=ChildDump,
+        CombinationDump=CombinationDump,
+        VertexDump=VertexDump,
+        DumpFileName=dump_filename)
+    # TODO remove make_odin() from the control flow when configuration
+    #      starts handling data flow in functors, see
+    #      https://gitlab.cern.ch/lhcb/Moore/issues/51
+    return track_mva_prefilters() + [make_odin(), combiner]

Hlt/Hlt1Conf/tests/qmtest/test_hlt1_example.qmt

+<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE extension  PUBLIC '-//QM/2.3/Extension//EN'  'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
+<!--
+    (c) Copyright 2000-2018 CERN for the benefit of the LHCb Collaboration
+
+    This software is distributed under the terms of the GNU General Public
+    Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
+
+    In applying this licence, CERN does not waive the privileges and immunities
+    granted to it by virtue of its status as an Intergovernmental Organization
+    or submit itself to any jurisdiction.
+-->
+<!--
+Make sure HLT1 configures and runs without errors on raw data
+-->
+<extension class="GaudiTest.GaudiExeTest" kind="test">
+<argument name="program"><text>gaudirun.py</text></argument>
+<argument name="args"><set>
+  <text>$HLT1CONFROOT/options/hlt1_example.py</text>
+  <text>--output=hlt1_example.opts.py</text>
+  <text>--all-opt</text>
+</set></argument>
+<argument name="options"><text>
+from Configurables import HiveDataBrokerSvc
+HiveDataBrokerSvc().OutputLevel = 5
+</text></argument>
+<argument name="environment"><set>
+  <text>LOKI_DISABLE_PYTHON=1</text>
+</set></argument>
+<argument name="use_temp_dir"><enumeral>true</enumeral></argument>
+<argument name="validator"><text>
+
+countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0})
+
+# test that at least one event fires
+import re
+m = re.search('NONLAZY_OR: hlt_decision .*Sum=(\d+)', stdout)
+if m:
+    result['hlt_decision_sum'] = result.Quote(m.group(1))
+    if int(m.group(1)) &lt; 1:
+        causes.append('no positive decisions found')
+else:
+    causes.append('hlt_decision counter not found')
+
+</text></argument>
+</extension>

Hlt/RecoConf/python/RecoConf/hlt1_tracking.py

@@ -19,7 +19,7 @@ from Configurables import (
     Gaudi__Hive__FetchDataFromFile,
     PrGECFilter,
     PrStoreUTHit,
-    LoKi__VoidFilter,
+    VoidFilter,
     PrVeloUT,
     ParameterizedKalmanFit,
 )
@@ -27,6 +27,7 @@ from Configurables import (
 # all trivial (not changing default) and fully datahandly imports can be done from here
 from PyConf.Algorithms import (
     MakeSelection__Track_v1 as TrackV1Selection,
+    MakeSelection__Track_v2 as TrackV2Selection,
     LHCb__Converters__Track__v1__fromV2TrackV1TrackVector as
     FromV2TrackV1TrackVector,
     LHCb__Converters__Track__v1__fromV2TrackV1Track as FromV2TrackV1Track,
@@ -41,6 +42,7 @@ from PyConf.Algorithms import (
     TrackBeamLineVertexFinderSoA,
     SciFiTrackForwardingStoreHit,
     VeloKalman,
+    createODIN,
 )
 
 from GaudiKernel.SystemOfUnits import mm, MeV
@@ -63,18 +65,24 @@ VeloUTTracking = make_algorithm(PrVeloUT, defaults=dict(minPT=300 * MeV))
 
 
 def __emptyfilter_input_transform(InputLocation):
-    return {"Code": "SIZE('{}')>0".format(InputLocation)}
+    from Functors import SIZE
+    fun = SIZE(InputLocation) > 0
+    return {"Code": fun.code(), "Headers": fun.headers()}
 
 
 EmptyFilter = make_algorithm(
-    LoKi__VoidFilter, input_transform=__emptyfilter_input_transform)
+    VoidFilter, input_transform=__emptyfilter_input_transform)
 
 
 def make_raw_data():
     return RawData().RawEvent
 
 
-# maker functions
+@configurable
+def make_odin(make_raw_data=make_raw_data):
+    return createODIN(RawEvent=make_raw_data()).ODIN
+
+
 @configurable
 def require_gec(make_raw=make_raw_data, FTDecodingVersion=None, **kwargs):
     # TODO use FTDecodingVersion=smart.REQUIRED above
@@ -224,11 +232,13 @@ def make_velokalman_fitted_tracks(tracks, make_hits=make_velo_hits):
     fitted_tracks_v1_sel = TrackV1Selection(
         Input=FromV2TrackV1TrackVector(
             InputTracksName=fitted_tracks_v2).OutputTracksName)
+    fitted_tracks_v2_sel = TrackV2Selection(Input=fitted_tracks_v2).Output
     return {
         "Pr": fitted_tracks,
         "v2": fitted_tracks_v2,
         "v1": fitted_tracks_v1,
-        "v1Sel": fitted_tracks_v1_sel
+        "v1Sel": fitted_tracks_v1_sel,
+        "v2Sel": fitted_tracks_v2_sel,
     }
 
 

MooreCache/CMakeLists.txt

+###############################################################################
+# (c) Copyright 2000-2018 CERN for the benefit of the LHCb Collaboration      #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+gaudi_subdir(MooreCache)
+
+gaudi_depends_on_subdirs(Hlt/Hlt1Conf
+                         Phys/FunctorCore)
+
+# Suppress compilation warnings from external packages
+find_package(Boost)
+find_package(ROOT)
+include_directories(SYSTEM ${Boost_INCLUDE_DIRS} ${ROOT_INCLUDE_DIRS})
+
+# Import the cache creation module
+include(LoKiFunctorsCache)
+
+# Allow build in satellite (lb-dev) projects
+if(TARGET MooreConfUserDB)
+  set(conf_deps DEPENDS MooreConfUserDB)
+endif()
+
+# Additional factories for Upgrade
+loki_functors_cache(Moore_FunctorCache
+                    ${Moore_SOURCE_DIR}/Hlt/Hlt1Conf/options/hlt1_example.py
+                    options/DisableLoKiCacheFunctors.py
+                    FACTORIES FunctorFactory
+                    LINK_LIBRARIES FunctorCoreLib
+                    ${conf_deps}
+                    SPLIT 1)

MooreCache/options/DisableLoKiCacheFunctors.py

+###############################################################################
+# (c) Copyright 2000-2018 CERN for the benefit of the LHCb Collaboration      #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+from Configurables import ApplicationMgr
+ApplicationMgr().Environment['LOKI_DISABLE_CACHE'] = '1'
+ApplicationMgr().Environment['LOKI_DISABLE_CLING'] = '1'

PyConf/python/PyConf/components.py

@@ -106,12 +106,12 @@ def _check_input_integrity(t, inputs, other_args, input_transform=None):
     dh_inputs = configurable_inputs(t)
     if set(dh_inputs).intersection(other_args):
         raise TypeError(
-            'Inputs must be provided as DataHandles or Algorithms, \
-              please check these arguments: {}'.format(dh_inputs))
+            'Inputs must be provided as DataHandles or Algorithms, '
+            'please check these arguments: {}'.format(dh_inputs))
     if not set(dh_inputs).issubset(inputs):
-        raise ConfigurationError(" ".join(
-            ('please provide all inputs.',
-             'The ones detected here are: {}'.format(dh_inputs))))
+        raise ConfigurationError(
+            'Please provide all inputs. The ones need here are: {}'.format(
+                dh_inputs))
     if input_transform:
         input_transform_args = _get_args(input_transform)
         assert set(inputs).issubset(

PyConf/python/PyConf/environment.py

@@ -10,14 +10,18 @@
 ###############################################################################
 from __future__ import absolute_import, division, print_function
 
+import logging
+
 from Configurables import (
     CallgrindProfile,
     DeterministicPrescaler,
+    DDDBConf,
+    CondDB,
     # FIXME(NN): We shouldn't need to refer to IOV explicitly in our framework
     LHCb__DetDesc__ReserveDetDescForEvent as reserveIOV,
     LHCb__Tests__FakeEventTimeProducer as DummyEventTime,
 )
-from Gaudi.Configuration import DEBUG
+from Gaudi.Configuration import ConfigurableUser, DEBUG
 from GaudiConf import IOHelper
 from PRConfig.TestFileDB import test_file_db
 
@@ -31,11 +35,30 @@ __all__ = [
     'EverythingHandler',
 ]
 
+log = logging.getLogger(__name__)
+
 
 def _is_node(arg):
     return isinstance(arg, CompositeNode)
 
 
+class PythonLoggingConf(ConfigurableUser):
+    """Takes care of configuring the python logging verbosity."""
+    # Make sure we're applied before anything else by listing
+    # configurables in __used_configurables__. This ensures that we can
+    # modify the python logging level before anything spits out messages.
+    __used_configurables__ = [
+        CondDB,
+        DDDBConf,
+    ]
+
+    def __apply_configuration__(self):
+        import GaudiKernel.ProcessJobOptions
+        # turn off printing for the rest of the configuration
+        log.info('Disabling info messages from python logging')
+        GaudiKernel.ProcessJobOptions.PrintOff()
+
+
 class EverythingHandler(object):
     """The high level application configurator.
 
@@ -100,6 +123,9 @@ class EverythingHandler(object):
         if debug:
             self._scheduler.OutputLevel = DEBUG
             self._hiveDataBroker.OutputLevel = DEBUG
+        # Instantiate the configurable which will set the python
+        # logging verbosity at the right time.
+        PythonLoggingConf()
 
     def find_configurable_alg(self, name):
         if not self.configurable_algs:
@@ -246,6 +272,7 @@ class EverythingHandler(object):
             configuration.update(tool.configuration())
 
         # FIXME we make the configurables accessible to be able to hack the ugly things
+        odin_loc = '/Event/DAQ/DummyODIN'
         self.configurable_algs, self.configurable_tools = configuration.apply()
         self.configurable_algs += [
             setup_component(
@@ -253,8 +280,10 @@ class EverythingHandler(object):
                 "DummyEventTime",
                 Start=self._eventClockSvc.InitialTime / 1E9,
                 Step=0,
+                ODIN=odin_loc,
                 IOVLockDep=False),
-            setup_component(reserveIOV, "reserveIOV", IOVLockDep=False)
+            setup_component(
+                reserveIOV, "reserveIOV", IOVLockDep=False, ODIN=odin_loc)
         ]
         self._hiveDataBroker.DataProducers = self.configurable_algs
         self._scheduler.CompositeCFNodes = [

PyConf/python/PyConf/tests/test_algorithm.py

@@ -53,7 +53,7 @@ def test_init():
     # Must always provide all inputs
     with pytest.raises(ConfigurationError) as e:
         consumer = Algorithm(IntDataConsumer)
-    assert re.search(r'please provide all inputs.*InputLocation.*', str(e))
+    assert re.match(r'.*provide all inputs.*InputLocation.*', str(e))
 
     # Type of inputs
     with pytest.raises(TypeError) as e:

PyConf/python/PyConf/tests/test_tool.py

@@ -25,7 +25,7 @@ def test_init():
 
     with pytest.raises(ConfigurationError) as e:
         Tool(FloatTool)
-    assert re.search(r'please provide all inputs.*Input.*', str(e))
+    assert re.match(r'.*provide all inputs.*Input.*', str(e))
 
     t = Tool(FloatTool, Input=Algorithm(Producer))
     assert len(t.inputs) == 1