Merge branch 'olupton_twotrack_studies' into 'master'

Add TrackMVA and TwoTrackMVA implementations

See merge request lhcb/Moore!223

Hlt/Hlt1Conf/options/hlt1_example_throughput.py

+###############################################################################
+# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+"""Run HLT1 configured for throughput tests.
+
+The differences with respect to hlt1_example.py are:
+
+* Assume the input files are available on a RAM disk `/run/swtest`, which
+  avoids the penalty of loading each file from disk.
+* Assumes the existance of a file 00067189.mdf on that RAM disk.
+* Uses the EvtStoreSvc to handle interactions with the TES, as this is faster
+  than the default HiveWhiteBoard (although doesn't work on DSTs).
+* Uses an alternative, faster IOSvc.
+* Removes the Hlt1DebugTwoTrackMVALine/debug_two_track_mva_line line, which has
+  unrealistically loose cuts and overlead from ROOT TTree I/O and should not,
+  therefore, be included in throughput tests.
+
+The latter two points may be made defaults for all HLT1 execution in the
+future, but are not currently compatible with certain input types.
+"""
+from PyConf.environment import EverythingHandler, setupInputFromTestFileDB
+from PyConf.Algorithms import FTRawBankDecoder
+from RecoConf.hlt1_tracking import (require_gec, make_raw_data,
+                                    make_RawData_IOSvc)
+from Hlt1Conf.lines.track_mva import (one_track_mva_line, two_track_mva_line)
+
+RAM_DISK = '/run/swtest'
+INPUT_FILE = '00067189.mdf'
+
+tes = 'EvtStoreSvc'
+env = EverythingHandler(
+    threadPoolSize=1, nEventSlots=1, evtMax=100000, debug=True, TESName=tes)
+
+ftdec_v = 4
+inputFiles = setupInputFromTestFileDB(
+    'MiniBrunel_2018_MinBias_FTv4_MDF',
+    # Run over the single file 50 times to make sure we process lots of events
+    inputFiles=[os.path.join(RAM_DISK, INPUT_FILE)] * 50,
+    TESName=tes,
+    fileType='MDF',
+    withEventSelector=False)
+make_input_data = lambda: make_RawData_IOSvc(inputFiles)
+with FTRawBankDecoder.bind(DecodingVersion=ftdec_v), \
+     require_gec.bind(FTDecodingVersion=ftdec_v), \
+     make_raw_data.bind(make_raw=make_input_data):
+    builders = {
+        'Hlt1TrackMVALine': one_track_mva_line,
+        'Hlt1TwoTrackMVALine': two_track_mva_line,
+    }
+    for name, builder in builders.items():
+        env.registerLine(name, builder())
+
+env.configure()
+# env.plotDataFlow()

Hlt/Hlt1Conf/python/Hlt1Conf/algorithms.py

@@ -13,9 +13,11 @@ from PyConf import configurable
 from Configurables import (
     PrFilter__Track_v1,
     PrFilter__Track_v2,
+    PrFilter__PrFittedForwardTracks,
     TrackCombiner,
     CombineTracks__2Body__Track_v2,
     CombineTracks__2Body__Track_v2__Dumper,
+    CombineTracks__2Body__PrFittedForwardTracks,
 )
 
 from PyConf.components import Algorithm
@@ -84,6 +86,16 @@ def PrFilterV2Tracks(functor, **kwargs):
         **kwargs)
 
 
+@configurable
+def FilterPrFittedForwardTracks(functor, **kwargs):
+    return Algorithm(
+        PrFilter__PrFittedForwardTracks,
+        Code=functor.code(),
+        Headers=functor.headers(),
+        Factory='FunctorFactory',
+        **kwargs)
+
+
 @configurable
 def CombineTracks(NBodies=2,
                   CombinationCut=None,
@@ -92,7 +104,40 @@ def CombineTracks(NBodies=2,
                   ChildDump=None,
                   CombinationDump=None,
                   VertexDump=None,
+                  PrTracks=False,
                   **kwargs):
+    """Return a configured CombineTracks instance.
+
+    If any of `ChildDump`, `CombinationDump`, or `VertexDump` are not None, a
+    version of the combiner is returned that will produce an ntuple. The ntuple
+    has one row for each combination, and is useful for inspecting values that
+    the combiner uses to evaluate selections. Each `Dump` argument is a
+    dictionary whose keys are strings, corresponding to the row in the ntuple
+    that will be filled with the functor value.
+
+    Parameters
+    ----------
+    NBodies : int
+        The number of tracks entering each combination. Currently only 2-track
+        combinations are supported.
+    CombinationCut : Functors.Functor
+        Functor to be applied to the N-track combination object.
+    VertexCut : Functors.Functor
+        Functor to be applied to the vertex object.
+    VoidDump : dict of str to Functors.Functor
+        Functors to be evaluated for each combination when creating an ntuple.
+    ChildDump : dict of str to Functors.Functor
+        Functors to be evaluated for each child in each combination when
+        creating an ntuple.
+    CombinationDump : dict of str to Functors.Functor
+        Functors to be evaluated for each combination when creating an ntuple.
+    VertexDump : dict of str to Functors.Functor
+        Functors to be evaluated for each combination when creating an ntuple.
+    PrTracks : bool
+        If True, a combiner algorithm instance that accepts
+        PrFittedForwardTrack objects is returned. Otherwise, an instance that
+        accepts Track::v2 objects is returned.
+    """
     assert NBodies == 2
 
     def parse(input_dict):
@@ -105,9 +150,14 @@ def CombineTracks(NBodies=2,
     VertexDump_dict = parse(VertexDump)
     enable_dumper = len(ChildDump_dict) or len(CombinationDump_dict) or len(
         VertexDump_dict)
+    if PrTracks:
+        AlgType = CombineTracks__2Body__PrFittedForwardTracks
+    elif enable_dumper:
+        AlgType = CombineTracks__2Body__Track_v2__Dumper
+    else:
+        AlgType = CombineTracks__2Body__Track_v2
     return Algorithm(
-        CombineTracks__2Body__Track_v2__Dumper
-        if enable_dumper else CombineTracks__2Body__Track_v2,
+        AlgType,
         VertexCut_Code=VertexCut.code(),
         VertexCut_Headers=VertexCut.headers(),
         CombinationCut_Code=CombinationCut.code(),

Hlt/Hlt1Conf/python/Hlt1Conf/lines/track_mva.py

@@ -15,8 +15,9 @@ from PyConf import configurable
 from RecoConf.hlt1_tracking import (require_gec, require_pvs, make_pvs,
                                     make_odin, make_hlt1_tracks,
                                     make_velokalman_fitted_tracks)
-from ..algorithms import CombineTracks, PrFilterV2Tracks
-from PyConf.Algorithms import FTRawBankDecoder
+from ..algorithms import (CombineTracks, PrFilterV2Tracks,
+                          FilterPrFittedForwardTracks)
+from PyConf.Algorithms import (FTRawBankDecoder, MakeIterableTracks)
 
 
 def make_tracks_mva_tracks():
@@ -47,33 +48,59 @@ def one_track_mva_line(
     hard_sel = (PT > max_pt) & MINIPCHI2CUT(IPChi2Cut=min_ipchi2, Vertices=pvs)
     bulk_sel = in_range(min_pt, PT, max_pt) & (
         log(MINIPCHI2(pvs)) >
-        (param1 / ((PT / GeV - param2)**2) +
+        (param1 / ((PT / GeV - param2) * (PT / GeV - param2)) +
          (param3 / max_pt) * (max_pt - PT) + math.log(min_ipchi2)))
     full_sel = pre_sel & (hard_sel | bulk_sel)
-    track_filter = PrFilterV2Tracks(
-        full_sel, Input=make_input_tracks()['v2Sel'])
+    track_filter = FilterPrFittedForwardTracks(
+        full_sel, Input=make_input_tracks()['Pr'])
     return track_mva_prefilters() + [track_filter]
 
 
 @configurable
 def two_track_mva_line(make_input_tracks=make_tracks_mva_tracks,
                        make_pvs=make_pvs):
-    # Compared to TwoTrackMVALoose in ZombieMoore this is missing:
-    # in_range( 2, BPVETA, 5 )
-    # BPVCORRM > 1. * GeV
-    # (ignoring a very loose upper cut on BPVCORRM)
-    # and, of course, the actual MVA....
-    from Functors import P, PT, CHI2DOF, DOCACHI2, BPVDIRA, MINIPCHI2CUT
-    from GaudiKernel.SystemOfUnits import MeV
+    # This corresponds to TwoTrackMVALoose in ZombieMoore
+    from Functors.math import in_range
+    from Functors import (P, PT, CHI2DOF, DOCACHI2, BPVDIRA, MINIPCHI2CUT, MVA,
+                          COMB, SUM, BPVIPCHI2, BPVFDCHI2, BPVETA, BPVCORRM)
+    from GaudiKernel.SystemOfUnits import MeV, GeV
     pvs = make_pvs().location
     ChildCut = ((PT > 800. * MeV) & (P > 5. * GeV) &
                 (CHI2DOF < 2.5) & MINIPCHI2CUT(IPChi2Cut=4., Vertices=pvs))
+    pr_children = FilterPrFittedForwardTracks(
+        ChildCut, Input=make_input_tracks()['Pr']).Output
+    children = MakeIterableTracks(Input=pr_children).Output
     CombinationCut = (PT > 2. * GeV) & (DOCACHI2 < 10.)
-    VertexCut = (CHI2DOF < 10.) & (BPVDIRA(pvs) > 0)
-    children = PrFilterV2Tracks(
-        ChildCut, Input=make_input_tracks()['v2Sel']).Output
+
+    def apply_to_combination(functor):
+        """Helper for applying 'Combination' cuts to a composite."""
+        return COMB(
+            Functor=functor,
+            ChildContainers=[children.location],
+            ChildContainerTypes=[
+                ('LHCb::Pr::Iterable::Fitted::Forward::Tracks',
+                 'Event/PrIterableTracks.h')
+            ])
+
+    # TODO there seems to be some magic that substitutes the value of
+    #      $PARAMFILESROOT before it gets to the C++. This is probably not what
+    #      we want -- better to resolve it at runtime than bake a value into
+    #      the functor cache, for example
+    VertexCut = ((CHI2DOF < 10.) & (BPVDIRA(pvs) > 0) & (
+        BPVCORRM(Mass=139.57018 * MeV, Vertices=pvs) > 1. * GeV
+    ) & in_range(2., BPVETA(pvs), 5.) & (MVA(
+        MVAType='MatrixNet',
+        Config={'MatrixnetFile': "${PARAMFILESROOT}/data/Hlt1TwoTrackMVA.mx"},
+        Inputs={
+            'chi2': CHI2DOF,
+            'fdchi2': BPVFDCHI2(pvs),
+            'sumpt': apply_to_combination(SUM(PT)),
+            'nlt16': apply_to_combination(SUM(BPVIPCHI2(pvs) < 16.)),
+        }) > 0.96))
+
     combination_filter = CombineTracks(
         NBodies=2,
+        PrTracks=True,
         VertexCut=VertexCut,
         InputTracks=children,
         CombinationCut=CombinationCut)
@@ -94,8 +121,8 @@ def debug_two_track_mva_line(make_input_tracks=make_tracks_mva_tracks,
 
     from Functors import (ALL, ETA, P, PT, CHI2DOF, SUM, DOCA, DOCACHI2,
                           BPVDIRA, BPVIPCHI2, MINIP, PHI, RUNNUMBER,
-                          EVENTNUMBER, EVENTTYPE, BPVIPCHI2STATE)
-    from GaudiKernel.SystemOfUnits import MeV, GeV, mm, mrad
+                          EVENTNUMBER, EVENTTYPE, BPVIPCHI2STATE, COMB)
+    from GaudiKernel.SystemOfUnits import MeV
     pv_loc, odin_loc = make_pvs().location, make_odin().location
     if VoidDump is None:
         VoidDump = {
@@ -105,6 +132,8 @@ def debug_two_track_mva_line(make_input_tracks=make_tracks_mva_tracks,
         }
     if ChildCut is None:
         ChildCut = (PT > 250. * MeV)
+    children = PrFilterV2Tracks(
+        ChildCut, Input=make_input_tracks()['v2Sel']).Output
     if ChildDump is None:
         ChildDump = {
             'P': P,
@@ -119,13 +148,26 @@ def debug_two_track_mva_line(make_input_tracks=make_tracks_mva_tracks,
         VertexCut = (CHI2DOF < 1e3)
     if VertexDump is None:
         VertexDump = {
-            'PT': PT,
-            'ETA': ETA,
-            'PHI': PHI,
-            'CHI2DOF': CHI2DOF,
-            'DIRA': BPVDIRA(pv_loc),
-            'IPCHI2': BPVIPCHI2(pv_loc),
-            'IPCHI2_STATE': BPVIPCHI2STATE(pv_loc),
+            'PT':
+            PT,
+            'ETA':
+            ETA,
+            'PHI':
+            PHI,
+            'CHI2DOF':
+            CHI2DOF,
+            'DIRA':
+            BPVDIRA(pv_loc),
+            'IPCHI2':
+            BPVIPCHI2(pv_loc),
+            'IPCHI2_STATE':
+            BPVIPCHI2STATE(pv_loc),
+            'SUMPT':
+            COMB(
+                Functor=SUM(PT),
+                ChildContainers=[children.location],
+                ChildContainerTypes=[('Pr::Selection<LHCb::Event::v2::Track>',
+                                      'PrKernel/PrSelection.h')]),
         }
     if CombinationCut is None:
         CombinationCut = ALL
@@ -136,8 +178,6 @@ def debug_two_track_mva_line(make_input_tracks=make_tracks_mva_tracks,
             'SUMPT': SUM(PT),
             'DOCACHI2': DOCACHI2,
         }
-    children = PrFilterV2Tracks(
-        ChildCut, Input=make_input_tracks()['v2Sel']).Output
     combiner = CombineTracks(
         NBodies=2,
         VertexCut=VertexCut,

Hlt/RecoConf/python/RecoConf/hlt1_tracking.py

@@ -13,6 +13,7 @@ from PyConf.components import (
     make_algorithm,
     Algorithm,
     force_location,
+    setup_component,
 )
 
 from Configurables import (
@@ -43,6 +44,7 @@ from PyConf.Algorithms import (
     SciFiTrackForwardingStoreHit,
     VeloKalman,
     createODIN,
+    LHCb__MDF__IOAlg,
 )
 
 from GaudiKernel.SystemOfUnits import mm, MeV
@@ -74,10 +76,21 @@ EmptyFilter = make_algorithm(
     VoidFilter, input_transform=__emptyfilter_input_transform)
 
 
-def make_raw_data():
+def make_RawData():
     return RawData().RawEvent
 
 
+def make_RawData_IOSvc(inputFiles):
+    setup_component('LHCb__MDF__IOSvcMM', Input=inputFiles)
+    return LHCb__MDF__IOAlg(
+        IOSvc="LHCb::MDF::IOSvcMM/LHCb__MDF__IOSvcMM").RawEventLocation
+
+
+@configurable
+def make_raw_data(make_raw=make_RawData):
+    return make_raw()
+
+
 @configurable
 def make_odin(make_raw_data=make_raw_data):
     return createODIN(RawEvent=make_raw_data()).ODIN

PyConf/python/PyConf/environment.py

@@ -75,12 +75,19 @@ def _output_writer(writer_cls, filename, **kwargs):
     return writer_cls(**kwargs)
 
 
-def setupInput(inputFiles, dataType, DDDBTag, CONDDBTag, Simulation,
-               inputFileType):
+def setupInput(inputFiles,
+               dataType,
+               DDDBTag,
+               CONDDBTag,
+               Simulation,
+               inputFileType,
+               outputFileType=None,
+               withEventSelector=True,
+               TESName='HiveWhiteBoard'):
     # FIXME(AP) we need to modify the ApplicationMgr and query the
     # HiveWhiteBoard held by the EverythingHandler; the former modifies global
     # state which is not ideal
-    whiteboard = setup_component('HiveWhiteBoard', instanceName='EventDataSvc')
+    whiteboard = setup_component(TESName, instanceName='EventDataSvc')
     if inputFileType != 'MDF' and whiteboard.EventSlots > 1:
         raise ConfigurationError(
             "only MDF files can run in multithreaded mode, please change number of eventslots to 1"
@@ -88,8 +95,15 @@ def setupInput(inputFiles, dataType, DDDBTag, CONDDBTag, Simulation,
     setup_component(
         'ApplicationMgr',
         packageName='Gaudi.Configuration',
-        EvtSel="EventSelector")
-    input_iohelper = IOHelper(inputFileType, None)
+        EvtSel="EventSelector" if withEventSelector else "NONE")
+    setup_component('DDDBConf', Simulation=Simulation, DataType=dataType)
+    setup_component(
+        'CondDB', Upgrade=True, Tags={
+            'DDDB': DDDBTag,
+            'SIMCOND': CONDDBTag
+        })
+    if not withEventSelector: return inputFiles
+    input_iohelper = IOHelper(inputFileType, outputFileType)
     input_iohelper.setupServices()
     evtSel = input_iohelper.inputFiles(inputFiles, clear=True)
     inputs = []
@@ -97,16 +111,16 @@ def setupInput(inputFiles, dataType, DDDBTag, CONDDBTag, Simulation,
         inputs.append(inp + " IgnoreChecksum='YES'")
     evtSel.Input = inputs
     evtSel.PrintFreq = 10000
-    setup_component('DDDBConf', Simulation=Simulation, DataType=dataType)
-    setup_component(
-        'CondDB', Upgrade=True, Tags={
-            'DDDB': DDDBTag,
-            'SIMCOND': CONDDBTag
-        })
     setup_component('IODataManager', DisablePFNWarning=True)
+    return inputs
 
 
-def setupInputFromTestFileDB(testFileDBkey, inputFiles=None, fileType=None):
+def setupInputFromTestFileDB(testFileDBkey,
+                             inputFiles=None,
+                             fileType=None,
+                             outputFileType=None,
+                             withEventSelector=True,
+                             TESName='HiveWhiteBoard'):
     """Run from files defined by a TestFileDB key.
 
     Parameters
@@ -128,10 +142,11 @@ def setupInputFromTestFileDB(testFileDBkey, inputFiles=None, fileType=None):
     DDDBTag = qualifiers['DDDB']
     if not inputFiles:
         inputFiles = test_file_db[testFileDBkey].filenames
-    setupInput(inputFiles, dataType, DDDBTag, CondDBTag, Simulation, fileType)
+    return setupInput(inputFiles, dataType, DDDBTag, CondDBTag, Simulation,
+                      fileType, outputFileType, withEventSelector, TESName)
 
 
-def setupOutput(filename, filetype):
+def setupOutput(filename, filetype, TESName='HiveWhiteBoard'):
     """Configure the application to write out a file.
 
     Only the raw event, under `/Event/DAQ/RawEvent`, is persisted.
@@ -216,7 +231,8 @@ class EverythingHandler(object):
                  nEventSlots=0,
                  evtMax=-1,
                  debug=True,
-                 HistoFile=None):
+                 HistoFile=None,
+                 TESName='HiveWhiteBoard'):
         self._nodes = []
         self._algs = []
         self._tools = []  # public tools
@@ -225,7 +241,7 @@ class EverythingHandler(object):
         if nEventSlots == 0:
             nEventSlots = int(threadPoolSize * 1.2)  # sensible default
         self._whiteboard = setup_component(
-            'HiveWhiteBoard',
+            TESName,
             instanceName='EventDataSvc',
             EventSlots=nEventSlots,
             ForceLeaves=True)