From 5c3ff1a8e6cfe471e3de714459d79017d0e392f0 Mon Sep 17 00:00:00 2001
From: Patrick Spradlin <patrick.spradlin@cern.ch>
Date: Sun, 17 Oct 2021 15:08:13 +0100
Subject: [PATCH 1/2] Update Moore allen_control_flow to transfer DecReports
 and SelReports from the Allen host buffers.  Added a new test for the
 SelReports saved to MDF files in this way.

---
 Hlt/Hlt1Conf/CMakeLists.txt                   |   1 +
 .../options/test_allen_comp_dec_sel_rep.py    | 100 ++++++++++++++++++
 .../tests/options/test_allen_decreports.py    |   5 +
 .../mdf_compare_dec_sel_allen.qmt             |  35 ++++++
 Hlt/Moore/python/Moore/config.py              |  11 +-
 Hlt/RecoConf/python/RecoConf/hlt1_allen.py    |  38 ++++++-
 6 files changed, 181 insertions(+), 9 deletions(-)
 create mode 100644 Hlt/Hlt1Conf/tests/options/test_allen_comp_dec_sel_rep.py
 create mode 100644 Hlt/Hlt1Conf/tests/qmtest/persistency.qms/mdf_compare_dec_sel_allen.qmt

diff --git a/Hlt/Hlt1Conf/CMakeLists.txt b/Hlt/Hlt1Conf/CMakeLists.txt
index e9c708ded14..dbe9103c7cc 100644
--- a/Hlt/Hlt1Conf/CMakeLists.txt
+++ b/Hlt/Hlt1Conf/CMakeLists.txt
@@ -25,6 +25,7 @@ if(NOT Allen_FOUND)
         Hlt1Conf.persistency.allen_mdf_write
         Hlt1Conf.persistency.mdf_read_decs_allen
         Hlt1Conf.persistency.hlt2_read_hlt1_filtered_mdf
+        Hlt1Conf.persistency.mdf_compare_dec_sel_allen
     )
     foreach(test_name IN LISTS allen_tests)
         set_property(TEST ${test_name} PROPERTY DISABLED TRUE)
diff --git a/Hlt/Hlt1Conf/tests/options/test_allen_comp_dec_sel_rep.py b/Hlt/Hlt1Conf/tests/options/test_allen_comp_dec_sel_rep.py
new file mode 100644
index 00000000000..2eee09dcff9
--- /dev/null
+++ b/Hlt/Hlt1Conf/tests/options/test_allen_comp_dec_sel_rep.py
@@ -0,0 +1,100 @@
+###############################################################################
+# (c) Copyright 2000-2021 CERN for the benefit of the LHCb Collaboration      #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+"""Check the consistency of the line decisions and presence of SelReport
+
+Takes two outputs from previously-run jobs:
+  1. The MDF
+  2. The TCK in TCKData/config.cdb
+For each line with a decision in the DecReport, the presence of a
+corresponding SelReport is checked.
+Two error conditions are recognized and considered failures of the test:
+  - A positive line decision without a corresponding SelReport
+  - A SelReport for a line with a negative decision
+"""
+from __future__ import print_function
+import argparse
+from collections import defaultdict
+from Configurables import (ApplicationMgr, HistogramPersistencySvc,
+                           IODataManager, LHCbApp, ConfigCDBAccessSvc)
+from DAQSys.Decoders import DecoderDB
+from GaudiConf import IOHelper
+import GaudiPython
+from Moore.config import setup_ann_service, get_allen_hlt1_decision_ids
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--input-mdf", help="Input MDF file")
+args = parser.parse_args()
+
+# Setup ANNsvc defaults (for TCK 0)
+hlt1_decision_ids = get_allen_hlt1_decision_ids()
+setup_ann_service(hlt1_decision_ids, {}, {})
+
+# Configure basic application with inputs
+LHCbApp(DataType="Upgrade", Simulation=True)
+IOHelper("MDF").inputFiles([args.input_mdf])
+# Disable warning about not being able to navigate ancestors
+IODataManager(DisablePFNWarning=True)
+# Disable warning about histogram saving not being required
+HistogramPersistencySvc(OutputLevel=5)
+# Decode Hlt DecReports
+ApplicationMgr(TopAlg=[
+    DecoderDB["HltDecReportsDecoder/Hlt1DecReportsDecoder"].setup(),
+    DecoderDB["HltSelReportsDecoder/Hlt1SelReportsDecoder"].setup()
+])
+
+ConfigCDBAccessSvc().File = "TCKData/config.cdb"
+
+# Set up counters for recording decisions and selreport existence from MDF
+counts_from_mdf = defaultdict(lambda: defaultdict(int))
+
+gaudi = GaudiPython.AppMgr()
+TES = gaudi.evtSvc()
+gaudi.run(1)
+
+error = False
+while TES["/Event"]:
+    decs = TES["/Event/Hlt1/DecReports"]
+    if not decs:
+        print("DecReports TES location not found")
+        error = True
+        break
+
+    sels = TES["/Event/Hlt1/SelReports"]
+    if not sels:
+        print("SelReports TES location not found")
+        error = True
+        break
+
+    for key in decs.decisionNames():
+        report = decs.decReport(key)
+        decision = report.decision()
+        hassel = sels.hasSelectionName(key)
+        counts_from_mdf[key][(decision, hassel)] += 1
+
+    gaudi.run(1)
+
+# Check for any inconsistent instances:
+missing_sel = (1, False)  # Event accepted by line without a SelReport
+extra_sel = (0, True)  # SelReport for a line that did not accept the event
+for key in counts_from_mdf.keys():
+    counters = counts_from_mdf[key]
+    line_name = key
+    if missing_sel in counters.keys():
+        error = True
+        print("Test ERROR: Missing SelReport for {}".format(line_name))
+    if extra_sel in counters.keys():
+        error = True
+        print("Test ERROR: Spurious SelReport for {}".format(line_name))
+
+    print("Coincidence counts for {}:  {}".format(line_name, counters))
+
+if error:
+    exit("Test failed")  # exit with a non-zero code
diff --git a/Hlt/Hlt1Conf/tests/options/test_allen_decreports.py b/Hlt/Hlt1Conf/tests/options/test_allen_decreports.py
index 46b760a83e3..48798c53819 100644
--- a/Hlt/Hlt1Conf/tests/options/test_allen_decreports.py
+++ b/Hlt/Hlt1Conf/tests/options/test_allen_decreports.py
@@ -24,6 +24,7 @@ from Configurables import (ApplicationMgr, HistogramPersistencySvc,
 from DAQSys.Decoders import DecoderDB
 from GaudiConf import IOHelper
 import GaudiPython
+from Moore.config import setup_ann_service, get_allen_hlt1_decision_ids
 
 
 def get_counts_from_log(f):
@@ -49,6 +50,10 @@ parser.add_argument("--input-mdf", help="Input MDF file")
 parser.add_argument("--input-log", help="Input log file")
 args = parser.parse_args()
 
+# Setup ANNsvc defaults (for TCK 0)
+hlt1_decision_ids = get_allen_hlt1_decision_ids()
+setup_ann_service(hlt1_decision_ids, {}, {})
+
 # Configure basic application with inputs
 LHCbApp(DataType="Upgrade", Simulation=True)
 IOHelper("MDF").inputFiles([args.input_mdf])
diff --git a/Hlt/Hlt1Conf/tests/qmtest/persistency.qms/mdf_compare_dec_sel_allen.qmt b/Hlt/Hlt1Conf/tests/qmtest/persistency.qms/mdf_compare_dec_sel_allen.qmt
new file mode 100644
index 00000000000..b5352a50bb1
--- /dev/null
+++ b/Hlt/Hlt1Conf/tests/qmtest/persistency.qms/mdf_compare_dec_sel_allen.qmt
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE extension  PUBLIC '-//QM/2.3/Extension//EN'  'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
+<!--
+    (c) Copyright 2000-2021 CERN for the benefit of the LHCb Collaboration
+
+    This software is distributed under the terms of the GNU General Public
+    Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
+
+    In applying this licence, CERN does not waive the privileges and immunities
+    granted to it by virtue of its status as an Intergovernmental Organization
+    or submit itself to any jurisdiction.
+-->
+<!--
+Check content of Allen DecReports and SelReports rawbanks and check that they
+have consistent entries.
+-->
+<extension class="GaudiTest.GaudiExeTest" kind="test">
+<argument name="prerequisites"><set>
+  <tuple><text>persistency.allen_mdf_write</text><enumeral>PASS</enumeral></tuple>
+  <tuple><text>persistency.make_allen_tck</text><enumeral>PASS</enumeral></tuple>
+  <tuple><text>persistency.assign_allen_tck</text><enumeral>PASS</enumeral></tuple>
+</set></argument>
+<argument name="program"><text>python</text></argument>
+<argument name="args"><set>
+  <text>$HLT1CONFROOT/tests/options/test_allen_comp_dec_sel_rep.py</text>
+  <text>--input-mdf=test_allen_hlt1_persistence_mdf_write.mdf</text>
+</set></argument>
+<argument name="use_temp_dir"><enumeral>true</enumeral></argument>
+<argument name="validator"><text>
+
+from Moore.qmtest.exclusions import preprocessor
+countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0},
+                stdout=preprocessor(stdout))
+
+</text></argument>
+</extension>
diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py
index c29d7e65755..cebc7578ecf 100644
--- a/Hlt/Moore/python/Moore/config.py
+++ b/Hlt/Moore/python/Moore/config.py
@@ -600,7 +600,7 @@ def get_allen_hlt1_decision_ids():
 
 
 def allen_control_flow(options):
-    from RecoConf.hlt1_allen import call_allen
+    from RecoConf.hlt1_allen import (call_allen, call_allen_raw_sel_reports)
     from Allen.config import setup_allen_non_event_data_service
 
     options.finalize()
@@ -609,15 +609,14 @@ def allen_control_flow(options):
 
     # Write DecReports raw banks
     call_allen_alg = call_allen(filter_hlt1=True, tck=options.tck)
-    allen_dec_reports = call_allen_alg.DecReportsLocation
+    allen_output = call_allen_alg.AllenOutput
 
     algs = []
     new_hlt_banks = {}
 
-    drw = HltDecReportsWriter(
-        InputHltDecReportsLocation=allen_dec_reports, SourceID='Hlt1')
-    algs.extend([drw])
-    new_hlt_banks['HltDecReports'] = drw.OutputRawEventLocation
+    srw = call_allen_raw_sel_reports(allen_output=lambda: allen_output)
+    algs.extend([srw])
+    new_hlt_banks['HltSelReports'] = srw.OutputSelReports
 
     decision_ids = get_allen_hlt1_decision_ids()
     setup_ann_service(
diff --git a/Hlt/RecoConf/python/RecoConf/hlt1_allen.py b/Hlt/RecoConf/python/RecoConf/hlt1_allen.py
index 88df0ef73bf..9fa7f090560 100644
--- a/Hlt/RecoConf/python/RecoConf/hlt1_allen.py
+++ b/Hlt/RecoConf/python/RecoConf/hlt1_allen.py
@@ -16,7 +16,8 @@ from PyConf.application import default_raw_event, make_odin
 from PyConf.Algorithms import (
     LHCb__Converters__Track__v1__fromV2TrackV1Track as FromV2TrackV1Track,
     AllenVeloToV2Tracks, AllenUTToV2Tracks, AllenForwardToV2Tracks,
-    AllenPVsToRecVertexV2, RunAllen, TransposeRawBanks)
+    AllenPVsToRecVertexV2, AllenSelReportsToTES, RunAllen,
+    TransposeRawBanks)
 
 
 def make_transposed_raw_banks(make_raw=default_raw_event):
@@ -53,8 +54,17 @@ def make_allen_output():
     return call_allen().AllenOutput
 
 
-def make_allen_dec_reports():
-    return call_allen().DecReportsLocation
+def make_allen_raw_dec_reports(allen_output=make_allen_output):
+    return call_allen_raw_dec_reports(allen_output).OutputSelReports
+
+
+def make_allen_raw_sel_reports(allen_output=make_allen_output):
+    return call_allen_raw_sel_reports(allen_output).OutputSelReports
+
+
+def make_allen_dec_reports(raw_dec_reports=make_allen_raw_dec_reports):
+    decoder =  decode_allen_dec_reports(raw_dec_reports)
+    return decoder.OutputHltDecReportsLocation
 
 
 @configurable
@@ -76,6 +86,28 @@ def call_allen(odin_location=make_odin,
         TCK=tck)
 
 
+def call_allen_raw_dec_reports(allen_output=make_allen_output):
+    return AllenSelReportsToTES(AllenOutput=allen_output())
+
+
+def call_allen_raw_sel_reports(allen_output=make_allen_output):
+    return AllenSelReportsToTES(AllenOutput=allen_output())
+
+
+def decode_allen_dec_reports(raw_dec_reports=make_allen_raw_dec_reports):
+    from PyConf.Algorithms import HltDecReportsDecoder
+    decoder = HltDecReportsDecoder(RawEventLocations=raw_dec_reports(),
+        SourceID='Hlt1')
+    return decoder
+
+
+def decode_allen_sel_reports(raw_sel_reports=make_allen_raw_sel_reports):
+    from PyConf.Algorithms import HltSelReportsDecoder
+    decoder = HltSelReportsDecoder(RawEventLocations=raw_sel_reports(),
+        SourceID='Hlt1')
+    return decoder
+
+
 def make_allen_velo_tracks():
     allen_output = make_allen_output()
     velo_tracks_v2 = AllenVeloToV2Tracks(AllenOutput=allen_output).OutputTracks
-- 
GitLab


From 3f72037f6a5b661852c972d598cac74a1cd32a8d Mon Sep 17 00:00:00 2001
From: Patrick Spradlin <patrick.spradlin@cern.ch>
Date: Mon, 18 Oct 2021 15:29:47 +0100
Subject: [PATCH 2/2] Updated Hlt1 interface.

---
 Hlt/Moore/python/Moore/config.py           |  6 ++--
 Hlt/RecoConf/python/RecoConf/hlt1_allen.py | 38 ++++++++--------------
 2 files changed, 17 insertions(+), 27 deletions(-)

diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py
index cebc7578ecf..2936c3182e0 100644
--- a/Hlt/Moore/python/Moore/config.py
+++ b/Hlt/Moore/python/Moore/config.py
@@ -600,7 +600,7 @@ def get_allen_hlt1_decision_ids():
 
 
 def allen_control_flow(options):
-    from RecoConf.hlt1_allen import (call_allen, call_allen_raw_sel_reports)
+    from RecoConf.hlt1_allen import (call_allen, call_allen_raw_reports)
     from Allen.config import setup_allen_non_event_data_service
 
     options.finalize()
@@ -614,9 +614,9 @@ def allen_control_flow(options):
     algs = []
     new_hlt_banks = {}
 
-    srw = call_allen_raw_sel_reports(allen_output=lambda: allen_output)
+    srw = call_allen_raw_reports(allen_output=lambda: allen_output)
     algs.extend([srw])
-    new_hlt_banks['HltSelReports'] = srw.OutputSelReports
+    new_hlt_banks['HltSelReports'] = srw.OutputRawReports
 
     decision_ids = get_allen_hlt1_decision_ids()
     setup_ann_service(
diff --git a/Hlt/RecoConf/python/RecoConf/hlt1_allen.py b/Hlt/RecoConf/python/RecoConf/hlt1_allen.py
index 9fa7f090560..96ff5f752b5 100644
--- a/Hlt/RecoConf/python/RecoConf/hlt1_allen.py
+++ b/Hlt/RecoConf/python/RecoConf/hlt1_allen.py
@@ -16,8 +16,7 @@ from PyConf.application import default_raw_event, make_odin
 from PyConf.Algorithms import (
     LHCb__Converters__Track__v1__fromV2TrackV1Track as FromV2TrackV1Track,
     AllenVeloToV2Tracks, AllenUTToV2Tracks, AllenForwardToV2Tracks,
-    AllenPVsToRecVertexV2, AllenSelReportsToTES, RunAllen,
-    TransposeRawBanks)
+    AllenPVsToRecVertexV2, AllenReportsToRawEvent, RunAllen, TransposeRawBanks)
 
 
 def make_transposed_raw_banks(make_raw=default_raw_event):
@@ -54,16 +53,12 @@ def make_allen_output():
     return call_allen().AllenOutput
 
 
-def make_allen_raw_dec_reports(allen_output=make_allen_output):
-    return call_allen_raw_dec_reports(allen_output).OutputSelReports
+def make_allen_raw_reports(allen_output=make_allen_output):
+    return call_allen_raw_reports(allen_output).OutputRawReports
 
 
-def make_allen_raw_sel_reports(allen_output=make_allen_output):
-    return call_allen_raw_sel_reports(allen_output).OutputSelReports
-
-
-def make_allen_dec_reports(raw_dec_reports=make_allen_raw_dec_reports):
-    decoder =  decode_allen_dec_reports(raw_dec_reports)
+def make_allen_dec_reports(raw_dec_reports=make_allen_raw_reports):
+    decoder = decode_allen_dec_reports(raw_dec_reports)
     return decoder.OutputHltDecReportsLocation
 
 
@@ -82,29 +77,24 @@ def call_allen(odin_location=make_odin,
         JSON=json,
         Sequence=seq,
         FilterHLT1=filter_hlt1,
-        FilterGEC=filter_gec,
-        TCK=tck)
-
-
-def call_allen_raw_dec_reports(allen_output=make_allen_output):
-    return AllenSelReportsToTES(AllenOutput=allen_output())
+        FilterGEC=filter_gec)
 
 
-def call_allen_raw_sel_reports(allen_output=make_allen_output):
-    return AllenSelReportsToTES(AllenOutput=allen_output())
+def call_allen_raw_reports(allen_output=make_allen_output):
+    return AllenReportsToRawEvent(AllenOutput=allen_output())
 
 
-def decode_allen_dec_reports(raw_dec_reports=make_allen_raw_dec_reports):
+def decode_allen_dec_reports(raw_dec_reports=make_allen_raw_reports):
     from PyConf.Algorithms import HltDecReportsDecoder
-    decoder = HltDecReportsDecoder(RawEventLocations=raw_dec_reports(),
-        SourceID='Hlt1')
+    decoder = HltDecReportsDecoder(
+        RawEventLocations=raw_dec_reports(), SourceID='Hlt1')
     return decoder
 
 
-def decode_allen_sel_reports(raw_sel_reports=make_allen_raw_sel_reports):
+def decode_allen_sel_reports(raw_sel_reports=make_allen_raw_reports):
     from PyConf.Algorithms import HltSelReportsDecoder
-    decoder = HltSelReportsDecoder(RawEventLocations=raw_sel_reports(),
-        SourceID='Hlt1')
+    decoder = HltSelReportsDecoder(
+        RawEventLocations=raw_sel_reports(), SourceID='Hlt1')
     return decoder
 
 
-- 
GitLab