Changes to make streaming setup more realistic

Hlt/Hlt2Conf/options/fest/hlt2_all_lines_with_reco_with_streams.py

@@ -67,7 +67,7 @@ Moore.streams_hlt2.stream_banks = {
     "sprucing": stream_banks,
     "turbo": stream_banks,
 }
-Moore.streams_hlt2.stream_bits = dict(sprucing=0, turbo=95)
+Moore.streams_hlt2.stream_bits = dict(sprucing=0, turbo=87)
 
 print("Moore.streams_hlt2.stream_banks")
 pprint(Moore.streams_hlt2.stream_banks)

Hlt/Hlt2Conf/options/streaming/hlt2_activity_trigger_2022.py

+###############################################################################
+# (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+"""Options for testing the streaming of HLT2 lines.
+
+Run like any other options file:
+
+    ./Moore/run gaudirun.py hlt2_streaming.py
+"""
+from Moore import options, run_moore
+from Moore.config import get_allen_hlt1_decision_ids
+from Moore.tcks import dump_hlt2_configuration
+from RecoConf.global_tools import stateProvider_with_simplified_geom
+from Moore.lines import Hlt2Line
+
+from pprint import pprint
+# options.set_input_and_conds_from_testfiledb('MiniBrunel_2018_MinBias_FTv4_MDF')
+options.input_files = [
+    "/scratch/rmatev/Run_0000238820_20220720-084749-135_HCEB01_0001.mdf"
+]
+options.dddb_tag = "upgrade/master"
+options.conddb_tag = "upgrade/master"
+options.input_type = 'MDF'
+options.evt_max = 200
+options.output_file = 'hlt2_test_data.{stream}.mdf'
+options.output_type = 'MDF'
+options.simulation = True
+
+from PyConf.Algorithms import VoidFilter
+from RecoConf.reconstruction_objects import reconstruction
+from RecoConf.calorimeter_reconstruction import make_calo_reduced
+import Functors as F
+
+
+def calo_activity_line(name='Hlt2_Calo_activity', prescale=1):
+    calo_clusters = make_calo_reduced()
+    # digits_ecal = calo_clusters["digitsEcal"]
+    clusters_ecal = calo_clusters["ecalClusters"]
+    # activity_filter = VoidFilter(name="CaloActivityFilter", Cut = F.SIZE(digits_ecal)>500)
+    activity_filter = VoidFilter(
+        name="CaloActivityFilter", Cut=F.SIZE(clusters_ecal) > 100)
+    return Hlt2Line(name=name, algs=[activity_filter], prescale=prescale)
+
+
+def hlt1_nobias_line(name='Hlt2_Hlt1nobias', prescale=1):
+    # intentionally have some overlap with the other line in order to test
+    # the routing bits in events going to both streams.
+    return Hlt2Line(
+        name=name,
+        algs=[],
+        hlt1_filter_code="Hlt1_ActivityDecision",
+        prescale=prescale)
+
+
+# def lumi_line():
+# pass through lumi
+
+
+def make_streams():
+    linedict = dict(
+        stream_A=[calo_activity_line()], stream_B=[hlt1_nobias_line()])
+    pprint(linedict)
+    return linedict
+
+
+def make_lines():
+    my_lines = [calo_activity_line(), hlt1_nobias_line()]
+    return my_lines
+
+
+# Modify stream dictionaries for testing purposes
+import Moore.streams_hlt2
+Moore.streams_hlt2.stream_banks = {
+    "stream_A": [
+        'ODIN', 'VP', 'UT', 'FTCluster', 'Rich', 'EcalPacked', 'HcalPacked',
+        'Muon'
+    ],
+    "stream_B": [
+        'ODIN', 'VP', 'UT', 'FTCluster', 'Rich', 'EcalPacked', 'HcalPacked',
+        'Muon'
+    ]
+}
+Moore.streams_hlt2.stream_bits = dict(stream_A=87, stream_B=88)
+
+print("Moore.streams_hlt2.stream_banks")
+pprint(Moore.streams_hlt2.stream_banks)
+
+public_tools = [stateProvider_with_simplified_geom()]
+
+
+def hack_hlt1_decision_ids():
+    decs = {"Hlt1_ActivityDecision": 1}
+    print("I actually use this")
+    return decs
+
+
+with reconstruction.bind(from_file=False):
+    with get_allen_hlt1_decision_ids.substitute(hack_hlt1_decision_ids):
+        config = run_moore(
+            options, make_streams, public_tools, allen_hlt1=True)
+dump_hlt2_configuration(config, "hlt2_activity_trigger_2022.tck.json")

Hlt/Hlt2Conf/options/streaming/hlt2_test_streaming.py

@@ -124,7 +124,7 @@ Moore.streams_hlt2.stream_banks = {
     ['ODIN', 'VP', 'UT', 'FTCluster', 'EcalPacked', 'HcalPacked',
      'Muon']  # Remove RICH raw bank
 }
-Moore.streams_hlt2.stream_bits = dict(test_stream_A=0, test_stream_B=95)
+Moore.streams_hlt2.stream_bits = dict(test_stream_A=0, test_stream_B=90)
 
 print("Moore.streams_hlt2.stream_banks")
 pprint(Moore.streams_hlt2.stream_banks)

Hlt/Hlt2Conf/tests/options/streaming/stream_check.py

@@ -136,11 +136,11 @@ while True:
         if "A" in args.stream:
             if '0' not in on_bits:
                 print("Routing bits ERROR Expected bit 0 in stream A")
-            if '95' in on_bits:
-                print("Routing bits ERROR Did NOT expect bit 95 in stream A")
+            if '90' in on_bits:
+                print("Routing bits ERROR Did NOT expect bit 90 in stream A")
         elif "B" in args.stream:
-            if '95' not in on_bits:
-                print("Routing bits ERROR Expected bit 95 in stream B")
+            if '90' not in on_bits:
+                print("Routing bits ERROR Expected bit 90 in stream B")
             if '0' in on_bits:
                 print("Routing bits ERROR Did NOT expect bit 0 in stream B")
         else:

Hlt/Moore/python/Moore/config.py

@@ -732,6 +732,7 @@ def run_moore(options,
     return config
 
 
+@configurable
 def get_allen_hlt1_decision_ids():
     """
     Read Allen HLT1 decision IDs from the Allen control node

Hlt/Moore/python/Moore/streams_hlt2.py

@@ -13,8 +13,15 @@
 Routing bits are used to decide where to send events, to Hlt2 only, or as well to calibration and monitoring tasks.
 Each bit is set according to trigger line decisions.
 
+The routing bits are defined in https://gitlab.cern.ch/lhcb/runII-routingbits.
+The meaning of some routing bits (eg. 40, 94, 95 ) has been agreed on and should not be changed.
+
 """
 from PyConf.utilities import ConfigurationError
+import re
+
+PHYSICS = "Hlt2(?!Lumi).*Decision"
+LUMI = "Hlt2LumiDecision"
 
 DETECTOR_RAW_BANK_TYPES = {
     'ODIN',
@@ -36,7 +43,7 @@ HLT2_REPORT_RAW_BANK_TYPES = {
     'HltDecReports', 'HltRoutingBits', 'HltLumiSummary', 'DstData'
 }
 
-stream_bits = dict(charm=2, b2cc=34, default=95, test=0)
+stream_bits = dict(charm=2, b2cc=34, default=85, test=0)
 stream_banks = {key: list(DETECTOR_RAW_BANK_TYPES) for key in stream_bits}
 stream_banks['test'].remove('Rich')
 
@@ -54,7 +61,24 @@ def get_default_routing_bits(streams):
     # Temporarily hard-code a default mapping of streams to bits, see #268
     routingBits = {}
     for stream, lines_in_stream in streams.items():
+        # Bit for stream
+        if (stream_bits[stream] == 94 or stream_bits[stream] == 95):
+            raise ConfigurationError(
+                f"Bits 94 and 95 are reserved. Do not use for stream {stream}."
+            )
+
         routingBits[stream_bits[stream]] = [
             l.decision_name for l in lines_in_stream
         ]
+        # Lumi bit, always 94
+        routingBits[94] = [
+            l.decision_name for l in lines_in_stream
+            if re.search(LUMI, l.decision_name)
+        ]
+        # Physics bit, always 95
+        routingBits[95] = [
+            l.decision_name for l in lines_in_stream
+            if re.search(PHYSICS, l.decision_name)
+        ]
+
     return routingBits

Hlt/RecoConf/python/RecoConf/hlt1_tracking.py

@@ -76,6 +76,7 @@ def require_gec(make_raw=default_raw_banks, cut=-1, **kwargs):
 @configurable
 def make_VeloClusterTrackingSIMD(algorithm=VeloClusterTrackingSIMD,
                                  raw_event=default_raw_event,
+                                 detector=["VP"],
                                  masked_sensors=[]):
     """Simple helper to make sure both, make_VeloClusterTrackingSIMD_tracks and make_VeloClusterTrackingSIMD_hits,
     access the identically configured version of VeloClusterTrackingSIMD
@@ -92,7 +93,8 @@ def make_VeloClusterTrackingSIMD(algorithm=VeloClusterTrackingSIMD,
     my_SensorMasks = [j in masked_sensors for j in range(208)
                       ]  # 208 = LHCb::Pr::Velo::VPInfos::NSensors
     return algorithm(
-        RawEventLocation=raw_event(["VP"]), SensorMasks=tuple(my_SensorMasks))
+        RawEventLocation=raw_event(detector),
+        SensorMasks=tuple(my_SensorMasks))
 
 
 @configurable