[RTADPA BW Tests] Introducing an Hlt1-bandwidth test via Moore_in_Allen

Adds a new BW test: for Nominal Hlt1 via Moore_in_Allen.

Makes several changes to make the scripts less 'X if Hlt2 else Y'.

Changes the main-page, so that the links at the bottom are dependent upon process. (This removes the dead-link on sprucing page, and prevents confusion on the new hlt1 page)

cc: @rjhunter

Goes with: lhcb/Moore!2528 (merged), lhcb-core/LHCbNightlyConf!1094 (merged) and lhcb-core/LHCbPR2HD!263 (merged)

ToDo:

• Insert process-dependence on N_EVTS for now. see: #12 (closed)
• CI-test
• BW chat for review (@shunan)
• Post-Merge
  • Allen label
  • Dmitry Popov "magic step" to register new test in LHCbPR

python/MooreTests/line-and-stream-rates.py

@@ -12,7 +12,8 @@
 import GaudiPython as GP
 from GaudiConf.reading import decoder, unpack_rawevent, hlt_decisions
 from Configurables import (ApplicationMgr, LHCbApp, IODataManager,
-                           EventSelector, createODIN)
+                           EventSelector, createODIN, LHCb__UnpackRawEvent,
+                           HltDecReportsDecoder)
 from GaudiConf import IOHelper
 from PyConf.application import configured_ann_svc
 import operator
@@ -46,6 +47,7 @@ from PRConfig.bandwidth_helpers import FileNameHelper
             6. Bandwidth
 
     When running wg-stream config, returns same figures as above (both per line and per stream)
+    When running streamless-stream config, returns just the per-line information.
 
 '''
 
@@ -247,14 +249,14 @@ if __name__ == '__main__':
         '-p',
         '--process',
         type=str,
-        help='Compute for Hlt2 or Sprucing lines',
-        choices=['hlt2', 'spruce'],
+        help='Compute for Hlt1, Hlt2 or Sprucing lines',
+        choices=['hlt1', 'hlt2', 'spruce'],
         required=True)
     parser.add_argument(
         '--stream-config',
         type=str,
-        help='Choose production or per-WG stream configuration',
-        choices=['production', 'wg'],
+        help='Choose production, per-WG or streamless stream configuration',
+        choices=['streamless', 'production', 'wg'],
         required=True)
     args = parser.parse_args()
 
@@ -264,17 +266,15 @@ if __name__ == '__main__':
 
     input_config = parse_yaml(args.config)
 
-    if args.process == "spruce" and args.stream_config == "production":
+    if args.process == "spruce" and args.stream_config != "wg":
         raise RuntimeError(
-            '"production" stream config not defined for sprucing. Please use "wg".'
+            '"production" and "streamless" stream configs are not defined for sprucing. Please use "wg".'
         )
-
-    LHCbApp(
-        DataType="Upgrade",
-        Simulation=True,
-        DDDBtag="dddb-20171126",
-        CondDBtag="sim-20171127-vc-md100",
-        EvtMax=n_events)
+    if args.process == "hlt1" and args.stream_config != "streamless":
+        raise RuntimeError(
+            '"production" and "wg" stream configs are not defined for hlt1. Please use "streamless".'
+        )
+    LHCbApp(DataType="Upgrade", Simulation=True, EvtMax=n_events)
     EventSelector().PrintFreq = 10000
     IODataManager(DisablePFNWarning=True)
 
@@ -282,43 +282,61 @@ if __name__ == '__main__':
     # because we need to set `input_process='Hlt2'` in `unpack_rawevent`
     # to read MDF output from Sprucing
     algs = []
-    unpack = unpack_rawevent(
-        bank_types=['ODIN', 'HltDecReports', 'DstData', 'HltRoutingBits'],
-        configurables=True)
-    hlt2 = [hlt_decisions(source="Hlt2", output_loc="/Event/Hlt2/DecReports")]
-    if args.process == 'spruce':
-        spruce = [
-            hlt_decisions(
-                source="Spruce", output_loc="/Event/Spruce/DecReports")
-        ]
-    else:
-        spruce = []
-    decoder = decoder(input_process=args.process.capitalize())
-    algs = [unpack] + hlt2 + spruce + [decoder] + [createODIN(ODIN='myODIN')]
-
-    appMgr = ApplicationMgr(TopAlg=algs)
-    appMgr.ExtSvc += [
-        configured_ann_svc(json_file=fname_helper.tck(args.stream_config))
-    ]
+    with open(fname_helper.stream_config_json_path(args.stream_config)) as f:
+        lines = json.load(f)[args.stream]
 
     IOHelper("MDF").inputFiles(
         [fname_helper.mdf_fname_for_reading(args.stream_config, args.stream)])
 
-    with open(fname_helper.stream_config_json_path(args.stream_config)) as f:
-        lines = json.load(f)[args.stream]
-
+    # Hlt1 requires different unpacking than hlt2/sprucing.
+    if args.process == "hlt1":
+        unpacker = LHCb__UnpackRawEvent(
+            "UnpackRawEvent",
+            RawBankLocations=["DAQ/RawBanks/HltDecReports"],
+            BankTypes=["HltDecReports"])
+        decDec = HltDecReportsDecoder(
+            "HltDecReportsDecoder/Hlt1DecReportsDecoder",
+            OutputHltDecReportsLocation="/Event/Hlt1/DecReports",
+            SourceID="Hlt1",
+            DecoderMapping="TCKANNSvc",
+            RawBanks=unpacker.RawBankLocations[0])
+        appMgr = ApplicationMgr(TopAlg=[unpacker, decDec])
+        appMgr.ExtSvc += [configured_ann_svc(name='TCKANNSvc')]
+    else:
+        unpack = unpack_rawevent(
+            bank_types=['ODIN', 'HltDecReports', 'DstData', 'HltRoutingBits'],
+            configurables=True)
+        hlt2 = [
+            hlt_decisions(source="Hlt2", output_loc="/Event/Hlt2/DecReports")
+        ]
+        if args.process == 'spruce':
+            spruce = [
+                hlt_decisions(
+                    source="Spruce", output_loc="/Event/Spruce/DecReports")
+            ]
+        else:
+            spruce = []
+        decoder = decoder(input_process=args.process.capitalize())
+        algs = [unpack] + hlt2 + spruce + [decoder
+                                           ] + [createODIN(ODIN='myODIN')]
+        appMgr = ApplicationMgr(TopAlg=algs)
+        appMgr.ExtSvc += [
+            configured_ann_svc(json_file=fname_helper.tck(args.stream_config))
+        ]
     appMgr = GP.AppMgr()
     evt = appMgr.evtsvc()
 
-    # Calculates retention, rate and bandwidth per line and stream (file)
     evts_all, rawbanks_all, dst_all, event_stats, exclusive, raw, dst = processing_events_per_line_and_stream(
         LHCbApp().EvtMax, lines, args.process)
-    rates_per_line(
-        event_stats, exclusive, raw, dst, input_config['input_rate'],
-        fname_helper.tmp_rate_table_per_line_path(args.stream_config,
-                                                  args.stream))
+
+    # Calculate key quantities per stream
     rates_per_stream(
         evts_all, rawbanks_all, dst_all, args.stream,
         input_config['input_rate'],
         fname_helper.tmp_rate_table_per_stream_path(args.stream_config,
                                                     args.stream))
+    # Calculate key quantities per line
+    rates_per_line(
+        event_stats, exclusive, raw, dst, input_config['input_rate'],
+        fname_helper.tmp_rate_table_per_line_path(args.stream_config,
+                                                  args.stream))