From d157f0fa8fed38be1baee01ae65d40ff10479321 Mon Sep 17 00:00:00 2001
From: Shunan Zhang <shunan.zhang@cern.ch>
Date: Mon, 11 Apr 2022 12:51:14 +0800
Subject: [PATCH] add spruce_all_lines_analytics for rate test
---
.../sprucing/spruce_all_lines_analytics.py | 155 ++++++++++++++++++
1 file changed, 155 insertions(+)
create mode 100644 Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_analytics.py
diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_analytics.py b/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_analytics.py
new file mode 100644
index 00000000000..a7f3bfa20be
--- /dev/null
+++ b/Hlt/Hlt2Conf/options/sprucing/spruce_all_lines_analytics.py
@@ -0,0 +1,155 @@
+###############################################################################
+# (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+"""
+Run with
+`./run gaudirun.py spruce_all_lines_analytics.py`
+
+Performs event size analytics on Sprucing events by exploiting the fact that each
+stream gets its own instance of `CombineRawBankViewsToRawEvent`.
+
+-Make each Sprucing line its own stream.
+-Report size of DstData RawBank (contains the physics and reconstruction objects) and
+ total event size where the event contains the RawBanks defined in `banks` in this options file
+-Will not produce output files through use of `stream_writer.bind(write=False)`.
+-In log will see StatCounter with name of line
+
+LineName... INFO Number of counters : X
+ | Counter | # | sum | mean/eff^* | rms/err^* | min | max |
+ | "DstData bank size (bytes)" | X | Y | YY | YYY | YYYY | YYYYY |
+ | "Event size (bytes)" | X | Z | ZZ | ZZZ | ZZZZ | ZZZZZ |
+
+
+Note that by running `CombineRawBankViewsToRawEvent` in VERBOSE mode - achieved through
+`CombineRawBankViewsToRawEvent.bind(OutputLevel=1)` one also has the log of the sizes of ALL RawBanks in `banks`.
+
+Runs Sprucing over HLT1 filtered Min bias sample that has been processed by TOPO{2, 3} HLT2 lines - 478 events.
+If you do not see this for your line then your line did not fire on any event in the input data.
+
+Eventually we should have a test like this over each stream as that will define `banks` and hence the appropriate
+event size. At the moment this script runs over ALL Sprucing lines and analyses ALL RawBanks so the event size contains
+everything.
+
+"""
+
+import json, re
+from Moore import options, run_moore
+from PyConf.Algorithms import CombineRawBankViewsToRawEvent
+from RecoConf.global_tools import stateProvider_with_simplified_geom
+from RecoConf.reconstruction_objects import reconstruction
+
+import XRootD.client
+from Configurables import HltANNSvc
+from PyConf import ConfigurationError
+
+from Moore.streams_hlt2 import DETECTOR_RAW_BANK_TYPES, HLT1_REPORT_RAW_BANK_TYPES, HLT2_REPORT_RAW_BANK_TYPES
+import Moore.streams_spruce
+
+from Hlt2Conf.lines import sprucing_lines as all_lines
+from Hlt2Conf.lines.b_to_charmonia import sprucing_lines as b2cc_lines
+from Hlt2Conf.lines.b_to_open_charm import sprucing_lines as b2oc_lines
+from Hlt2Conf.lines.bandq import sprucing_lines as bandq_lines
+from Hlt2Conf.lines.semileptonic import sprucing_lines as sl_lines
+from Hlt2Conf.lines.rd import sprucing_lines as rd_lines
+#from Hlt2Conf.lines.charm import sprucing_lines as charm_lines
+from Hlt2Conf.lines.qee import sprucing_lines as qee_lines
+
+########################
+WGs = {
+ "b2oc": b2oc_lines,
+ "bandq": bandq_lines,
+ "b2cc": b2cc_lines,
+ "sl": sl_lines,
+ #"charm": charm_lines,
+ "qee": qee_lines,
+ "rd": rd_lines,
+ "all": all_lines
+}
+
+##CHANGE wg to desired
+wg = 'all'
+########################
+if wg not in WGs.keys():
+ raise ConfigurationError('Working group ("wg") not recognised.')
+
+## Return HltANNSvc when tck is on eos
+def tck_from_eos(url):
+ with XRootD.client.File() as f:
+ status, _ = f.open(url)
+ if not status.ok:
+ raise RuntimeError(f"could not open {url}: {status.message}")
+ status, data = f.read()
+ if not status.ok:
+ raise RuntimeError(f"could not read {url}: {status.message}")
+ dict = json.loads(data.decode('utf-8'))
+ ann_config = dict["HltANNSvc/HltANNSvc"]
+ packed_object_locs = {
+ str(k): v
+ for k, v in ann_config["PackedObjectLocations"].items()
+ }
+ hlt2_sel_ids = {
+ str(k): v
+ for k, v in ann_config["Hlt2SelectionID"].items()
+ }
+
+ return HltANNSvc(
+ "HltANNSvcReading",
+ Hlt2SelectionID=hlt2_sel_ids,
+ PackedObjectLocations=packed_object_locs,
+ )
+
+
+## Configure `HltANNSvc`
+url = 'root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp1/hlt2_persistreco_output/hlt2_persistreco_realtime.tck.json'
+tck_from_eos(url)
+
+options.input_raw_format = 0.3
+options.output_type = 'ROOT'
+options.monitoring_file = "monitoring.json"
+
+# Create new dictionary based on `sprucing_lines` with "readable" line names:
+# SpruceRD_LbToKpTauE_TauTo3Pi_OS_Line -> RD_LbToKpTauE_TauTo3Pi_OS
+sprucing_lines = WGs[wg]
+streamdict = {}
+for k, v in sprucing_lines.items():
+ try:
+ wg, name = re.search('Spruce(.+?)_', k).group(1), re.search(
+ '_(.+?)_Line', k).group(1)
+ except AttributeError:
+ wg, name = re.search('Spruce(.+?)_', k).group(1), re.search(
+ '_(.+?)Line', k).group(1)
+
+ streamdict[wg + "_" + name] = [v]
+
+
+def make_streams():
+ "Makes each line its own stream"
+ dict = {}
+ for k, v in streamdict.items():
+ # v can only contain one element
+ dict[k] = [v[0]()]
+
+ return dict
+
+
+##Must declare "new" streams (one for each line) and the RawBanks you wish to analyse
+banks = set.union(DETECTOR_RAW_BANK_TYPES, HLT1_REPORT_RAW_BANK_TYPES,
+ HLT2_REPORT_RAW_BANK_TYPES)
+Moore.streams_spruce.stream_banks = {
+ k: [i for i in banks]
+ for k in streamdict.keys()
+}
+
+public_tools = [stateProvider_with_simplified_geom()]
+
+with reconstruction.bind(
+ from_file=True,
+ spruce=True), CombineRawBankViewsToRawEvent.bind(OutputLevel=4):
+ config = run_moore(options, make_streams, public_tools, analytics=True)
--
GitLab