TrigOnlineMonitor: Add histograms to monitor number of forks, threads and slots

Changes

Add two new HLT framework online monitoring histograms in TrigOpMonitor:

• MTConfig - showing number of threads and slots in each HLT worker
• MPWorkers - showing ID of each HLT worker, which allows to determine how many workers there are per HLT node

It is not possible to monitor the number of forks directly on the athena side, but the information can be easily deduced by just looking at the worker IDs from the latter histogram.

Motivation

It came up during one of the Technical Runs at P1 that it would be useful to determine the MP/MT configuration while running by just looking at the online monitoring histograms. Otherwise, the configuration is only present somewhere in OKS and one needs to know where to look for it.

Somewhat relevant Jira: ATR-19667

Testing

Running with 3 forks with 4 threads and 2 slots each:

athenaHLT.py \
--nprocs=3 --threads=4 --concurrent-events=2 \
-c "doEmptyMenu=True;BFieldAutoConfig=False;" \
--number-of-events=100 \
--file=/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/data18_13TeV.00360026.physics_EnhancedBias.merge.RAW._lb0151._SFO-1._0001.1 \
TriggerJobOpts/runHLT_standalone.py

produced the following histograms:

cc @tamartin, @astruebi, @fwinkl

HLT/Trigger/TrigMonitoring/TrigOnlineMonitor/src/TrigOpMonitor.cxx

@@ -76,6 +76,8 @@ void TrigOpMonitor::handle( const Incident& incident ) {
     fillMagFieldHist();
     fillIOVDbHist();
     fillSubDetHist();
+    const AthenaInterprocess::UpdateAfterFork& updinc = dynamic_cast<const AthenaInterprocess::UpdateAfterFork&>(incident);
+    fillProcThreadHist(updinc.workerID());
   }
 }
 
@@ -128,6 +130,14 @@ StatusCode TrigOpMonitor::bookHists()
 
   m_releaseHist = new TH1I("GeneralOpInfo", "General operational info;;Applications", 1, 0, 1);
 
+  m_mtConfigHist = new TH2I("MTConfig", "Multi-threading configuration", 2, 0, 2, 10, 0, 10);
+  m_mtConfigHist->SetCanExtend(TH1::kYaxis);
+  m_mtConfigHist->GetXaxis()->SetBinLabel(1, "N threads");
+  m_mtConfigHist->GetXaxis()->SetBinLabel(2, "N slots");
+
+  m_workersHist = new TH1I("MPWorkers", "Worker IDs;Worker ID;Number of workers", 10, 0, 10);
+  m_workersHist->SetCanExtend(TH1::kXaxis);
+
   m_subdetHist = new TH2I("Subdetectors", "State of subdetectors", 1, 0, 1, 3, 0, 3);
   m_subdetHist->SetCanExtend(TH1::kXaxis);
   m_subdetHist->GetYaxis()->SetBinLabel(1, "# ROB");
@@ -140,7 +150,10 @@ StatusCode TrigOpMonitor::bookHists()
   m_muHist = new TProfile("Pileup", "Pileup;Lumiblock;Interactions per BX", m_maxLB, 0, m_maxLB);
 
   // Register histograms
-  TH1* hist[] = {m_releaseHist, m_subdetHist, m_iovChangeHist, m_magFieldHist, m_lumiHist, m_muHist};
+  TH1* hist[] = {
+    m_releaseHist, m_mtConfigHist, m_workersHist, m_subdetHist,
+    m_iovChangeHist, m_magFieldHist, m_lumiHist, m_muHist
+  };
   for (TH1* h : hist) {
     if (h) ATH_CHECK(m_histSvc->regHist(m_histPath + h->GetName(), h));
   }
@@ -347,6 +360,38 @@ void TrigOpMonitor::fillReleaseDataHist()
   }
 }
 
+void TrigOpMonitor::fillProcThreadHist(const int workerID)
+{
+  m_workersHist->Fill(workerID, 1);
+
+  auto getIntProp = [this](std::string_view name, const std::string& prop) -> std::optional<int> {
+    IProperty* svc = serviceLocator()->service(name).as<IProperty>();
+    if (svc == nullptr) return std::nullopt;
+    try {
+      return std::stoi(svc->getProperty(prop).toString());
+    }
+    catch (...) {
+      return std::nullopt;
+    }
+  };
+
+  const std::optional<int> numThreads = getIntProp("AvalancheSchedulerSvc","ThreadPoolSize");
+  if (numThreads.has_value()) {
+    m_mtConfigHist->Fill("N threads", numThreads.value(), 1.0);
+  }
+  else {
+    ATH_MSG_WARNING("Could not retrieve the number of threads to fill the monitoring histogram");
+  }
+
+  const std::optional<int> numSlots = getIntProp("EventDataSvc","NSlots");
+  if (numSlots.has_value()) {
+    m_mtConfigHist->Fill("N slots", numSlots.value(), 1.0);
+  }
+  else {
+    ATH_MSG_WARNING("Could not retrieve the number of slots to fill the monitoring histogram");
+  }
+}
+
 void TrigOpMonitor::fillSubDetHist()
 {
   // Retrieve the enabled ROBs/SubDets list from DataFlowConfig which is a special object