Merge branch 'master-tile-mon-onl-postproc-run3' into 'master'

TileMonitoring: Add post processing for Tile online monitoring

See merge request !46622

DataQuality/DQOnlinePostprocessing/python/atlas_oh.py

@@ -46,7 +46,7 @@ class OHInputModule(InputModule):
             raise ValueError("Must specify 'source' as an "
                              "option to OHInputModule")
         self.source = options['source']
-        self.partition, self.server, self.provider = self.source.split(':')
+        self.partition, self.server, self.provider = self.source.split(';')
         self.prefix = options.get('prefix', '')
         if not IPCPartition(self.partition).isValid():
             raise ValueError(f'Input partition {self.partition} does not exist')
@@ -134,7 +134,7 @@ class OHOutputModule(OutputModule):
             raise ValueError("Must specify 'target' as an option "
                              "to OHInputModule")
         self.target = options['target']
-        self.partition, self.server, self.provider = self.target.split(':')
+        self.partition, self.server, self.provider = self.target.split(';')
         self.partition = ispy.IPCPartition(self.partition)
         if not self.partition.isValid():
             raise ValueError(f'Output partition {self.partition.name()} is not valid')

TileCalorimeter/TileMonitoring/CMakeLists.txt

@@ -32,6 +32,7 @@ atlas_add_component( TileMonitoring
 atlas_install_headers( TileMonitoring )
 atlas_install_python_modules( python/*.py POST_BUILD_CMD ${ATLAS_FLAKE8} )
 atlas_install_joboptions( share/*.py )
+atlas_install_data( data/*.yaml )
 
 # Tests:
 atlas_add_test( TileCellMonitorAlgorithm_test

TileCalorimeter/TileMonitoring/data/TileDigiNoisePostProc.yaml

+#
+#  Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration
+#
+# Post-processing of histograms from Tile digi noise monitoring
+## Separate blocks with ---
+---
+Input: [ 'Tile/DigiNoise/TileDigiNoisePed_(?P<partition>[L|E]B[C|A])_(?P<gain>[H|L]G)' ]
+Output: [ 'Tile/DigiNoise/TileDigiNoiseLFN_{partition}_{gain}' ]
+Function: TileMonitoring.TilePostProcessing.getProfile2D_RMS
+Parameters: { title: 'Low frequency noise (LFN)', name: 'TileDigiNoiseLFN'}
+Description: Produce Tile 2D with low frequency noise (LFN) per partition
+

TileCalorimeter/TileMonitoring/data/TileRawChanNoisePostProc.yaml

+#
+#  Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration
+#
+# Post-processing of histograms from Tile Raw channel noise monitoring
+## Separate blocks with ---
+Input: [ 'Tile/RawChannelNoise/LBA/(?P<module>LBA[0-6]\d{1})/TileRawChannelNoise_(?P=module)_[ABCDE]\d[0-5\*]?_ch_(?P<channel>\d{1,2})_HG' ]
+Output: [ 'Tile/RawChannelNoise/Summary/TileRawChannelNoise_RMS_LBA', 'Tile/RawChannelNoise/Summary/TileRawChannelNoise_Sigma_LBA',
+          'Tile/RawChannelNoise/Summary/TileRawChannelNoise_Chi2_LBA', 'Tile/RawChannelNoise/Summary/TileRawChannelNoise_Probability_LBA',
+          'Tile/RawChannelNoise/Summary/TileRawChannelNoise_RmsOverSigma_LBA']
+Function: TileMonitoring.TilePostProcessing.getTilePartitionRawChannelNoise
+Description: Produces 2D histograms with Tile raw channel noise for LBA partition
+---
+Input: [ 'Tile/RawChannelNoise/LBC/(?P<module>LBC[0-6]\d{1})/TileRawChannelNoise_(?P=module)_[ABCDE]\d[0-5\*]?_ch_(?P<channel>\d{1,2})_HG' ]
+Output: [ 'Tile/RawChannelNoise/Summary/TileRawChannelNoise_RMS_LBC', 'Tile/RawChannelNoise/Summary/TileRawChannelNoise_Sigma_LBC',
+          'Tile/RawChannelNoise/Summary/TileRawChannelNoise_Chi2_LBC', 'Tile/RawChannelNoise/Summary/TileRawChannelNoise_Probability_LBC',
+          'Tile/RawChannelNoise/Summary/TileRawChannelNoise_RmsOverSigma_LBC' ]
+Function: TileMonitoring.TilePostProcessing.getTilePartitionRawChannelNoise
+Description: Produces 2D histograms with Tile raw channel noise for LBC partition
+---
+Input: [ 'Tile/RawChannelNoise/EBA/(?P<module>EBA[0-6]\d{1})/TileRawChannelNoise_(?P=module)_[ABCDE]\d[0-5\*]?_ch_(?P<channel>\d{1,2})_HG' ]
+Output: [ 'Tile/RawChannelNoise/Summary/TileRawChannelNoise_RMS_EBA', 'Tile/RawChannelNoise/Summary/TileRawChannelNoise_Sigma_EBA',
+          'Tile/RawChannelNoise/Summary/TileRawChannelNoise_Chi2_EBA', 'Tile/RawChannelNoise/Summary/TileRawChannelNoise_Probability_EBA',
+          'Tile/RawChannelNoise/Summary/TileRawChannelNoise_RmsOverSigma_EBA' ]
+Function: TileMonitoring.TilePostProcessing.getTilePartitionRawChannelNoise
+Description: Produces 2D histograms with Tile raw channel noise for EBA partition
+---
+Input: [ 'Tile/RawChannelNoise/EBC/(?P<module>EBC[0-6]\d{1})/TileRawChannelNoise_(?P=module)_[ABCDE]\d[0-5\*]?_ch_(?P<channel>\d{1,2})_HG' ]
+Output: [ 'Tile/RawChannelNoise/Summary/TileRawChannelNoise_RMS_EBC', 'Tile/RawChannelNoise/Summary/TileRawChannelNoise_Sigma_EBC',
+          'Tile/RawChannelNoise/Summary/TileRawChannelNoise_Chi2_EBC', 'Tile/RawChannelNoise/Summary/TileRawChannelNoise_Probability_EBC',
+          'Tile/RawChannelNoise/Summary/TileRawChannelNoise_RmsOverSigma_EBC' ]
+Function: TileMonitoring.TilePostProcessing.getTilePartitionRawChannelNoise
+Description: Produces 2D histograms with Tile raw channel noise for EBC partition
+

TileCalorimeter/TileMonitoring/data/TileTMDBPostProc.yaml

+#
+#  Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration
+#
+# Post-processing of histograms from Tile TMDB monitoring
+## Separate blocks with ---
+---
+Input: [ 'Tile/TMDBDigits/TMDB_DigitsPedestal_(?P<partition>[L|E]B[C|A])' ]
+Output: [ 'Tile/TMDBDigits/TMDB_DigitsLFN_{partition}' ]
+Function: TileMonitoring.TilePostProcessing.getProfile2D_RMS
+Parameters: { title: 'LFN in TMDB', name: 'TMDB_DigitsLFN' }
+Description: Produce Tile TMDB 2D plots with LFN per partition
+---
+Input: [ 'Tile/TMDB/TMDB_Energy_(?P<partition>[L|E]B[C|A])' ]
+Output: [ 'Tile/TMDB/TMDB_Noise_{partition}' ]
+Function: TileMonitoring.TilePostProcessing.getProfile2D_RMS
+Parameters: { title: 'Noise [MeV] in TMDB', name: 'TMDB_Noise'}
+Description: Produce Tile TMDB 2D plots with noise per partition
+

TileCalorimeter/TileMonitoring/python/TilePostProcessing.py

+#!/usr/bin/env python
 #
 #  Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration
 #
@@ -7,6 +8,9 @@
 @brief Functions for post-processing of histograms from TileMonitoring for the Run III
 '''
 
+import os, sys, signal, subprocess
+from AthenaCommon.Utils.unixtools import find_datafile
+
 def getProfile2D_RMS(inputs, title, name):
     """ This function produces 2D histogram with RMS from Profile2D per Tile partition """
 
@@ -28,3 +32,140 @@ def getProfile2D_RMS(inputs, title, name):
     inputProfile.GetYaxis().Copy(outputHistogram.GetYaxis())
 
     return [outputHistogram]
+
+
+def getTilePartitionRawChannelNoise(inputs):
+    """ This function produces 2D histograms with Tile raw channel noise per partition """
+
+    import ROOT
+    # Force batch mode
+    ROOT.gROOT.SetBatch(True)
+
+    from TileMonitoring.TileMonitoringCfgHelper import getLabels
+    from TileCalibBlobObjs.Classes import TileCalibUtils as Tile
+
+    def _getTilePartitionHistogram2D(name, title, partition, run):
+        fullName = 'TileRawChannelNoise_{}_{}'.format(name, partition)
+        fullTitle = 'Run {} Partition {}: {}'.format(run, partition, title)
+        histogram = ROOT.TH2F(fullName, fullTitle,
+                              Tile.MAX_DRAWER, 0.5, Tile.MAX_DRAWER + 0.5,
+                              Tile.MAX_CHAN, -0.5, Tile.MAX_CHAN - 0.5)
+        moduleLables = getLabels(('modules'), partition)
+        channelLables = getLabels(('channels'), partition)
+        for axis,labels in ((histogram.GetXaxis(), moduleLables), (histogram.GetYaxis(), channelLables)):
+            for bin in range(0, len(labels)):
+                axis.SetBinLabel(bin + 1, labels[bin])
+        return histogram
+
+
+    partition = inputs[0][0]['module'][:3]
+    run = str(inputs[0][1][0].GetTitle()).split(' ')[1]
+
+    rmsHistogram = _getTilePartitionHistogram2D('RMS', 'RMS of gaussians', partition, run)
+    sigmaHistogram = _getTilePartitionHistogram2D('Sigma', 'Sigma of gaussians', partition, run)
+    chi2Histogram = _getTilePartitionHistogram2D('Chi2', 'Chi2 of gaussians', partition, run)
+    probabilityHistogram = _getTilePartitionHistogram2D('Prob', 'Probability of gaussians', partition, run)
+    rmsOverSigmaHistogram = _getTilePartitionHistogram2D('RmsOverSigma', 'RMS/Sigma of gaussians', partition, run)
+
+    fitfunction = ROOT.TF1("total", "gaus(0)")
+    fitfunction.SetLineColor(2)
+
+    for input in inputs:
+
+        module = int(input[0]['module'][3:])
+        channel = int(input[0]['channel'])
+
+        plot = input[1][0]
+
+        nEntries = plot.GetEntries()
+
+        if nEntries > 100:
+
+            rms = plot.GetRMS()
+            rmsHistogram.Fill(module, channel, rms)
+
+            fitfunction.SetParameters(0.1 * nEntries, 0., 0.7 * rms)
+
+            binWidth = plot.GetBinWidth(0)
+            lowLimit = binWidth * 0.5
+            highLimit = max(rms * 1.05, binWidth * 2.0)
+
+            fitfunction.SetParLimits(0, 0., nEntries)
+            fitfunction.FixParameter(1, 0.)
+            fitfunction.SetParLimits(2, lowLimit, highLimit)
+
+            plot.Fit(fitfunction, "BQ")
+
+            sigma = fitfunction.GetParameter(2)
+            chi2 = fitfunction.GetChisquare()
+            prob = fitfunction.GetProb()
+
+            sigmaHistogram.Fill(module, channel, sigma)
+            chi2Histogram.Fill(module, channel, chi2)
+            probabilityHistogram.Fill(module, channel, prob)
+            rmsOverSigmaHistogram.Fill(module, channel, rms/sigma)
+
+    return [rmsHistogram, sigmaHistogram, chi2Histogram, probabilityHistogram, rmsOverSigmaHistogram]
+
+
+if __name__== '__main__':
+
+    import argparse
+    parser= argparse.ArgumentParser()
+
+    parser.add_argument('--stateless', action="store_true", help='Run Online Tile Post Processing in partition')
+    parser.add_argument('--tmdb', action="store_true", help='Run Tile TMDB Post Processing')
+    parser.add_argument('--digi-noise', action="store_true", dest='digiNoise', help='Run Tile Digi Noise Post Processing')
+    parser.add_argument('--raw-chan-noise', action="store_true", dest='rawChanNoise', help='Run Tile Digi Noise Post Processing')
+    parser.add_argument('--interval', type=int, default=60, help='Interval (in seconds) to run Tile Monitoring Post Processing')
+    parser.add_argument('--noise-interval', type=int, default=300, dest='noiseInterval',
+                        help='Interval (in seconds) to run Tile Noise Monitoring Post Processing')
+
+    args = parser.parse_args()
+
+    if args.stateless:
+
+        server = 'Histogramming'
+        partition = os.getenv("TDAQ_PARTITION", "ATLAS")
+
+        dataPath = find_datafile('TileMonitoring')
+
+        def sigtermHandler(signal, frame):
+            print('Terminating postporcessing ...')
+            sys.exit(0)
+
+        signal.signal(signal.SIGTERM, sigtermHandler)
+
+        if args.tmdb:
+
+            inputPath = f'{partition};{server};TileMIG:TileGatherer'
+            outputPath = f'{partition};{server};TilePT-stateless-PP'
+
+            physConfigurations = []
+            if args.tmdb:
+                physConfigurations += [os.path.join(dataPath, 'TileTMDBPostProc.yaml')]
+
+            physPostProcess = (['histgrinder', inputPath, outputPath,
+                                 '--inmodule', 'DQOnlinePostprocessing.atlas_oh.OHInputModule',
+                                 '--outmodule', 'DQOnlinePostprocessing.atlas_oh.OHOutputModule',
+                                 '-c'] + physConfigurations)
+
+            subprocess.run(physPostProcess)
+
+        elif any([args.digiNoise, args.rawChanNoise]):
+
+            inputPath = f'{partition};{server};TilePT-stateless-noise-01'
+            outputPath = f'{partition};{server};TilePT-stateless-noise-PP'
+
+            noiseConfigurations = []
+            if args.digiNoise:
+                noiseConfigurations += [os.path.join(dataPath, 'TileDigiNoisePostProc.yaml')]
+            if args.rawChanNoise:
+                noiseConfigurations += [os.path.join(dataPath, 'TileRawChanNoisePostProc.yaml')]
+
+            noisePostProcess = (['histgrinder', inputPath, outputPath,
+                                 '--inmodule', 'DQOnlinePostprocessing.atlas_oh.OHInputModule',
+                                 '--outmodule', 'DQOnlinePostprocessing.atlas_oh.OHOutputModule',
+                                 '-c'] + noiseConfigurations)
+
+            subprocess.run(noisePostProcess)