From f38bda70b0a1955e4dada14ddc732e7afb66aeb7 Mon Sep 17 00:00:00 2001
From: Aleksandra Poreba <aleksandra.poreba@cern.ch>
Date: Wed, 20 Jul 2022 12:31:08 +0200
Subject: [PATCH 1/2] Save additional metadata along online rates

---
 .../share/RatesAnalysisOnlineProcessing.py    | 89 +++++++++++++++----
 1 file changed, 70 insertions(+), 19 deletions(-)

diff --git a/Trigger/TrigCost/RatesAnalysis/share/RatesAnalysisOnlineProcessing.py b/Trigger/TrigCost/RatesAnalysis/share/RatesAnalysisOnlineProcessing.py
index fa299fd741f4..c58e9ec2b75a 100755
--- a/Trigger/TrigCost/RatesAnalysis/share/RatesAnalysisOnlineProcessing.py
+++ b/Trigger/TrigCost/RatesAnalysis/share/RatesAnalysisOnlineProcessing.py
@@ -115,16 +115,15 @@ def getKeyForLimits(psk, lbStart, lbEnd):
     # COOL format of prescale entry is (key, lbStart, lbEnd)
     foundKeys = []
     for prescaleEntry in pskList:
-        if lbStart >= prescaleEntry[1] and lbEnd <= prescaleEntry[2]:
+        if prescaleEntry[1] <= lbEnd and prescaleEntry[2] >= lbStart:
             log.info("Found prescale key in the range: {0}".format(prescaleEntry))
             foundKeys.append(prescaleEntry[0])
 
     if not foundKeys:
-        log.error("Cannot find one prescale for lumiblocks {0} to {1}. Available values: {2}".format(lbStart, lbEnd, pskList))
+        log.warning("Cannot find one prescale for lumiblocks {0} to {1}. Available values: {2}".format(lbStart, lbEnd, pskList))
         return -1
     elif len(foundKeys) > 1:
-        log.error("Found more than one prescale key! {0}".format(pskList))
-        return -1
+        log.warning("Found more than one prescale key! {0} Will use the first one {1}".format(foundKeys, foundKeys[0]))
 
     return foundKeys[0]
 
@@ -136,15 +135,52 @@ def toCSV(fileName, dirName, data):
     import csv, os
 
     os.makedirs(dirName, exist_ok=True)
-    with open(dirName + '/' + fileName, mode='w') as outputCSV_file:
-        rates_csv_writer = csv.writer(outputCSV_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
+    with open(dirName + '/' + fileName, mode='w') as outputFile:
+        ratesWriter = csv.writer(outputFile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
 
-        rates_csv_writer.writerow(['Name','Group','Rate [Hz]','Rate Err [Hz]', 'Prescale'])
-        rates_csv_writer.writerow(['Trigger name','The group this chain belongs to','Online rate','Error on rate','The prescale of this chain. Only displayed for simple combinations'])
+        ratesWriter.writerow(['Name','Group','Rate [Hz]','Rate Err [Hz]', 'Prescale'])
+        ratesWriter.writerow(['Trigger name','The group this chain belongs to','Online rate','Error on rate','The prescale of this chain. Only displayed for simple combinations'])
 
         for trig in data:
-            rates_csv_writer.writerow(trig)
-    
+            ratesWriter.writerow(trig)
+
+
+def saveMetadata(dirName, data):
+    ''' 
+    Save metadata dictionary
+    '''
+
+    mdDict = {}
+    mdDict['text'] = 'metadata'
+    mdDict['children'] = data
+
+    outFilename = dirName + '/' + "metadata.json"
+    with open(outFilename, mode='w') as outputFile:
+        import json
+        json.dump(obj=mdDict, fp=outputFile, indent=2, sort_keys=True)
+
+
+def readAvgLumi(runNumber, lbStart, lbEnd):
+    ''' 
+    Read Inst Lumi and calculate the average for given run number and lumiblock ranges
+    '''
+    from PyCool import cool
+    from DQUtils.sugar import RunLumi
+    from TrigConfStorage.TriggerCoolUtil import TriggerCoolUtil
+    db = TriggerCoolUtil.GetConnection('CONDBR2')
+    folder = db.getFolder("/TRIGGER/LUMI/OnlPrefLumi")
+    folderIterator = folder.browseObjects(RunLumi(runNumber, lbStart), RunLumi(runNumber, lbEnd), cool.ChannelSelection())    
+
+    avg = 0
+    counter = 0
+    while folderIterator.goToNext():
+        payload=folderIterator.currentRef().payload()
+        avg += payload["LBAvInstLumi"]
+        counter +=1
+
+    log.info("Avg inst lumi {0} for {1} events".format(avg, counter))
+
+    return avg/counter * 1e30
 
 def main():
     from argparse import ArgumentParser
@@ -236,7 +272,7 @@ def main():
         rateUn = rate * l1Prescales[itemName]
 
         L1Items.append([itemName, "-", round(rate, 3), "0", "L1:{0}".format(l1Prescales[itemName])])
-        L1Items.append([itemName, "-", round(rateUn, 3), "0", "L1:{0}".format(l1Prescales[itemName])])
+        L1ItemsUnps.append([itemName, "-", round(rateUn, 3), "0", "L1:{0}".format(l1Prescales[itemName])])
 
     groups = []
     groupRates = groupRates | streamRates | recordingRates
@@ -256,17 +292,32 @@ def main():
     HLTTable = getTableName("HLT")
     GroupTable = getTableName("Group")
 
-
-    prescaledDirName = "costMonitoring_OnlineTRPRates-onlinePS-LB{0}-{1}_{2}/csv/".format(args.lbStart, args.lbEnd, args.runNumber)
-    unprescaledDirName = "costMonitoring_OnlineTRPRates-noPS-LB{0}-{1}_{2}/csv/".format(args.lbStart, args.lbEnd, args.runNumber)
+    metadata = [
+        {'PredictionLumi' : readAvgLumi(args.runNumber, args.lbStart, args.lbEnd)},
+        {'RunNumber' : args.runNumber},
+        {"First lumiblock" : args.lbStart},
+        {"Last lumiblock" : args.lbEnd},
+        {'SMK' :  configMetadata[1]["SMK"]},
+        {'DB' : configMetadata[2]["DB"]},
+        {'LVL1PSK' :  getKeyForLimits(configMetadata[5]["LVL1PSK"], args.lbStart, args.lbEnd)},
+        {'HLTPSK' : getKeyForLimits(configMetadata[4]["HLTPSK"], args.lbStart, args.lbEnd)}
+    ]
+
+    prescaledDirName = "costMonitoring_OnlineTRPRates-onlinePS-LB{0}-{1}_{2}/".format(args.lbStart, args.lbEnd, args.runNumber)
+    unprescaledDirName = "costMonitoring_OnlineTRPRates-noPS-LB{0}-{1}_{2}/".format(args.lbStart, args.lbEnd, args.runNumber)
     log.info("Exporting " + HLTTable)
-    toCSV(HLTTable, prescaledDirName, HLTChains)
-    toCSV(HLTTable, unprescaledDirName, HLTChainsUnps)
+    toCSV(HLTTable, prescaledDirName + "csv/", HLTChains)
+    toCSV(HLTTable, unprescaledDirName + "csv/", HLTChainsUnps)
     log.info("Exporting " + L1Table)
-    toCSV(L1Table, prescaledDirName, L1Items)
-    toCSV(L1Table, unprescaledDirName, L1ItemsUnps)
+    toCSV(L1Table, prescaledDirName + "csv/", L1Items)
+    toCSV(L1Table, unprescaledDirName + "csv/", L1ItemsUnps)
     log.info("Exporting " + GroupTable)
-    toCSV(GroupTable, prescaledDirName, groups)
+    toCSV(GroupTable, prescaledDirName + "csv/", groups)
+
+    prescaledMd = [*metadata, {"Details" : "Averaged rates with online prescales from online monitoring"}]
+    unprescaledMd = [*metadata, {"Details" : "Averaged rates with prescales removed from online monitoring"}]
+    saveMetadata(prescaledDirName, prescaledMd)
+    saveMetadata(unprescaledDirName, unprescaledMd)
 
 
 if __name__== "__main__":
-- 
GitLab


From 282f766bb551ace3dbad75ce2677c828ba915d68 Mon Sep 17 00:00:00 2001
From: Aleksandra Poreba <aleksandra.poreba@cern.ch>
Date: Mon, 25 Jul 2022 08:08:53 +0200
Subject: [PATCH 2/2] Review log level

---
 .../RatesAnalysis/share/RatesAnalysisOnlineProcessing.py    | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Trigger/TrigCost/RatesAnalysis/share/RatesAnalysisOnlineProcessing.py b/Trigger/TrigCost/RatesAnalysis/share/RatesAnalysisOnlineProcessing.py
index c58e9ec2b75a..6748ee524818 100755
--- a/Trigger/TrigCost/RatesAnalysis/share/RatesAnalysisOnlineProcessing.py
+++ b/Trigger/TrigCost/RatesAnalysis/share/RatesAnalysisOnlineProcessing.py
@@ -178,7 +178,11 @@ def readAvgLumi(runNumber, lbStart, lbEnd):
         avg += payload["LBAvInstLumi"]
         counter +=1
 
-    log.info("Avg inst lumi {0} for {1} events".format(avg, counter))
+    if counter == 0:
+        log.error("No entries in COOL database, in /TRIGGER/LUMI/OnlPrefLumi folder were found for run {0}, lumiblocks {1}-{2}".format(runNumber, lbStart, lbEnd))
+        return 0
+
+    log.debug("Avg inst lumi {0} for {1} events".format((avg/counter), counter))
 
     return avg/counter * 1e30
 
-- 
GitLab