diff --git a/Trigger/TrigCost/TrigCostAnalysis/python/CostMetadataUtil.py b/Trigger/TrigCost/TrigCostAnalysis/python/CostMetadataUtil.py
new file mode 100644
index 0000000000000000000000000000000000000000..b30ad08d4adc9de81a7438417174b2535dafb5e8
--- /dev/null
+++ b/Trigger/TrigCost/TrigCostAnalysis/python/CostMetadataUtil.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
+#
+
+'''
+@file CostMetadataUtil.py
+@brief Helper functions to create cost metadata json file based on input ntuple
+ and histogram under/overflows
+'''
+
+from AthenaCommon.Logging import logging
+log = logging.getLogger('CostAnalysisPostProcessing')
+
+
+def saveMetadata(inputFile, userDetails, processingWarnings=[]):
+ ''' @brief Save metadata from ntuple to json file
+ '''
+ import json
+
+ metatree = inputFile.Get("metadata")
+ if metatree is None:
+ return None
+
+ metatree.GetEntry(0)
+ metadata = []
+
+ metadata.append({'runNumber' : metatree.runNumber})
+ metadata.append({'AtlasProject' : str(metatree.AtlasProject)})
+ metadata.append({'AtlasVersion' : str(metatree.AtlasVersion)})
+
+ metadata.append({'ChainMonitor' : metatree.ChainMonitor})
+ metadata.append({'AlgorithmMonitor' : metatree.AlgorithmMonitor})
+ metadata.append({'AlgorithmClassMonitor' : metatree.AlgorithmClassMonitor})
+ metadata.append({'ROSMonitor' : metatree.ROSMonitor})
+ metadata.append({'GlobalsMonitor' : metatree.GlobalsMonitor})
+ metadata.append({'ThreadMonitor' : metatree.ThreadMonitor})
+
+ metadata.append({'AdditionalHashMap' : str(metatree.AdditionalHashMap)})
+ metadata.append({'DoEBWeighting' : metatree.DoEBWeighting})
+ metadata.append({'BaseEventWeight' : metatree.BaseEventWeight})
+
+ metadata.append({'HLTMenu' : json.loads(str(metatree.HLTMenu))})
+
+ metadata.append({'Details' : userDetails})
+
+ metadata.append({'Histogram under/overflows' : processingWarnings})
+
+ with open('metadata.json', 'w') as outMetaFile:
+ metafile = {}
+ metafile['text'] = 'metadata'
+ metafile['children'] = metadata
+ json.dump(obj=metafile, fp=outMetaFile, indent=2, sort_keys=True)
+
+
+def createOverflowSummary(warnings):
+ ''' @brief Create summery of under/overflows based on passed warnings
+ '''
+ histogramStats = {}
+ log.debug("Received %s warnings", len(warnings))
+ for entry in warnings:
+ histFullName = entry.split(" ")[-1]
+ histType = histFullName.split("_")[-2] + "_" + histFullName.split("_")[-1]
+ summary = entry.split(" ")[-1].split("HLT")[0] + "HLT"
+
+ if "LumiBlock" in summary:
+ # format LumiBlock_000XX_SummaryName...
+ summary = summary.split('_', 1)[1]
+ summary = summary.split('_', 1)[1]
+ elif "All" in summary:
+ # format All_SummaryName...
+ summary = summary.split('_', 1)[1]
+
+ entryName = summary + "_" + histType
+ if entryName in histogramStats:
+ histogramStats[entryName] += 1
+ else:
+ histogramStats[entryName] = 1
+
+ histogramStatsStr = []
+ for name, value in histogramStats.items():
+ histogramStatsStr.append("{0}: {1} histograms with over/underflows".format(name, value))
+
+ return {"Summary": histogramStatsStr}
+
+
+def ignoreUnderflow(histogramName):
+ ''' @brief Filter out the histograms to ignore in underflow check
+ '''
+
+ # Ignore time histograms for filters
+ if "FStep" in histogramName and "Time" in histogramName:
+ log.debug("Filter %s underflow will be ignored", histogramName)
+ return True
+
+ return False
diff --git a/Trigger/TrigCost/TrigCostAnalysis/python/TableConstructorBase.py b/Trigger/TrigCost/TrigCostAnalysis/python/TableConstructorBase.py
index f56ae8eec038445945928bc1299aa6c85c2612f4..e8d9c851a966f9954fa6353399ece6a14751a5b7 100644
--- a/Trigger/TrigCost/TrigCostAnalysis/python/TableConstructorBase.py
+++ b/Trigger/TrigCost/TrigCostAnalysis/python/TableConstructorBase.py
@@ -10,7 +10,7 @@
from math import fabs
from collections import OrderedDict
-
+from TrigCostAnalysis.CostMetadataUtil import ignoreUnderflow
from AthenaCommon.Logging import logging
log = logging.getLogger('TableConstructor')
@@ -136,7 +136,7 @@ class TableConstructorBase:
underflowThreshold = self.underflowThreshold * histIntegral
underflow = hist.GetBinContent(0)
- if underflow > underflowThreshold:
+ if underflow > underflowThreshold and not ignoreUnderflow(fullHistName):
log.warning("Histogram {0} contains underflow of {1}".format(fullHistName, underflow))
self.warningMsg.append("Underflow of {0} ({1} histogram integral) in {2}".format(underflow, histIntegral, fullHistName))
diff --git a/Trigger/TrigCost/TrigCostAnalysis/python/Util.py b/Trigger/TrigCost/TrigCostAnalysis/python/Util.py
index 538362e2858f89900454e8f3e5c6990b57a1dfd4..ca2f22332f37faeaa8bed5046287544257ff6c8c 100644
--- a/Trigger/TrigCost/TrigCostAnalysis/python/Util.py
+++ b/Trigger/TrigCost/TrigCostAnalysis/python/Util.py
@@ -10,48 +10,11 @@
import ROOT
from math import fabs
+from TrigCostAnalysis.CostMetadataUtil import createOverflowSummary
from AthenaCommon.Logging import logging
log = logging.getLogger('CostAnalysisPostProcessing')
-def saveMetadata(inputFile, userDetails, processingWarnings=[]):
- import json
-
- metatree = inputFile.Get("metadata")
- if metatree is None:
- return None
-
- metatree.GetEntry(0)
- metadata = []
-
- metadata.append({'runNumber' : metatree.runNumber})
- metadata.append({'AtlasProject' : str(metatree.AtlasProject)})
- metadata.append({'AtlasVersion' : str(metatree.AtlasVersion)})
-
- metadata.append({'ChainMonitor' : metatree.ChainMonitor})
- metadata.append({'AlgorithmMonitor' : metatree.AlgorithmMonitor})
- metadata.append({'AlgorithmClassMonitor' : metatree.AlgorithmClassMonitor})
- metadata.append({'ROSMonitor' : metatree.ROSMonitor})
- metadata.append({'GlobalsMonitor' : metatree.GlobalsMonitor})
- metadata.append({'ThreadMonitor' : metatree.ThreadMonitor})
-
- metadata.append({'AdditionalHashMap' : str(metatree.AdditionalHashMap)})
- metadata.append({'DoEBWeighting' : metatree.DoEBWeighting})
- metadata.append({'BaseEventWeight' : metatree.BaseEventWeight})
-
- metadata.append({'HLTMenu' : json.loads(str(metatree.HLTMenu))})
-
- metadata.append({'Details' : userDetails})
-
- metadata.append({'Histogram under/overflows' : processingWarnings})
-
- with open('metadata.json', 'w') as outMetaFile:
- metafile = {}
- metafile['text'] = 'metadata'
- metafile['children'] = metadata
- json.dump(obj=metafile, fp=outMetaFile, indent=2, sort_keys=True)
-
-
def exploreTree(inputFile, dumpSummary=False, underflowThreshold=0.1, overflowThreshold=0.1):
''' @brief Explore ROOT Tree to find tables with histograms to be saved in csv
@@ -117,35 +80,6 @@ def exploreTree(inputFile, dumpSummary=False, underflowThreshold=0.1, overflowTh
return processingWarnings + [summary]
-def createOverflowSummary(warnings):
- histogramStats = {}
- for entry in warnings:
- histFullName = entry.split(" ")[-1]
- histType = histFullName.split("_")[-2] + "_" + histFullName.split("_")[-1]
- summary = entry.split(" ")[-1].split("HLT")[0] + "HLT"
-
- if "LumiBlock" in summary:
- # format LumiBlock_000XX_SummaryName...
- summary = summary.split('_', 1)[1]
- summary = summary.split('_', 1)[1]
- elif "All" in summary:
- # format All_SummaryName...
- summary = summary.split('_', 1)[1]
-
- entryName = summary + "_" + histType
- if entryName in histogramStats:
- histogramStats[entryName] += 1
- else:
- histogramStats[entryName] = 1
-
- histogramStatsStr = []
- for name, value in histogramStats.items():
- histogramStatsStr.append("{0}: {1} histograms with over/underflows".format(name, value))
-
- return {"Summary": histogramStatsStr}
-
-
-
def getWalltime(inputFile, rootName):
''' @brief Extract walltime value from histogram
@@ -183,6 +117,7 @@ def getAlgorithmTotalTime(inputFile, rootName):
return totalTime * 1e-3
+
def convert(entry):
''' @brief Save entry number in scientific notation'''
if type(entry) is float or type(entry) is int:
diff --git a/Trigger/TrigCost/TrigCostAnalysis/share/CostAnalysisPostProcessing.py b/Trigger/TrigCost/TrigCostAnalysis/share/CostAnalysisPostProcessing.py
index 9585448fa5757e69d614cbc307ddd0740f0c5c17..b7be6496d015d34c8ce832e104023087da9d6be6 100755
--- a/Trigger/TrigCost/TrigCostAnalysis/share/CostAnalysisPostProcessing.py
+++ b/Trigger/TrigCost/TrigCostAnalysis/share/CostAnalysisPostProcessing.py
@@ -10,7 +10,8 @@
'''
import ROOT
-from TrigCostAnalysis.Util import exploreTree, saveMetadata
+from TrigCostAnalysis.Util import exploreTree
+from TrigCostAnalysis.CostMetadataUtil import saveMetadata
from AthenaCommon.Logging import logging
log = logging.getLogger('CostAnalysisPostProcessing')
@@ -19,8 +20,8 @@ def main():
parser = ArgumentParser()
parser.add_argument('--file', default='TrigCostRoot_Results.root', help='Input ROOT file to generate output from')
parser.add_argument('--userDetails', help='User supplied metadata string giving any extra details about this run.')
- parser.add_argument('--underflowThreshold', default=0.5, help='Proportion of underflow entries to all entries (weighted) to save warning in metadata tree.')
- parser.add_argument('--overflowThreshold', default=0.1, help='Proportion of overflow entries to all entries (weighted) to save warning in metadata tree.')
+ parser.add_argument('--underflowThreshold', default=0.5, help='Threshold of underflow percent value to save warning in metadata tree.')
+ parser.add_argument('--overflowThreshold', default=0.1, help='Threshold of underflow percent value to save warning in metadata tree.')
parser.add_argument('--dumpAlgorithmSummary', action='store_true', help='Print algorithm\'s mean time of execution to the log file')
args = parser.parse_args()
@@ -32,6 +33,10 @@ def main():
else:
log.error("File %s not found", args.file)
+ # to speed up closing the ROOT file
+ ROOT.gROOT.GetListOfFiles().Remove(inputFile)
+ inputFile.Close()
+
if __name__== "__main__":
- main()
\ No newline at end of file
+ main()