From d870a9a7f5e8928fcc5db8553a59e4a9e9575b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jim=C3=A9nez=20Pe=C3=B1a?= <javier.jimenez.pena@cern.ch> Date: Fri, 21 Mar 2025 17:05:49 +0100 Subject: [PATCH 1/2] Adding additional funcionalities to BinThreshold DQ algorithm: List of Vetos and X and Y ranges --- .../dqm_algorithms/BinThreshold.h | 3 +- .../dqm_algorithms/src/BinThreshold.cxx | 88 +++++++++++++++++++ .../python/L1TopoOnlineMonitoringConfig.py | 26 ++++-- 3 files changed, 109 insertions(+), 8 deletions(-) diff --git a/DataQuality/dqm_algorithms/dqm_algorithms/BinThreshold.h b/DataQuality/dqm_algorithms/dqm_algorithms/BinThreshold.h index 789d2f713ec1..1b3a752561fb 100644 --- a/DataQuality/dqm_algorithms/dqm_algorithms/BinThreshold.h +++ b/DataQuality/dqm_algorithms/dqm_algorithms/BinThreshold.h @@ -25,7 +25,8 @@ namespace dqm_algorithms bool CompareBinThreshold( const std::string & objname, double bincontent, double threshold ); using dqm_core::Algorithm::printDescription; void printDescription(std::ostream& out); - + std::vector<int> parseNumbers(const std::string& input); + static std::vector<std::vector<int>> parseNumberPairs(const std::string& input); private: std::string m_name; }; diff --git a/DataQuality/dqm_algorithms/src/BinThreshold.cxx b/DataQuality/dqm_algorithms/src/BinThreshold.cxx index 11422f7f111f..6369dc1ce277 100644 --- a/DataQuality/dqm_algorithms/src/BinThreshold.cxx +++ b/DataQuality/dqm_algorithms/src/BinThreshold.cxx @@ -66,6 +66,32 @@ dqm_algorithms::BinThreshold::execute( const std::string & name, const double minstat = dqm_algorithms::tools::GetFirstFromMap( "MinStat", config.getParameters(), -1); const bool publish = (bool) dqm_algorithms::tools::GetFirstFromMap( "PublishBins", config.getParameters(), 0); const int maxpublish = (int) dqm_algorithms::tools::GetFirstFromMap( "MaxPublish", config.getParameters(), 20); + const int xmax = (int) dqm_algorithms::tools::GetFirstFromMap( "xMax", config.getParameters(), -1); + const int xmin = (int) dqm_algorithms::tools::GetFirstFromMap( "xMin", config.getParameters(), -1); + const int ymax = (int) dqm_algorithms::tools::GetFirstFromMap( "yMax", config.getParameters(), -1); + const int ymin = (int) dqm_algorithms::tools::GetFirstFromMap( "yMin", config.getParameters(), -1); + + //Strings with lists of vetoed rows, columns and bins + std::string listHotColumns = dqm_algorithms::tools::GetFirstFromMap("ListHotColumns", config.getGenericParameters(), "-1"); //The format is a list of int split by comas. Ex: "2,5,87" + std::string listHotRows = dqm_algorithms::tools::GetFirstFromMap("ListHotRows", config.getGenericParameters(), "-1"); //The format is a list of int split by comas. Ex: "2,5,87" + std::string listHotBins = dqm_algorithms::tools::GetFirstFromMap("ListHotBins", config.getGenericParameters(), "-1"); //The format is a list of int pairs, separated with : and split by comas. Ex: "2:3,5:1,8:7" + + //Removing the Quotes which are needed to properly transmit the strings + std::string listHotRowsNoQuotes; + for (char c : listHotRows) if (c != '"') listHotRowsNoQuotes += c; + std::string listHotBinsNoQuotes; + for (char c : listHotBins) if (c != '"') listHotBinsNoQuotes += c; + std::string listHotColumnsNoQuotes; + for (char c : listHotColumns) if (c != '"') listHotColumnsNoQuotes += c; + + bool vetoHotRows = false; if (listHotRows != "-1") vetoHotRows = true; + bool vetoHotBins = false; if (listHotBins != "-1") vetoHotBins = true; + bool vetoHotColumns = false; if (listHotColumns != "-1") vetoHotColumns = true; + + //Converting the string into vectors + std::vector<int> hotColumns = parseNumbers(listHotColumnsNoQuotes); + std::vector<int> hotRows = parseNumbers(listHotRowsNoQuotes); + auto hotBins = parseNumberPairs(listHotBinsNoQuotes); if (histogram->GetEntries() < minstat ) { dqm_core::Result *result = new dqm_core::Result(dqm_core::Result::Undefined); @@ -123,8 +149,40 @@ dqm_algorithms::BinThreshold::execute( const std::string & name, result->tags_["Effective_BinThreshold"] = bin_threshold; } + //bools for skiping threshold comparison of vetoed rows, columns and bins + bool skipColumn = false; + bool skipRow = false; + bool skipBin = false; + for ( int i = range[0]; i <= range[1]; ++i ) { + skipColumn = false; + for (int column : hotColumns){ + if (vetoHotColumns && ( column == i || skipColumn == true)) skipColumn = true; + } + //Skip column if it is in list of vetoed columns + if (skipColumn) continue; + //Skip bin threshold comparison if xmax!=-1 and i>xmax + if (xmax!=-1 && i>xmax) continue; + //Skip bin threshold comparison if xmin!=-1 and i<xmin + if (xmin!=-1 && i<xmin) continue; + for ( int j = range[2]; j <= range[3]; ++j ) { + skipRow = false; skipBin = false; + for (int row : hotRows){ + if (vetoHotRows && ( row == j || skipRow == true)) skipRow = true; + } + for (const auto& pair : hotBins){ + if (vetoHotBins && ( (pair[0] == i && pair[1] == j) || skipBin == true)) skipBin = true; + } + //Skip Bin if it is in list of vetoed bins + if (skipBin) continue; + //Skip Row if it is in list of vetoed rows + if (skipRow) continue; + //Skip bin threshold comparison if ymax!=-1 and j>ymax + if (ymax!=-1 && j>ymax) continue; + //Skip bin threshold comparison if ymin!=-1 and j<ymin + if (ymin!=-1 && j<ymin) continue; + double content= histogram -> GetBinContent(i,j); if ( CompareBinThreshold(m_name, content, bin_threshold )) { ++count; @@ -188,6 +246,33 @@ dqm_algorithms::BinThreshold::CompareBinThreshold(const std::string & type, doub return 0; } +std::vector<int> dqm_algorithms::BinThreshold::parseNumbers(const std::string& input) { + std::vector<int> numbers; + std::stringstream ss(input); + std::string token; + + while (std::getline(ss, token, ',')) { + numbers.push_back(std::stoi(token)); + } + + return numbers; +} + +std::vector<std::vector<int>> dqm_algorithms::BinThreshold::parseNumberPairs(const std::string& input) { + std::vector<std::vector<int>> numbers; + std::stringstream ss(input); + std::string pairToken; + + while (std::getline(ss, pairToken, ',')) { + std::stringstream pairStream(pairToken); + std::string first, second; + + if (std::getline(pairStream, first, ':') && std::getline(pairStream, second, ':')) { + numbers.push_back({std::stoi(first), std::stoi(second)}); + } + } + return numbers; +} void dqm_algorithms::BinThreshold::printDescription(std::ostream& out) @@ -206,6 +291,9 @@ dqm_algorithms::BinThreshold::printDescription(std::ostream& out) out<<"Optional Parameter: xmax: maximum x range"<<std::endl; out<<"Optional Parameter: ymin: minimum y range"<<std::endl; out<<"Optional Parameter: ymax: maximum y range\n"<<std::endl; + out<<"Optional Parameter: listHotBins: list of known hot bins (x,y) in a string, separated by comas and semicolons. Ex: 1:3,6:7 \n"<<std::endl; + out<<"Optional Parameter: listHotRows: list of known hot rows in a string, separated by comas. Ex: 1,3,6,7 \n"<<std::endl; + out<<"Optional Parameter: listHotColumns: list of known hot columnss in a string, separated by comas. Ex: 1,3,6,7 \n"<<std::endl; } diff --git a/Trigger/TrigT1/L1Topo/L1TopoOnlineMonitoring/python/L1TopoOnlineMonitoringConfig.py b/Trigger/TrigT1/L1Topo/L1TopoOnlineMonitoring/python/L1TopoOnlineMonitoringConfig.py index b7525e123a01..e1f14a78ea5f 100644 --- a/Trigger/TrigT1/L1Topo/L1TopoOnlineMonitoring/python/L1TopoOnlineMonitoringConfig.py +++ b/Trigger/TrigT1/L1Topo/L1TopoOnlineMonitoring/python/L1TopoOnlineMonitoringConfig.py @@ -36,12 +36,17 @@ def getL1TopoLabels(flags,connectors = {0: 'LegacyTopo0', 1: 'LegacyTopo1'}, bma else: for topo_trigline in topo_triglines_dict: topo_trigline_name = topo_trigline['name'] + topo_trigline_name= topo_trigline_name.replace("TOPO_","") bit_id = topo_trigline['startbit'] fpga_id = topo_trigline['fpga'] clock_id = topo_trigline['clock'] topo_trigline_index = 64*connector_id + 32*fpga_id + 2*bit_id + clock_id topo_trigline_labels[topo_trigline_index] = topo_trigline_name + for i in range(len(topo_trigline_labels)): + if ( topo_trigline_labels[i] == ""): + topo_trigline_labels[i] = "-- Unassigned Item --" + return topo_trigline_labels def getMultiplicityLabels(flags,topoModule): @@ -53,6 +58,8 @@ def getMultiplicityLabels(flags,topoModule): topo_trigline_name = topo_trigline['name'] bit_id = topo_trigline['startbit'] topo_trigline_labels[bit_id] = topo_trigline_name + if ( topo_trigline_labels[bit_id].find("SPARE") >= 0): + topo_trigline_labels[bit_id] = "-- Unassigned Item --" return topo_trigline_labels @@ -69,7 +76,7 @@ def getL1TopoPhase1OnlineMonitor(flags, name='L1TopoOnlineMonitor', doSimMon=Tru doComp = doComp, doMultComp = doMultComp, forceCTPasHdw=forceCtp, - MultiplicityVetoList=["ZeroBiasC","ZeroBiasB"], + MultiplicityVetoList=["ZeroBiasA","ZeroBiasB"], AlgorithmVetoList =["jXE40delay"]) if logLevel : alg.OutputLevel=logLevel alg.MonTool = GenericMonitoringTool(flags, 'MonTool') @@ -87,14 +94,19 @@ def getL1TopoPhase1DQMonitor(flags, name='L1TopoDQMonitor', doSimMon=True, doHwM doHwMonCTP = doHwMonCtp, doComp = doComp, doMultComp = doMultComp, - forceCTPasHdw=forceCtp, MultiplicityVetoList=["ZeroBiasA","ZeroBiasB"], - AlgorithmVetoList =["jXE40delay"]) + AlgorithmVetoList =["jXE40delay"], + forceCTPasHdw=forceCtp) #Define the Monitoring plots for L1Calo DQ helper.defineDQAlgorithm("L1TopoMismatchRate", - hanConfig={"libname":"libdqm_summaries.so","name":"Bins_GreaterThan_Threshold","BinThreshold":"0.1"}, # counts bins with value>0.9 - thresholdConfig={"NBins":[0,1]}, # warn if any high rate, error if more than 10 bins anywhere. + hanConfig={"libname":"libdqm_summaries.so","name":"Bins_GreaterThan_Threshold","BinThreshold":"0.001","PublishBins":"1"}, #,"ListHotRows":"\"25,58\""},#"ListHotRows":"\"25,58\""}, # counts bins with value>0.001 + thresholdConfig={"NBins":[0,1]}, # warn if any high rate, error if more than 1 bin anywhere. + ) + + helper.defineDQAlgorithm("L1TopoMismatchCountVerticalRange", + hanConfig={"libname":"libdqm_summaries.so","name":"Bins_GreaterThan_Threshold","BinThreshold":"0.001","PublishBins":"1","yMax":"2"}, #,"ListHotBins":"\"25:1,26:2\""}, # counts bins with value>0.001 for Bins in Y between 0 and 1. + thresholdConfig={"NBins":[0,1]}, # warn if any high rate, error if more than 1 bin anywhere. ) #Multiplicity mismatches between Sim and Hdw @@ -130,7 +142,7 @@ def getL1TopoPhase1DQMonitor(flags, name='L1TopoDQMonitor', doSimMon=True, doHwM helper.defineHistogram(name, fillGroup="L1TopoDQ_mismatches", paths=['Expert/Sim/detail/L1Topo/Multiplicities'], - hanConfig={"description":"Agreements and Mismatches between L1Topo Simulation and Hdw per L1Topo Item (x-axis). The upper row should be filled (Sim and Hdw agrees), while the lower two rows shouldn't have any entry","display":"SetPalette(55)"}, + hanConfig={"algorithm":"L1TopoMismatchCountVerticalRange","description":"Agreements and Mismatches between L1Topo Simulation and Hdw per L1Topo Item (x-axis). The upper row should be filled (Sim and Hdw agrees), while the lower two rows shouldn't have any entry","display":"SetPalette(55)"}, type='TH2D', title=title, xbins=len(xlabels), ybins=3, xlabels=xlabels,ylabels=ylabels, @@ -169,7 +181,7 @@ def getL1TopoPhase1DQMonitor(flags, name='L1TopoDQMonitor', doSimMon=True, doHwM helper.defineHistogram(name, fillGroup="L1TopoDQ_mismatches", paths=['Expert/Sim/detail/L1Topo/Algos'], - hanConfig={"description":"Agreements and Mismatches between L1Topo Simulation and Hardware per L1Topo Item (x-axis). The upper row should be filled (Sim and Hdw agrees), while the lower two rows shouldn't have any entry","display":"SetPalette(55)"}, + hanConfig={"algorithm":"L1TopoMismatchCountVerticalRange","description":"Agreements and Mismatches between L1Topo Simulation and Hardware per L1Topo Item (x-axis). The upper row should be filled (Sim and Hdw agrees), while the lower two rows shouldn't have any entry","display":"SetPalette(55)"}, type='TH2F', title=title,xbins=32,ybins=3, #weight=f'Phase1TopoWeight_{topo[0]}', -- GitLab From 9bdf3abbce157c543953ffb24dc676464e9d0c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jim=C3=A9nez=20Pe=C3=B1a?= <javier.jimenez.pena@cern.ch> Date: Mon, 24 Mar 2025 17:45:50 +0100 Subject: [PATCH 2/2] Modifing list of veto to ignoreBins. Now it accepts bins, rows and columns with a single string --- .../dqm_algorithms/BinThreshold.h | 3 +- .../dqm_algorithms/src/BinThreshold.cxx | 64 +++++++------------ .../python/L1TopoOnlineMonitoringConfig.py | 4 +- 3 files changed, 26 insertions(+), 45 deletions(-) diff --git a/DataQuality/dqm_algorithms/dqm_algorithms/BinThreshold.h b/DataQuality/dqm_algorithms/dqm_algorithms/BinThreshold.h index 1b3a752561fb..abfb0125f0c7 100644 --- a/DataQuality/dqm_algorithms/dqm_algorithms/BinThreshold.h +++ b/DataQuality/dqm_algorithms/dqm_algorithms/BinThreshold.h @@ -25,8 +25,7 @@ namespace dqm_algorithms bool CompareBinThreshold( const std::string & objname, double bincontent, double threshold ); using dqm_core::Algorithm::printDescription; void printDescription(std::ostream& out); - std::vector<int> parseNumbers(const std::string& input); - static std::vector<std::vector<int>> parseNumberPairs(const std::string& input); + void parseVetoList(const std::string& input, std::vector<int>& rows, std::vector<int>& columns, std::vector<std::vector<int>>& bins); private: std::string m_name; }; diff --git a/DataQuality/dqm_algorithms/src/BinThreshold.cxx b/DataQuality/dqm_algorithms/src/BinThreshold.cxx index 6369dc1ce277..53a4b3dd7a04 100644 --- a/DataQuality/dqm_algorithms/src/BinThreshold.cxx +++ b/DataQuality/dqm_algorithms/src/BinThreshold.cxx @@ -71,27 +71,16 @@ dqm_algorithms::BinThreshold::execute( const std::string & name, const int ymax = (int) dqm_algorithms::tools::GetFirstFromMap( "yMax", config.getParameters(), -1); const int ymin = (int) dqm_algorithms::tools::GetFirstFromMap( "yMin", config.getParameters(), -1); - //Strings with lists of vetoed rows, columns and bins - std::string listHotColumns = dqm_algorithms::tools::GetFirstFromMap("ListHotColumns", config.getGenericParameters(), "-1"); //The format is a list of int split by comas. Ex: "2,5,87" - std::string listHotRows = dqm_algorithms::tools::GetFirstFromMap("ListHotRows", config.getGenericParameters(), "-1"); //The format is a list of int split by comas. Ex: "2,5,87" - std::string listHotBins = dqm_algorithms::tools::GetFirstFromMap("ListHotBins", config.getGenericParameters(), "-1"); //The format is a list of int pairs, separated with : and split by comas. Ex: "2:3,5:1,8:7" + std::string ignoreBins = dqm_algorithms::tools::GetFirstFromMap("IgnoreBins", config.getGenericParameters(), "-1"); //The format is a list of int pairs, separated with : and split by comas. The symbol * can be used to indicate rows or columns. Ex: "2:3,5:*,*:7" - //Removing the Quotes which are needed to properly transmit the strings - std::string listHotRowsNoQuotes; - for (char c : listHotRows) if (c != '"') listHotRowsNoQuotes += c; - std::string listHotBinsNoQuotes; - for (char c : listHotBins) if (c != '"') listHotBinsNoQuotes += c; - std::string listHotColumnsNoQuotes; - for (char c : listHotColumns) if (c != '"') listHotColumnsNoQuotes += c; + std::vector<int> hotColumns, hotRows; + std::vector<std::vector<int>> hotBins; - bool vetoHotRows = false; if (listHotRows != "-1") vetoHotRows = true; - bool vetoHotBins = false; if (listHotBins != "-1") vetoHotBins = true; - bool vetoHotColumns = false; if (listHotColumns != "-1") vetoHotColumns = true; + parseVetoList(ignoreBins, hotRows, hotColumns, hotBins); - //Converting the string into vectors - std::vector<int> hotColumns = parseNumbers(listHotColumnsNoQuotes); - std::vector<int> hotRows = parseNumbers(listHotRowsNoQuotes); - auto hotBins = parseNumberPairs(listHotBinsNoQuotes); + bool vetoHotRows = false; if (hotRows.size() > 0) vetoHotRows = true; + bool vetoHotBins = false; if (hotBins.size() > 0) vetoHotBins = true; + bool vetoHotColumns = false; if (hotColumns.size() > 0) vetoHotColumns = true; if (histogram->GetEntries() < minstat ) { dqm_core::Result *result = new dqm_core::Result(dqm_core::Result::Undefined); @@ -246,32 +235,28 @@ dqm_algorithms::BinThreshold::CompareBinThreshold(const std::string & type, doub return 0; } -std::vector<int> dqm_algorithms::BinThreshold::parseNumbers(const std::string& input) { - std::vector<int> numbers; - std::stringstream ss(input); - std::string token; +void dqm_algorithms::BinThreshold::parseVetoList(const std::string& input, std::vector<int>& rows, std::vector<int>& columns, std::vector<std::vector<int>>& bins) { - while (std::getline(ss, token, ',')) { - numbers.push_back(std::stoi(token)); - } - - return numbers; -} + std::string inputNoQuotes; + for (char c : input) if (c != '"') inputNoQuotes += c; -std::vector<std::vector<int>> dqm_algorithms::BinThreshold::parseNumberPairs(const std::string& input) { - std::vector<std::vector<int>> numbers; - std::stringstream ss(input); - std::string pairToken; + std::stringstream ss(inputNoQuotes); + std::string token; - while (std::getline(ss, pairToken, ',')) { - std::stringstream pairStream(pairToken); + while (std::getline(ss, token, ',')) { + std::stringstream pairStream(token); std::string first, second; - + if (std::getline(pairStream, first, ':') && std::getline(pairStream, second, ':')) { - numbers.push_back({std::stoi(first), std::stoi(second)}); + if (first == "*") { + rows.push_back(std::stoi(second)); + } else if (second == "*") { + columns.push_back(std::stoi(first)); + } else { + bins.push_back({std::stoi(first), std::stoi(second)}); + } } } - return numbers; } void @@ -291,10 +276,7 @@ dqm_algorithms::BinThreshold::printDescription(std::ostream& out) out<<"Optional Parameter: xmax: maximum x range"<<std::endl; out<<"Optional Parameter: ymin: minimum y range"<<std::endl; out<<"Optional Parameter: ymax: maximum y range\n"<<std::endl; - out<<"Optional Parameter: listHotBins: list of known hot bins (x,y) in a string, separated by comas and semicolons. Ex: 1:3,6:7 \n"<<std::endl; - out<<"Optional Parameter: listHotRows: list of known hot rows in a string, separated by comas. Ex: 1,3,6,7 \n"<<std::endl; - out<<"Optional Parameter: listHotColumns: list of known hot columnss in a string, separated by comas. Ex: 1,3,6,7 \n"<<std::endl; - + out<<"Optional Parameter: ignoreBins: list of bins to ignore in the threshold comparison (x,y) in a string, separated by comas and semicolons. Ex: 1:3,6:7. It also allows to pass rows and columns like *:4 and *:4 respectively \n"<<std::endl; } diff --git a/Trigger/TrigT1/L1Topo/L1TopoOnlineMonitoring/python/L1TopoOnlineMonitoringConfig.py b/Trigger/TrigT1/L1Topo/L1TopoOnlineMonitoring/python/L1TopoOnlineMonitoringConfig.py index e1f14a78ea5f..bbff37ecabc0 100644 --- a/Trigger/TrigT1/L1Topo/L1TopoOnlineMonitoring/python/L1TopoOnlineMonitoringConfig.py +++ b/Trigger/TrigT1/L1Topo/L1TopoOnlineMonitoring/python/L1TopoOnlineMonitoringConfig.py @@ -100,12 +100,12 @@ def getL1TopoPhase1DQMonitor(flags, name='L1TopoDQMonitor', doSimMon=True, doHwM #Define the Monitoring plots for L1Calo DQ helper.defineDQAlgorithm("L1TopoMismatchRate", - hanConfig={"libname":"libdqm_summaries.so","name":"Bins_GreaterThan_Threshold","BinThreshold":"0.001","PublishBins":"1"}, #,"ListHotRows":"\"25,58\""},#"ListHotRows":"\"25,58\""}, # counts bins with value>0.001 + hanConfig={"libname":"libdqm_summaries.so","name":"Bins_GreaterThan_Threshold","BinThreshold":"0.001","PublishBins":"1"}, # counts bins with value>0.001 thresholdConfig={"NBins":[0,1]}, # warn if any high rate, error if more than 1 bin anywhere. ) helper.defineDQAlgorithm("L1TopoMismatchCountVerticalRange", - hanConfig={"libname":"libdqm_summaries.so","name":"Bins_GreaterThan_Threshold","BinThreshold":"0.001","PublishBins":"1","yMax":"2"}, #,"ListHotBins":"\"25:1,26:2\""}, # counts bins with value>0.001 for Bins in Y between 0 and 1. + hanConfig={"libname":"libdqm_summaries.so","name":"Bins_GreaterThan_Threshold","BinThreshold":"0.001","PublishBins":"1","yMax":"2"}, #,"IgnoreBins":"\"25:1,26:2,*:25,*:58,500:*\""}, # counts bins with value>0.001 for Bins in Y between 0 and 1. thresholdConfig={"NBins":[0,1]}, # warn if any high rate, error if more than 1 bin anywhere. ) -- GitLab