Commit 1a247999 authored by Sebastien Ponce's avatar Sebastien Ponce
Browse files

Implementation of a new set of counters for Gaudi, replacing StatEntity

The new counters have a few new features compared to existing StatEntity :
  - they are templated by the counted type
  - they come with different flavors counting only what you need (e.g. simple count, no stat)
  - more flavour can easily be created at will thanks to the AccumulatorSet object
  - they come with 2 implementations : thread safe or not
  - the thread safe ones use atomics and should be as efficient as possible
  - a Buffer object allows to easily buffer updates to a counter made in a tight loop, without paying the atomic price each time

A Small unit test is available in the test suite and example code is provided in the comments of Counters.h where they are implemented.
Existing code has been adapted, in particulat the ChronoSvc.
parent 94045bef
......@@ -121,8 +121,8 @@ protected: // definitions
protected: // few actual data types
// ==========================================================================
/// the actual type of general counters
typedef std::list<StatEntity> StatisticsOwn;
typedef std::map<std::string, std::reference_wrapper<StatEntity>> Statistics;
typedef std::map<std::string, StatEntity> StatisticsOwn;
typedef std::map<std::string, std::reference_wrapper<Gaudi::Accumulators::PrintableCounter>> Statistics;
/// the actual type error/warning counter
typedef std::map<std::string, unsigned int> Counter;
/// storage for active tools
......@@ -483,6 +483,9 @@ public:
void Exception( const std::string& msg = "no message",
const StatusCode sc = StatusCode( StatusCode::FAILURE, true ) ) const;
private:
/// accessor to all owned counters
inline StatisticsOwn countersOwn() const { return m_countersOwn; }
public:
// ==========================================================================
/// accessor to all counters
......@@ -505,7 +508,8 @@ public:
* @param tag counter name
* @return the counter itself
*/
[[deprecated( "see LHCBPS-1758" )]] inline StatEntity& counter( const std::string& tag ) const
//[[deprecated( "see LHCBPS-1758" )]]
inline StatEntity& counter( const std::string& tag ) const
{
return const_cast<GaudiCommon<PBASE>*>( this )->counter( tag );
}
......@@ -515,12 +519,12 @@ public:
// Return referenced StatEntity if it already exists, else create it
auto p = m_counters.find( tag );
if ( p == end( m_counters ) ) {
m_countersOwn.emplace_back();
p = m_counters.emplace( tag, m_countersOwn.back() ).first;
auto& counter = m_countersOwn[tag];
p = m_counters.emplace( tag, counter ).first;
}
return p->second;
return m_countersOwn[tag];
}
inline void registerCounter( const std::string& tag, StatEntity& r )
inline void registerCounter( const std::string& tag, Gaudi::Accumulators::PrintableCounter& r )
{
std::lock_guard<std::mutex> lock( m_countersMutex );
m_counters.emplace( tag, r );
......@@ -775,7 +779,7 @@ private:
mutable Counter m_exceptions;
/// General counters
StatisticsOwn m_countersOwn;
Statistics m_counters;
Statistics m_counters;
/// The counters mutex
std::mutex m_countersMutex;
// ==========================================================================
......
......@@ -221,8 +221,7 @@ StatusCode GaudiCommon<PBASE>::
Gaudi::Utils::RegEx::matchList statList{m_statEntityList.value()};
Gaudi::Utils::RegEx::matchList counterList{m_counterList.value()};
std::map<std::string, StatEntity&> ordered_counters{begin( this->m_counters ), end( this->m_counters )};
for ( const auto& i : ordered_counters ) {
for ( const auto& i : m_countersOwn ) {
if ( statList.Or( i.first ) )
m_counterSummarySvc->addCounter( this->name(), i.first, i.second, Gaudi::CounterSummary::SaveStatEntity );
else if ( counterList.Or( i.first ) )
......@@ -528,13 +527,13 @@ long GaudiCommon<PBASE>::printStat( const MSG::Level level ) const
msg << "Number of counters : " << m_counters.size();
//
if ( !m_counters.empty() ) {
msg << std::endl << m_header.value();
msg << "\n" << m_header.value();
}
//
std::map<std::string, StatEntity&> ordered_counters{begin( m_counters ), end( m_counters )};
for ( const auto& entry : ordered_counters ) {
msg << std::endl
<< Gaudi::Utils::formatAsTableRow( entry.first, entry.second, m_useEffFormat, m_format1, m_format2 );
for ( const auto& entry : m_counters ) {
std::ostringstream ost;
entry.second.get().print( ost, entry.first );
msg << "\n |" << ost.str();
}
//
msg << endmsg;
......
......@@ -145,7 +145,9 @@ std::string GaudiAlg::Print2DProf::toString( const AIDA::IProfile2D* aida, const
// ============================================================================
std::string GaudiAlg::PrintStat::print( const StatEntity& stat, const std::string& tag )
{
return Gaudi::Utils::formatAsTableRow( tag, stat );
std::ostringstream ost;
stat.print( ost, true, tag );
return ost.str();
}
// ============================================================================
std::string GaudiAlg::PrintTuple::print( const INTuple* tuple, const GaudiAlg::TupleID& ID )
......
......@@ -123,8 +123,6 @@ StatusCode ChronoStatSvc::initialize()
}
}
info() << " Number of skipped events for MemStat" << m_numberOfSkippedEventsForMemStat.value() << endmsg;
if ( m_chronoTableFlag && !m_printUserTime && !m_printSystemTime && !m_printEllapsedTime ) {
m_printUserTime = true;
}
......@@ -365,7 +363,6 @@ void ChronoStatSvc::stat( const IChronoStatSvc::StatTag& statTag, const IChronoS
// new stat entity
StatEntity& theSe = m_statEntities[statTag];
theStat = &theSe;
theStat->setnEntriesBeforeReset( m_numberOfSkippedEventsForMemStat );
} else {
// existing stat entity
theStat = &theIter->second;
......@@ -400,7 +397,7 @@ const ChronoEntity* ChronoStatSvc::chrono( const IChronoStatSvc::ChronoTag& t )
* @return pointer to stat entity
*/
// ============================================================================
const StatEntity* ChronoStatSvc::stat( const IChronoStatSvc::StatTag& t ) const
StatEntity* ChronoStatSvc::stat( const IChronoStatSvc::StatTag& t )
{
auto it = m_statEntities.find( t );
return m_statEntities.end() != it ? &( it->second ) : nullptr;
......@@ -528,10 +525,11 @@ void ChronoStatSvc::printStats()
} /// CONTINUE
///
if ( m_statCoutFlag ) {
std::cout << Gaudi::Utils::formatAsTableRow( *tag, *entity, m_useEffFormat, m_format1, m_format2 ) << std::endl;
entity->print( std::cout, true, *tag, m_useEffFormat, "%|-15.15s|%|17t|" );
} else {
log << m_statPrintLevel << Gaudi::Utils::formatAsTableRow( *tag, *entity, m_useEffFormat, m_format1, m_format2 )
<< endmsg;
std::ostringstream ost;
entity->print( ost, true, *tag, m_useEffFormat, "%|-15.15s|%|17t|" );
log << m_statPrintLevel << ost.str() << endmsg;
}
}
tmpCont.clear();
......
......@@ -97,7 +97,7 @@ public:
* @param t stat tag(name)
* @return pointer to stat entity
*/
const StatEntity* stat( const IChronoStatSvc::StatTag& t ) const override;
StatEntity* stat( const IChronoStatSvc::StatTag& t ) override;
// ============================================================================
/** Default constructor.
* @param name service instance name
......@@ -154,10 +154,6 @@ private:
Gaudi::Property<int> m_intStatPrintLevel{this, "StatPrintLevel", MSG::INFO, "print level"};
Gaudi::Property<bool> m_statOrderFlag{this, "StatTableToBeOrdered", true, "should the printout be ordered"};
Gaudi::Property<long> m_numberOfSkippedEventsForMemStat{
this, "NumberOfSkippedEventsForMemStat", -1,
"specify the number of events to be skipped by the memory auditor in order to better spot memory leak"};
Gaudi::Property<std::string> m_statsOutFileName{
this, "AsciiStatsOutputFile", "",
"Name of the output file storing the stats. If empty, no statistics will be saved (default)"};
......@@ -166,13 +162,6 @@ private:
this, "StatTableHeader",
" Counter | # | sum | mean/eff^* | rms/err^* | min | max |",
"The header row for the output Stat-table"};
Gaudi::Property<std::string> m_format1{
this, "RegularRowFormat", " %|-15.15s|%|17t||%|10d| |%|11.7g| |%|#11.5g| |%|#11.5g| |%|#12.5g| |%|#12.5g| |",
"The format for the regular row in the output Stat-table"};
Gaudi::Property<std::string> m_format2{
this, "EfficiencyRowFormat",
"*%|-15.15s|%|17t||%|10d| |%|11.5g| |(%|#9.7g| +- %|-#9.7g|)%%| ------- | ------- |",
"The format for the regular row in the output Stat-table"};
Gaudi::Property<bool> m_useEffFormat{this, "UseEfficiencyRowFormat", true,
"Use the special format for printout of efficiency counters"};
......
......@@ -195,3 +195,5 @@ gaudi_add_test(WriteAndReadHandleWhiteBoard
gaudi_add_test(nose
COMMAND nosetests -v
${CMAKE_CURRENT_SOURCE_DIR}/tests/nose)
gaudi_add_executable(countersUnitTest src/CounterEx/CountersUnitTest.cpp)
......@@ -61,15 +61,15 @@ public:
private:
// counters
StatEntity m_assign_counter{this, "assign"};
StatEntity m_eff_counter{this, "eff"};
StatEntity m_executed_counter{this, "executed"};
StatEntity m_G_counter{this, "G"};
StatEntity m_g2_counter{this, "g2"};
StatEntity m_gauss_counter{this, "gauss"};
StatEntity m_Gneg_counter{this, "Gneg"};
StatEntity m_Gpos_counter{this, "Gpos"};
StatEntity m_NG_counter{this, "NG"};
mutable StatEntity m_assign_counter{this, "assign"};
mutable StatEntity m_eff_counter{this, "eff"};
mutable StatEntity m_executed_counter{this, "executed"};
mutable StatEntity m_G_counter{this, "G"};
mutable StatEntity m_g2_counter{this, "g2"};
mutable StatEntity m_gauss_counter{this, "gauss"};
mutable StatEntity m_Gneg_counter{this, "Gneg"};
mutable StatEntity m_Gpos_counter{this, "Gpos"};
mutable StatEntity m_NG_counter{this, "NG"};
};
// ============================================================================
......@@ -95,8 +95,8 @@ int CounterNewAlg::operator()() const
( 0 < value ) ? ++m_Gpos_counter : ++m_Gneg_counter;
const StatEntity& stat1 = m_NG_counter;
const StatEntity& stat2 = m_G_counter;
StatEntity& stat1 = m_NG_counter;
StatEntity& stat2 = m_G_counter;
const int num = (int)poisson();
for ( int i = 0; i < num; ++i ) {
......@@ -111,8 +111,8 @@ int CounterNewAlg::operator()() const
m_eff_counter += ( 0 < value );
// print the statistics every 1000 events
const StatEntity& executed = m_executed_counter;
const int print = (int)executed.flag();
StatEntity& executed = m_executed_counter;
const int print = (int)executed.flag();
if ( 0 == print % 1000 ) {
info() << " Event number " << print << endmsg;
printStat();
......
#include "GaudiKernel/Counters.h"
#include <iostream>
using namespace Gaudi::Accumulators;
int main()
{
// Testing Buffer moving
{
Counter<double, atomicity::full> c;
c += 3.5;
c += 1.2;
{
auto buf = c.buffer();
buf += 7.2; // buf1 internal count = 1
decltype( buf ) buf2( std::move( buf ) );
buf2 += 3.5; // buf3 internal count = 2
decltype( buf ) buf3 = std::move( buf2 );
buf3 += 3.7; // buf3 internal count = 3
}
std::cout << c << std::endl; // should output 5
}
// Testing counter resets
{
SigmaCounter<> sig;
sig += 3;
sig += 5;
sig += 6;
std::cout << sig << std::endl;
sig.reset();
std::cout << sig << std::endl;
}
// Testing AveragingCounter
{
AveragingCounter<> avg;
avg += 3;
avg += 5;
avg += 6;
std::cout << avg << std::endl;
}
// Testing SigmaCounter
{
SigmaCounter<> sig;
sig += 3;
sig += 5;
sig += 6;
std::cout << sig << std::endl;
}
// Testing AveragingCounter wiht buffering
{
AveragingCounter<float, atomicity::full> avg2;
{
auto bufAvg = avg2.buffer();
for ( int i = 0; i < 1000; i++ ) bufAvg += i;
}
std::cout << avg2 << std::endl;
}
// Testing Binomialcounter
{
BinomialCounter<> bin;
bin += false;
bin += true;
bin += true;
bin += false;
bin += false;
std::cout << bin << std::endl;
}
// Testing StatEntity, the backward compatible counter
{
StatEntity se;
se += 3;
se += 5;
se += 6;
std::cout << se << std::endl;
}
// Testing StatEntity, the backward compatible counter with binomial usage
{
StatEntity sb;
sb += 0;
sb += 1;
sb += 1;
sb += 0;
sb += 0;
std::cout << sb << std::endl;
}
// Testing StatEntity, setting directly values
{
StatEntity sb{3, 14, 70, 3, 6};
std::cout << sb << std::endl;
}
// Set of strings
// vector of values (extended binomial ?)
}
<?xml version="1.0" ?><!DOCTYPE extension PUBLIC '-//QM/2.3/Extension//EN' 'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
<extension class="GaudiTest.GaudiExeTest" kind="test">
<argument name="program"><text>countersUnitTest</text></argument>
<argument name="reference"><text>refs/CountersUnitTest.ref</text></argument>
</extension>
......@@ -272,7 +272,6 @@ SimpleHistos DEBUG Booked 1D Histogram : ID='test1' Path=SimpleHistos Tit
SimpleHistos INFO GaudiHistoAlgorithm:: Filling Histograms...... Please be patient !
Histos2 WARNING Gaudi::Examples::HistoProps:: Cannot generate automatic literal ID from an empty title ! Using numeric ID instead for histogram ID
HistogramDataSvc DEBUG Redefine the parameters for the histogram 'Histos2/2' to be ('TEST2',-100,200,100)
ChronoStatSvc INFO Number of skipped events for MemStat-1
ApplicationMgr INFO Application Manager Stopped successfully
SimpleHistos SUCCESS Booked 30 Histogram(s) : 1D=10 2D=5 3D=3 1DProf=9 2DProf=3
SimpleHistos SUCCESS List of booked 1D histograms in directory "SimpleHistos" :-
......
......@@ -36,7 +36,6 @@ ApplicationMgr SUCCESS
====================================================================================================================================
ApplicationMgr INFO Application Manager Configured successfully
StatusCodeSvc INFO initialize
ChronoStatSvc INFO Number of skipped events for MemStat-1
RndmGenSvc.Engine INFO Generator engine type:CLHEP::RanluxEngine
RndmGenSvc.Engine INFO Current Seed:1234567 Luxury:3
RndmGenSvc INFO Using Random engine:HepRndm::Engine<CLHEP::RanluxEngine>
......
......@@ -17,7 +17,6 @@ RndmGenSvc.Engine INFO Generator engine type:CLHEP::RanluxEngine
RndmGenSvc.Engine INFO Current Seed:1234567 Luxury:3
RndmGenSvc INFO Using Random engine:HepRndm::Engine<CLHEP::RanluxEngine>
TIMER.TIMER INFO This machine has a speed about 2.63 times the speed of a 2.8 GHz Xeon.
ChronoStatSvc INFO Number of skipped events for MemStat-1
ParentAlg INFO creating sub-algorithms....
SubAlg1 INFO initializing....
SubAlg2 INFO initializing....
......
#=5
#=3 Sum=14 Mean= 4.667 +- 1.2472
#=0 Sum=0 Mean= 0.000 +- 0.0000
#=3 Sum=14 Mean= 4.667
#=3 Sum=14 Mean= 4.667 +- 1.2472
#=1000 Sum=4.995e+05 Mean= 499.5
#=5 Sum=2 Eff=|( 40.00000 +- 21.9089 )%|
#=3 Sum=14 Mean= 4.667 +- 1.2472 Min/Max= 3.000/6.000
#=5 Sum=2 Mean= 0.4000 +- 0.48990 Min/Max= 0.000/1.000
#=3 Sum=14 Mean= 4.667 +- 1.2472 Min/Max= 3.000/6.000
......@@ -214,7 +214,6 @@ SimpleHistos DEBUG Booked 1D Histogram : ID='test1' Path=SimpleHistos Tit
SimpleHistos INFO GaudiHistoAlgorithm:: Filling Histograms...... Please be patient !
Histos2 WARNING Gaudi::Examples::HistoProps:: Cannot generate automatic literal ID from an empty title ! Using numeric ID instead for histogram ID
HistogramDataSvc DEBUG Redefine the parameters for the histogram 'Histos2/2' to be ('TEST2',-100,200,100)
ChronoStatSvc INFO Number of skipped events for MemStat-1
ApplicationMgr INFO Application Manager Stopped successfully
SimpleHistos SUCCESS Booked 30 Histogram(s) : 1D=10 2D=5 3D=3 1DProf=9 2DProf=3
SimpleHistos SUCCESS List of booked 1D histograms in directory "SimpleHistos" :-
......
......@@ -21,7 +21,6 @@ ApplicationMgr SUCCESS
====================================================================================================================================
ApplicationMgr INFO Application Manager Configured successfully
StatusCodeSvc INFO initialize
ChronoStatSvc INFO Number of skipped events for MemStat-1
EventLoopMgr WARNING Unable to locate service "EventSelector"
EventLoopMgr WARNING No events will be processed from external input.
HistogramPersis...WARNING Histograms saving not required.
......
......@@ -21,7 +21,6 @@ RndmGenSvc.Engine INFO Generator engine type:CLHEP::RanluxEngine
RndmGenSvc.Engine INFO Current Seed:1234567 Luxury:3
RndmGenSvc INFO Using Random engine:HepRndm::Engine<CLHEP::RanluxEngine>
TIMER.TIMER INFO This machine has a speed about 3.03 times the speed of a 2.8 GHz Xeon.
ChronoStatSvc INFO Number of skipped events for MemStat-1
ParentAlg INFO creating sub-algorithms....
SubAlg1 INFO initializing....
SubAlg2 INFO initializing....
......
......@@ -14,7 +14,6 @@ ApplicationMgr SUCCESS
ApplicationMgr INFO Application Manager Configured successfully
ApplicationMgr INFO Successfully loaded modules : GaudiAlg, RootHistCnv
NTupleSvc INFO Added stream file:TupleEx3.root as MYLUN
ChronoStatSvc INFO Number of skipped events for MemStat-1
RootHistSvc INFO Writing ROOT histograms to: UndefinedROOTOutputFileName
HistogramPersis... INFO Added successfully Conversion service:RootHistSvc
DetectorDataSvc INFO Detector description not requested to be loaded
......
......@@ -26,7 +26,6 @@ ApplicationMgr SUCCESS
ApplicationMgr INFO Application Manager Configured successfully
StatusCodeSvc INFO initialize
Google::CPUProf... INFO Initialised
ChronoStatSvc INFO Number of skipped events for MemStat-1
Google::CPUProf... INFO Starting Auditor for HelloWorld:Initialize
HelloWorld INFO initializing....
EventLoopMgr WARNING Unable to locate service "EventSelector"
......
......@@ -25,7 +25,6 @@ ApplicationMgr SUCCESS
ApplicationMgr INFO Application Manager Configured successfully
StatusCodeSvc INFO initialize
Google::HeapChe... INFO Initialised
ChronoStatSvc INFO Number of skipped events for MemStat-1
Google::HeapChe... INFO Starting Auditor for HelloWorld:Initialize
HelloWorld INFO initializing....
EventLoopMgr WARNING Unable to locate service "EventSelector"
......
......@@ -25,7 +25,6 @@ ApplicationMgr SUCCESS
ApplicationMgr INFO Application Manager Configured successfully
StatusCodeSvc INFO initialize
Google::HeapPro... INFO Initialised
ChronoStatSvc INFO Number of skipped events for MemStat-1
Google::HeapPro... INFO Starting Auditor for HelloWorld:Initialize
HelloWorld INFO initializing....
EventLoopMgr WARNING Unable to locate service "EventSelector"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment