Commit 739980f5 authored by Marco Clemencic's avatar Marco Clemencic
Browse files

Implementation of a new set of counters (mr !629)

parents 5d78051d 1a247999
......@@ -6,7 +6,10 @@
// from STL
// ============================================================================
#include <algorithm>
#include <functional>
#include <list>
#include <map>
#include <mutex>
#include <string>
#include <vector>
// ============================================================================
......@@ -28,17 +31,6 @@
#include "GaudiKernel/StatEntity.h"
#include "GaudiKernel/StatusCode.h"
#include "GaudiKernel/System.h"
// ============================================================================
// TBB
// ============================================================================
#if defined( __GNUC__ ) && __GNUC__ >= 5
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsuggest-override"
#endif
#include "tbb/concurrent_unordered_map.h"
#if defined( __GNUC__ ) && __GNUC__ >= 5
#pragma GCC diagnostic pop
#endif
#ifdef __CLING__
#define WARN_UNUSED
......@@ -129,7 +121,8 @@ protected: // definitions
protected: // few actual data types
// ==========================================================================
/// the actual type of general counters
typedef tbb::concurrent_unordered_map<std::string, StatEntity> Statistics;
typedef std::map<std::string, StatEntity> StatisticsOwn;
typedef std::map<std::string, std::reference_wrapper<Gaudi::Accumulators::PrintableCounter>> Statistics;
/// the actual type error/warning counter
typedef std::map<std::string, unsigned int> Counter;
/// storage for active tools
......@@ -490,10 +483,13 @@ public:
void Exception( const std::string& msg = "no message",
const StatusCode sc = StatusCode( StatusCode::FAILURE, true ) ) const;
private:
/// accessor to all owned counters
inline StatisticsOwn countersOwn() const { return m_countersOwn; }
public:
// ==========================================================================
/// accessor to all counters
inline const Statistics& counters() const { return m_counters; }
inline const Statistics counters() const { return m_counters; }
/** accessor to certain counter by name
*
* @code
......@@ -512,7 +508,27 @@ public:
* @param tag counter name
* @return the counter itself
*/
inline StatEntity& counter( const std::string& tag ) const { return m_counters[tag]; }
//[[deprecated( "see LHCBPS-1758" )]]
inline StatEntity& counter( const std::string& tag ) const
{
return const_cast<GaudiCommon<PBASE>*>( this )->counter( tag );
}
inline StatEntity& counter( const std::string& tag )
{
std::lock_guard<std::mutex> lock( m_countersMutex );
// Return referenced StatEntity if it already exists, else create it
auto p = m_counters.find( tag );
if ( p == end( m_counters ) ) {
auto& counter = m_countersOwn[tag];
p = m_counters.emplace( tag, counter ).first;
}
return m_countersOwn[tag];
}
inline void registerCounter( const std::string& tag, Gaudi::Accumulators::PrintableCounter& r )
{
std::lock_guard<std::mutex> lock( m_countersMutex );
m_counters.emplace( tag, r );
}
// ==========================================================================
public:
/// Insert the actual C++ type of the algorithm/tool in the messages ?
......@@ -762,7 +778,10 @@ private:
/// Counter of exceptions
mutable Counter m_exceptions;
/// General counters
mutable Statistics m_counters;
StatisticsOwn m_countersOwn;
Statistics m_counters;
/// The counters mutex
std::mutex m_countersMutex;
// ==========================================================================
/// Pointer to the Update Manager Service instance
mutable IUpdateManagerSvc* m_updMgrSvc = nullptr;
......
......@@ -210,7 +210,7 @@ StatusCode GaudiCommon<PBASE>::
StatusCode sc = StatusCode::SUCCESS;
// print the general information about statistical counters
if ( this->msgLevel( MSG::DEBUG ) || ( statPrint() && !counters().empty() ) ) {
if ( this->msgLevel( MSG::DEBUG ) || ( statPrint() && !m_counters.empty() ) ) {
// print general statistical counters
printStat( statPrint() ? MSG::ALWAYS : MSG::DEBUG );
}
......@@ -221,8 +221,7 @@ StatusCode GaudiCommon<PBASE>::
Gaudi::Utils::RegEx::matchList statList{m_statEntityList.value()};
Gaudi::Utils::RegEx::matchList counterList{m_counterList.value()};
std::map<std::string, StatEntity> ordered_counters{begin( this->counters() ), end( this->counters() )};
for ( const auto& i : ordered_counters ) {
for ( const auto& i : m_countersOwn ) {
if ( statList.Or( i.first ) )
m_counterSummarySvc->addCounter( this->name(), i.first, i.second, Gaudi::CounterSummary::SaveStatEntity );
else if ( counterList.Or( i.first ) )
......@@ -264,7 +263,8 @@ StatusCode GaudiCommon<PBASE>::
}
// clear *ALL* counters explicitly
m_counters.clear();
m_counters.clear(); // delete pointers first
m_countersOwn.clear(); // then delete any owned counters
m_exceptions.clear();
m_infos.clear();
m_warnings.clear();
......@@ -519,26 +519,26 @@ template <class PBASE>
long GaudiCommon<PBASE>::printStat( const MSG::Level level ) const
{
// print statistics
if ( counters().empty() ) {
if ( m_counters.empty() ) {
return 0;
}
MsgStream& msg = this->msgStream( level );
//
msg << "Number of counters : " << counters().size();
msg << "Number of counters : " << m_counters.size();
//
if ( !counters().empty() ) {
msg << std::endl << m_header.value();
if ( !m_counters.empty() ) {
msg << "\n" << m_header.value();
}
//
std::map<std::string, StatEntity> ordered_counters{begin( counters() ), end( counters() )};
for ( const auto& entry : ordered_counters ) {
msg << std::endl
<< Gaudi::Utils::formatAsTableRow( entry.first, entry.second, m_useEffFormat, m_format1, m_format2 );
for ( const auto& entry : m_counters ) {
std::ostringstream ost;
entry.second.get().print( ost, entry.first );
msg << "\n |" << ost.str();
}
//
msg << endmsg;
//
return counters().size();
return m_counters.size();
}
// ============================================================================
......
......@@ -145,7 +145,9 @@ std::string GaudiAlg::Print2DProf::toString( const AIDA::IProfile2D* aida, const
// ============================================================================
std::string GaudiAlg::PrintStat::print( const StatEntity& stat, const std::string& tag )
{
return Gaudi::Utils::formatAsTableRow( tag, stat );
std::ostringstream ost;
stat.print( ost, true, tag );
return ost.str();
}
// ============================================================================
std::string GaudiAlg::PrintTuple::print( const INTuple* tuple, const GaudiAlg::TupleID& ID )
......
......@@ -123,8 +123,6 @@ StatusCode ChronoStatSvc::initialize()
}
}
info() << " Number of skipped events for MemStat" << m_numberOfSkippedEventsForMemStat.value() << endmsg;
if ( m_chronoTableFlag && !m_printUserTime && !m_printSystemTime && !m_printEllapsedTime ) {
m_printUserTime = true;
}
......@@ -365,7 +363,6 @@ void ChronoStatSvc::stat( const IChronoStatSvc::StatTag& statTag, const IChronoS
// new stat entity
StatEntity& theSe = m_statEntities[statTag];
theStat = &theSe;
theStat->setnEntriesBeforeReset( m_numberOfSkippedEventsForMemStat );
} else {
// existing stat entity
theStat = &theIter->second;
......@@ -400,7 +397,7 @@ const ChronoEntity* ChronoStatSvc::chrono( const IChronoStatSvc::ChronoTag& t )
* @return pointer to stat entity
*/
// ============================================================================
const StatEntity* ChronoStatSvc::stat( const IChronoStatSvc::StatTag& t ) const
StatEntity* ChronoStatSvc::stat( const IChronoStatSvc::StatTag& t )
{
auto it = m_statEntities.find( t );
return m_statEntities.end() != it ? &( it->second ) : nullptr;
......@@ -528,10 +525,11 @@ void ChronoStatSvc::printStats()
} /// CONTINUE
///
if ( m_statCoutFlag ) {
std::cout << Gaudi::Utils::formatAsTableRow( *tag, *entity, m_useEffFormat, m_format1, m_format2 ) << std::endl;
entity->print( std::cout, true, *tag, m_useEffFormat, "%|-15.15s|%|17t|" );
} else {
log << m_statPrintLevel << Gaudi::Utils::formatAsTableRow( *tag, *entity, m_useEffFormat, m_format1, m_format2 )
<< endmsg;
std::ostringstream ost;
entity->print( ost, true, *tag, m_useEffFormat, "%|-15.15s|%|17t|" );
log << m_statPrintLevel << ost.str() << endmsg;
}
}
tmpCont.clear();
......
......@@ -97,7 +97,7 @@ public:
* @param t stat tag(name)
* @return pointer to stat entity
*/
const StatEntity* stat( const IChronoStatSvc::StatTag& t ) const override;
StatEntity* stat( const IChronoStatSvc::StatTag& t ) override;
// ============================================================================
/** Default constructor.
* @param name service instance name
......@@ -154,10 +154,6 @@ private:
Gaudi::Property<int> m_intStatPrintLevel{this, "StatPrintLevel", MSG::INFO, "print level"};
Gaudi::Property<bool> m_statOrderFlag{this, "StatTableToBeOrdered", true, "should the printout be ordered"};
Gaudi::Property<long> m_numberOfSkippedEventsForMemStat{
this, "NumberOfSkippedEventsForMemStat", -1,
"specify the number of events to be skipped by the memory auditor in order to better spot memory leak"};
Gaudi::Property<std::string> m_statsOutFileName{
this, "AsciiStatsOutputFile", "",
"Name of the output file storing the stats. If empty, no statistics will be saved (default)"};
......@@ -166,13 +162,6 @@ private:
this, "StatTableHeader",
" Counter | # | sum | mean/eff^* | rms/err^* | min | max |",
"The header row for the output Stat-table"};
Gaudi::Property<std::string> m_format1{
this, "RegularRowFormat", " %|-15.15s|%|17t||%|10d| |%|11.7g| |%|#11.5g| |%|#11.5g| |%|#12.5g| |%|#12.5g| |",
"The format for the regular row in the output Stat-table"};
Gaudi::Property<std::string> m_format2{
this, "EfficiencyRowFormat",
"*%|-15.15s|%|17t||%|10d| |%|11.5g| |(%|#9.7g| +- %|-#9.7g|)%%| ------- | ------- |",
"The format for the regular row in the output Stat-table"};
Gaudi::Property<bool> m_useEffFormat{this, "UseEfficiencyRowFormat", true,
"Use the special format for printout of efficiency counters"};
......
......@@ -195,3 +195,5 @@ gaudi_add_test(WriteAndReadHandleWhiteBoard
gaudi_add_test(nose
COMMAND nosetests -v
${CMAKE_CURRENT_SOURCE_DIR}/tests/nose)
gaudi_add_executable(countersUnitTest src/CounterEx/CountersUnitTest.cpp)
// ============================================================================
/** @file
* configuration file to run "CounterNewEx" : example showing the usage
* of the new 'counter' facilities from the GaudiAlgorithm base class
* @see CounterNewAlg
* @see GaudiAlgorithm
* @author Vanya BELYAEV Ivan.Belyaev@lapp.in2p3.fr
* @date 2005-08-06
* @author Andrea Valassi
* @date 2018-02-13
*/
ApplicationMgr.ExtSvc += { "RndmGenSvc" } ;
ApplicationMgr.TopAlg = { "CounterNewAlg/Counter" } ;
// Set output level threshold (2=DEBUG, 3=INFO, 4=WARNING, 5=ERROR, 6=FATAL )
MessageSvc.OutputLevel = 3;
ApplicationMgr.EvtMax = 5400 ;
ApplicationMgr.EvtSel = "NONE" ;
// ============================================================================
//
// ============================================================================
// The END
// ============================================================================
// ============================================================================
// Include files
// ============================================================================
// GaudiKernel
// ============================================================================
#include "GaudiKernel/RndmGenerators.h"
// ============================================================================
// GaudiAlg
// ============================================================================
#include "GaudiAlg/Producer.h"
// ============================================================================
// ============================================================================
/** @file
* Simple example showing the usage of the new 'counter' facilities
* (with a simple 'producer' type of functional algorithm)
*
* @see GaudiAlgorithm
* @see StatEntity
*
* @author Vanya BELYAEV Ivan.Belyaev@lapp.in2p3.fr
* @date 2008-08-06
*
* @author Andrea Valassi
* @date 2018-02-13
*/
// ============================================================================
// ============================================================================
/** @class CounterNewAlg
*
* Simple example showing the usage of the new 'counter' facilities
* (with a simple 'producer' type of functional algorithm)
*
* @see GaudiAlgorithm
* @see StatEntity
*
* @author Vanya BELYAEV Ivan.Belyaev@lapp.in2p3.fr
* @date 2008-08-06
*
* @author Andrea Valassi
* @date 2018-02-13
*/
// ============================================================================
class CounterNewAlg : public Gaudi::Functional::Producer<int()>
{
public:
int operator()() const override;
CounterNewAlg( const std::string& name, ISvcLocator* pSvc )
: Producer( name, pSvc, KeyValue( "OutputLocation", "dummy" ) )
{
setProperty( "StatPrint", "true" ).ignore();
}
// copy constructor is disabled
CounterNewAlg( const CounterNewAlg& ) = delete;
// assignement operator is disabled
CounterNewAlg& operator=( const CounterNewAlg& ) = delete;
private:
// counters
mutable StatEntity m_assign_counter{this, "assign"};
mutable StatEntity m_eff_counter{this, "eff"};
mutable StatEntity m_executed_counter{this, "executed"};
mutable StatEntity m_G_counter{this, "G"};
mutable StatEntity m_g2_counter{this, "g2"};
mutable StatEntity m_gauss_counter{this, "gauss"};
mutable StatEntity m_Gneg_counter{this, "Gneg"};
mutable StatEntity m_Gpos_counter{this, "Gpos"};
mutable StatEntity m_NG_counter{this, "NG"};
};
// ============================================================================
// ============================================================================
DECLARE_COMPONENT( CounterNewAlg )
// ============================================================================
// ============================================================================
int CounterNewAlg::operator()() const
{
// count overall number of executions:
++m_executed_counter;
Rndm::Numbers gauss( randSvc(), Rndm::Gauss( 0.0, 1.0 ) );
Rndm::Numbers poisson( randSvc(), Rndm::Poisson( 5.0 ) );
// 'accumulate' gauss
const double value = gauss();
m_gauss_counter += value;
m_g2_counter += value * value;
( 0 < value ) ? ++m_Gpos_counter : ++m_Gneg_counter;
StatEntity& stat1 = m_NG_counter;
StatEntity& stat2 = m_G_counter;
const int num = (int)poisson();
for ( int i = 0; i < num; ++i ) {
stat1++;
stat2 += gauss();
}
// assignement
m_assign_counter = value;
// counter of efficiency
m_eff_counter += ( 0 < value );
// print the statistics every 1000 events
StatEntity& executed = m_executed_counter;
const int print = (int)executed.flag();
if ( 0 == print % 1000 ) {
info() << " Event number " << print << endmsg;
printStat();
info() << " Efficiency (binomial counter: \"eff\"): (" << m_eff_counter.eff() * 100.0 << " +- "
<< m_eff_counter.effErr() * 100.0 << ")%" << endmsg;
}
return 42;
}
// ============================================================================
#include "GaudiKernel/Counters.h"
#include <iostream>
using namespace Gaudi::Accumulators;
int main()
{
// Testing Buffer moving
{
Counter<double, atomicity::full> c;
c += 3.5;
c += 1.2;
{
auto buf = c.buffer();
buf += 7.2; // buf1 internal count = 1
decltype( buf ) buf2( std::move( buf ) );
buf2 += 3.5; // buf3 internal count = 2
decltype( buf ) buf3 = std::move( buf2 );
buf3 += 3.7; // buf3 internal count = 3
}
std::cout << c << std::endl; // should output 5
}
// Testing counter resets
{
SigmaCounter<> sig;
sig += 3;
sig += 5;
sig += 6;
std::cout << sig << std::endl;
sig.reset();
std::cout << sig << std::endl;
}
// Testing AveragingCounter
{
AveragingCounter<> avg;
avg += 3;
avg += 5;
avg += 6;
std::cout << avg << std::endl;
}
// Testing SigmaCounter
{
SigmaCounter<> sig;
sig += 3;
sig += 5;
sig += 6;
std::cout << sig << std::endl;
}
// Testing AveragingCounter wiht buffering
{
AveragingCounter<float, atomicity::full> avg2;
{
auto bufAvg = avg2.buffer();
for ( int i = 0; i < 1000; i++ ) bufAvg += i;
}
std::cout << avg2 << std::endl;
}
// Testing Binomialcounter
{
BinomialCounter<> bin;
bin += false;
bin += true;
bin += true;
bin += false;
bin += false;
std::cout << bin << std::endl;
}
// Testing StatEntity, the backward compatible counter
{
StatEntity se;
se += 3;
se += 5;
se += 6;
std::cout << se << std::endl;
}
// Testing StatEntity, the backward compatible counter with binomial usage
{
StatEntity sb;
sb += 0;
sb += 1;
sb += 1;
sb += 0;
sb += 0;
std::cout << sb << std::endl;
}
// Testing StatEntity, setting directly values
{
StatEntity sb{3, 14, 70, 3, 6};
std::cout << sb << std::endl;
}
// Set of strings
// vector of values (extended binomial ?)
}
<?xml version="1.0" ?><!DOCTYPE extension PUBLIC '-//QM/2.3/Extension//EN' 'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
<extension class="GaudiTest.GaudiExeTest" kind="test">
<argument name="args"><set><text>$GAUDIEXAMPLESROOT/options/CounterNewEx.opts</text></set></argument>
<argument name="use_temp_dir"><enumeral>true</enumeral></argument>
<argument name="reference"><text>refs/CounterNewEx.ref</text></argument>
</extension>
<?xml version="1.0" ?><!DOCTYPE extension PUBLIC '-//QM/2.3/Extension//EN' 'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
<extension class="GaudiTest.GaudiExeTest" kind="test">
<argument name="program"><text>countersUnitTest</text></argument>
<argument name="reference"><text>refs/CountersUnitTest.ref</text></argument>
</extension>
......@@ -272,7 +272,6 @@ SimpleHistos DEBUG Booked 1D Histogram : ID='test1' Path=SimpleHistos Tit
SimpleHistos INFO GaudiHistoAlgorithm:: Filling Histograms...... Please be patient !
Histos2 WARNING Gaudi::Examples::HistoProps:: Cannot generate automatic literal ID from an empty title ! Using numeric ID instead for histogram ID
HistogramDataSvc DEBUG Redefine the parameters for the histogram 'Histos2/2' to be ('TEST2',-100,200,100)
ChronoStatSvc INFO Number of skipped events for MemStat-1
ApplicationMgr INFO Application Manager Stopped successfully
SimpleHistos SUCCESS Booked 30 Histogram(s) : 1D=10 2D=5 3D=3 1DProf=9 2DProf=3
SimpleHistos SUCCESS List of booked 1D histograms in directory "SimpleHistos" :-
......
......@@ -36,7 +36,6 @@ ApplicationMgr SUCCESS
====================================================================================================================================
ApplicationMgr INFO Application Manager Configured successfully
StatusCodeSvc INFO initialize
ChronoStatSvc INFO Number of skipped events for MemStat-1
RndmGenSvc.Engine INFO Generator engine type:CLHEP::RanluxEngine
RndmGenSvc.Engine INFO Current Seed:1234567 Luxury:3
RndmGenSvc INFO Using Random engine:HepRndm::Engine<CLHEP::RanluxEngine>
......
......@@ -17,7 +17,6 @@ RndmGenSvc.Engine INFO Generator engine type:CLHEP::RanluxEngine
RndmGenSvc.Engine INFO Current Seed:1234567 Luxury:3
RndmGenSvc INFO Using Random engine:HepRndm::Engine<CLHEP::RanluxEngine>
TIMER.TIMER INFO This machine has a speed about 2.63 times the speed of a 2.8 GHz Xeon.
ChronoStatSvc INFO Number of skipped events for MemStat-1
ParentAlg INFO creating sub-algorithms....
SubAlg1 INFO initializing....
SubAlg2 INFO initializing....
......
JobOptionsSvc INFO
//GP:================================================================================
//GP: include "../options/CounterNewEx.opts" (0,0)
ApplicationMgr.ExtSvc += [ "RndmGenSvc" ] ; //GP: (1,1)
ApplicationMgr.TopAlg = [ "CounterAlg/Counter" ] ; //GP: (1,1)
MessageSvc.OutputLevel = 3; //GP: (1,1)
ApplicationMgr.EvtMax = 5400; //GP: (1,1)
ApplicationMgr.EvtSel = "NONE"; //GP: (1,1)
//GP: end "../options/CounterNewEx.opts" (27,1)
//GP:================================================================================
JobOptionsSvc INFO Job options successfully read in from ../options/CounterNewEx.opts
ApplicationMgr SUCCESS
====================================================================================================================================
Welcome to ApplicationMgr $Revision: 1.31 $
running on pclhcb55.cern.ch on Wed Jun 4 15:23:35 2008
====================================================================================================================================
ApplicationMgr INFO Application Manager Configured successfully
RndmGenSvc.Engine INFO Generator engine type:CLHEP::RanluxEngine
RndmGenSvc.Engine INFO Current Seed:1234567 Luxury:3
RndmGenSvc INFO Using Random engine:HepRndm::Engine<CLHEP::RanluxEngine>
EventLoopMgr WARNING Unable to locate service "EventSelector"
EventLoopMgr WARNING No events will be processed from external input.
HistogramPersis...WARNING Histograms saving not required.
ApplicationMgr INFO Application Manager Initialized successfully
ApplicationMgr INFO Application Manager Started successfully
Counter INFO Event number 1000
Counter SUCCESS Number of counters : 9
| Counter | # | sum | mean/eff^* | rms/err^* | min | max |
| "G" | 4887 | -82.31362 | -0.016843 | 0.97934 | -3.8263 | 3.4606 |
| "Gneg" | 502 | 502 | 1.0000 | 0.0000 | 1.0000 | 1.0000 |
| "Gpos" | 498 | 498 | 1.0000 | 0.0000 | 1.0000 | 1.0000 |
| "NG" | 4887 | 4887 | 1.0000 | 0.0000 | 1.0000 | 1.0000 |
| "assign" | 1 | -0.4383993 | -0.43840 | 0.0000 | -0.43840 | -0.43840 |
|*"eff" | 1000 | 498 |( 49.8000 +- 1.58113 )%| ------- | ------- |
| "executed" | 1000 | 1000 | 1.0000 | 0.0000 | 1.0000 | 1.0000 |
| "g2" | 1000 | 994.1202 | 0.99412 | 1.4171 | 1.1641e-05 | 12.305 |
| "gauss" | 1000 | -24.66952 | -0.024670 | 0.99675 | -3.1296 | 3.5079 |
Counter INFO Efficiency (binomial counter: "eff"): (49.8 +- 1.58113)%
Counter INFO Event number 2000
Counter SUCCESS Number of counters : 9
| Counter | # | sum | mean/eff^* | rms/err^* | min | max |
| "G" | 9802 | -137.399 | -0.014017 | 0.99570 | -3.8836 | 3.7179 |
| "Gneg" | 998 | 998 | 1.0000 | 0.0000 | 1.0000 | 1.0000 |
| "Gpos" | 1002 | 1002 | 1.0000 | 0.0000 | 1.0000 | 1.0000 |
| "NG" | 9802 | 9802 | 1.0000 | 0.0000 | 1.0000 | 1.0000 |
| "assign" | 1 | 0.2301197 | 0.23012 | 0.0000 | 0.23012 | 0.23012 |
|*"eff" | 2000 | 1002 |( 50.1000 +- 1.11803 )%| ------- | ------- |
| "executed" | 2000 | 2000 | 1.0000 | 0.0000 | 1.0000 | 1.0000 |
| "g2" | 2000 | 1915.24 | 0.95762 | 1.3824 | 2.8347e-06 | 12.305 |
| "gauss" | 2000 | 17.20366 | 0.0086018 | 0.97854 | -3.1296 | 3.5079 |
Counter INFO Efficiency (binomial counter: "eff"): (50.1 +- 1.11803)%
Counter INFO Event number 3000
Counter SUCCESS Number of counters : 9
| Counter | # | sum | mean/eff^* | rms/err^* | min | max |
| "G" | 14691 | -162.8489 | -0.011085 | 0.99656 | -3.8836 | 3.7179 |
<