diff --git a/GaudiExamples/tests/qmtest/refs/ToolHandles.ref b/GaudiExamples/tests/qmtest/refs/ToolHandles.ref index c45f0a6e1dd61c0fc3095a5118b04252b3b58896..b49f8f720f83ca67c646120b926394c0a095dbab 100644 --- a/GaudiExamples/tests/qmtest/refs/ToolHandles.ref +++ b/GaudiExamples/tests/qmtest/refs/ToolHandles.ref @@ -41,6 +41,7 @@ HiveSlimEventLo... INFO 0 events were SKIPed ApplicationMgr INFO Application Manager Stopped successfully HiveSlimEventLo... INFO Histograms converted successfully according to request. AvalancheSchedu... INFO Joining Scheduler thread +ThreadPoolSvc WARNING TBB reports 0 worker threads were used, of the 2 requested ToolSvc INFO Removing all tools created by ToolSvc ApplicationMgr INFO Application Manager Finalized successfully ApplicationMgr INFO Application Manager Terminated successfully diff --git a/GaudiHive/src/AvalancheSchedulerSvc.cpp b/GaudiHive/src/AvalancheSchedulerSvc.cpp index cdf8c4bf5b51c4fe1dc3a4332a06dc84cd2cedb7..63d49f5732a93d169496f515a9ab8348c26f1a18 100644 --- a/GaudiHive/src/AvalancheSchedulerSvc.cpp +++ b/GaudiHive/src/AvalancheSchedulerSvc.cpp @@ -34,11 +34,8 @@ #include "boost/thread.hpp" #include "boost/tokenizer.hpp" // DP waiting for the TBB service -#include "tbb/tbb_stddef.h" -#if TBB_INTERFACE_VERSION_MAJOR < 12 -# include "tbb/task_scheduler_init.h" -#endif // TBB_INTERFACE_VERSION_MAJOR < 12 #include "tbb/task.h" +#include "tbb/tbb_stddef.h" // Instantiation of a static factory class used by clients to create instances of this service DECLARE_COMPONENT( AvalancheSchedulerSvc ) @@ -665,11 +662,7 @@ StatusCode AvalancheSchedulerSvc::iterate() { << thisAlgsStates.sizeOfSubset( AState::DATAREADY ) << ", " << thisAlgsStates.sizeOfSubset( AState::SCHEDULED ) << ", " << std::chrono::high_resolution_clock::now().time_since_epoch().count() << "\n"; auto threads = ( m_threadPoolSize != -1 ) ? std::to_string( m_threadPoolSize ) -#if TBB_INTERFACE_VERSION_MAJOR < 12 - : std::to_string( tbb::task_scheduler_init::default_num_threads() ); -#else : std::to_string( std::thread::hardware_concurrency() ); -#endif // TBB_INTERFACE_VERSION_MAJOR < 12 std::ofstream myfile; myfile.open( "IntraEventFSMOccupancy_" + threads + "T.csv", std::ios::app ); myfile << s.str(); diff --git a/GaudiHive/src/ThreadPoolSvc.cpp b/GaudiHive/src/ThreadPoolSvc.cpp index 432590434aa32b74a74d88fd8b1c3ee9f483186d..90f44b792580c7b97d2950c457c35ccd213f590a 100644 --- a/GaudiHive/src/ThreadPoolSvc.cpp +++ b/GaudiHive/src/ThreadPoolSvc.cpp @@ -103,9 +103,8 @@ StatusCode ThreadPoolSvc::initPool( const int& poolSize ) { return StatusCode::FAILURE; } -#if TBB_INTERFACE_VERSION_MAJOR < 12 - m_tbbSchedInit = std::make_unique<tbb::task_scheduler_init>( m_threadPoolSize + 1 ); -#endif // TBB_INTERFACE_VERSION_MAJOR < 12 + // create an observer for the TBB scheduler, to count threads used + m_thread_counter = new thread_count_observer(); ON_VERBOSE verbose() << "Maximum allowed parallelism before adjusting: " << tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) << endmsg; @@ -117,10 +116,6 @@ StatusCode ThreadPoolSvc::initPool( const int& poolSize ) { Gaudi::Concurrency::ConcurrencyFlags::setNumThreads( m_threadPoolSize ); - // Create the barrier for task synchronization at termination - // (here we increase the number of threads by one to account for calling thread) - m_barrier = std::make_unique<boost::barrier>( m_threadPoolSize + 1 ); - } else { // don't use a thread pool Gaudi::Concurrency::ConcurrencyFlags::setNumThreads( 1 ); @@ -162,12 +157,20 @@ StatusCode ThreadPoolSvc::launchTasks( bool terminate ) { const std::string taskType = terminate ? "termination" : "initialization"; + // Create the barrier for task synchronization at termination + // (here we increase the number of threads by one to account for calling thread) + int const threadsUsed = m_thread_counter->getCount(); + if ( threadsUsed < m_threadPoolSize ) + warning() << "TBB reports " << threadsUsed << " worker threads were used, of the " << m_threadPoolSize + << " requested" << endmsg; + m_barrier = std::make_unique<boost::barrier>( threadsUsed + 1 ); + // If we have a thread pool (via a scheduler), then we want to queue // the tasks in TBB to execute on each thread. if ( tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) > 0 ) { // Create one task for each worker thread in the pool - for ( int i = 0; i < m_threadPoolSize; ++i ) { + for ( int i = 0; i < threadsUsed; ++i ) { ON_DEBUG debug() << "creating ThreadInitTask " << i << endmsg; tbb::task* t = new ( tbb::task::allocate_root() ) diff --git a/GaudiHive/src/ThreadPoolSvc.h b/GaudiHive/src/ThreadPoolSvc.h index b4fd65724870e190ad6408be21aee52673434003..f314c02e7b0a4560ac0256b1d492429c65d5a83d 100644 --- a/GaudiHive/src/ThreadPoolSvc.h +++ b/GaudiHive/src/ThreadPoolSvc.h @@ -18,16 +18,29 @@ #include "boost/thread/barrier.hpp" -#include "tbb/spin_mutex.h" -#if TBB_INTERFACE_VERSION_MAJOR < 12 -# include "tbb/task_scheduler_init.h" -# define TBB_PREVIEW_GLOBAL_CONTROL 1 -#endif // TBB_INTERFACE_VERSION_MAJOR < 12 #include "tbb/global_control.h" +#include "tbb/spin_mutex.h" +#include "tbb/task_scheduler_observer.h" #include <memory> #include <vector> +class thread_count_observer : public tbb::task_scheduler_observer { +public: + int m_count = 0; + + thread_count_observer() { + observe( true ); // activate the observer + } + void on_scheduler_entry( bool worker ) override { + if ( worker ) m_count++; + } + void on_scheduler_exit( bool worker ) override { + if ( worker ) m_count--; + } + int getCount() { return m_count; } +}; + /** @class ThreadPoolSvc * @brief A service which initializes a TBB thread pool. * @@ -77,16 +90,14 @@ private: /// Mutex used to protect the initPool and terminatePool methods. tbb::spin_mutex m_initMutex; -#if TBB_INTERFACE_VERSION_MAJOR < 12 - /// TBB task scheduler initializer - std::unique_ptr<tbb::task_scheduler_init> m_tbbSchedInit; -#endif // TBB_INTERFACE_VERSION_MAJOR < 12 - /// Barrier used to synchronization thread init tasks std::unique_ptr<boost::barrier> m_barrier; /// TBB global control parameter std::unique_ptr<tbb::global_control> m_tbbgc; + + /// TBB scheduler observer to count threads + thread_count_observer* m_thread_counter; }; #endif