Create a queue for all schedule-able algorithms

The AvalancheScheduler preserves algorithm ordering by only scheduling them when there is an available thread. However, the current implementation has the side-effect of creating scheduler overhead when there are very many algorithms available to schedule, relative to the number of threads. The majority of these schedule attempts will be discarded, causing a repeated cycle of retrieving precedence graph states and then calling promoteToScheduled.

By creating a queue for all algorithms that can be scheduled, the order of execution is preserved in the queue, and all results of evaluating the precedence graph are guaranteed to be used first time. This gives a dramatic improvement to scheduler performance in scenarios where many algorithms can be executed concurrently.

This is illustrated in the attached plot:

The test is conducted using the JO in !865 (merged), and comparison is made to the Gaudi master branch, and to !863 (merged) (scouting).

NB: this MR should be applied as well as !863 (merged)

GaudiHive/src/AlgoExecutionTask.cpp

@@ -20,12 +20,31 @@ namespace Gaudi {
 
 tbb::task* AlgoExecutionTask::execute() {
 
-  IAlgorithm*       ialg      = m_algorithm.get();
-  Gaudi::Algorithm* this_algo = dynamic_cast<Gaudi::Algorithm*>( ialg );
+  // Get queue data
+  std::tuple<unsigned int, int, EventContext*> queuePop;
+  m_scheduler->m_scheduledQueue.pop( queuePop );
+
+  unsigned int  algIndex  = std::get<0>( queuePop );
+  int           slotIndex = std::get<1>( queuePop );
+  std::string   algName   = m_scheduler->index2algname( algIndex );
+  EventContext& evtCtx    = *std::get<2>( queuePop );
+
+  // Get algorithm pointer
+  IAlgorithm* iAlgoPtr = nullptr;
+  StatusCode  getAlgSC( m_scheduler->m_algResourcePool->acquireAlgorithm( algName, iAlgoPtr ) );
+
+  // Retry if the algorithm is unavailable
+  if ( !getAlgSC.isSuccess() ) {
+    m_scheduler->m_scheduledQueue.push( queuePop );
+    this->recycle_as_safe_continuation();
+    return nullptr;
+  }
+
+  Gaudi::Algorithm* this_algo = dynamic_cast<Gaudi::Algorithm*>( iAlgoPtr );
   if ( !this_algo ) { throw GaudiException( "Cast to Algorithm failed!", "AlgoExecutionTask", StatusCode::FAILURE ); }
 
   bool eventfailed = false;
-  Gaudi::Hive::setCurrentContext( m_evtCtx );
+  Gaudi::Hive::setCurrentContext( evtCtx );
 
   // Get the IProperty interface of the ApplicationMgr to pass it to RetCodeGuard
   const SmartIF<IProperty> appmgr( m_serviceLocator );
@@ -47,38 +66,47 @@ tbb::task* AlgoExecutionTask::execute() {
   }
 
   // select the appropriate store
-  this_algo->whiteboard()->selectStore( m_evtCtx.valid() ? m_evtCtx.slot() : 0 ).ignore();
-
+  this_algo->whiteboard()->selectStore( evtCtx.valid() ? evtCtx.slot() : 0 ).ignore();
   StatusCode sc( StatusCode::FAILURE );
   try {
     RetCodeGuard rcg( appmgr, Gaudi::ReturnCode::UnhandledException );
-    sc = m_algorithm->sysExecute( m_evtCtx );
+    sc = iAlgoPtr->sysExecute( evtCtx );
     if ( UNLIKELY( !sc.isSuccess() ) ) {
-      log << MSG::WARNING << "Execution of algorithm " << m_algorithm->name() << " failed" << endmsg;
+      log << MSG::WARNING << "Execution of algorithm " << algName << " failed" << endmsg;
       eventfailed = true;
     }
     rcg.ignore(); // disarm the guard
   } catch ( const GaudiException& Exception ) {
-    log << MSG::FATAL << ".executeEvent(): Exception with tag=" << Exception.tag() << " thrown by "
-        << m_algorithm->name() << endmsg;
+    log << MSG::FATAL << ".executeEvent(): Exception with tag=" << Exception.tag() << " thrown by " << algName
+        << endmsg;
     log << MSG::ERROR << Exception << endmsg;
     eventfailed = true;
   } catch ( const std::exception& Exception ) {
-    log << MSG::FATAL << ".executeEvent(): Standard std::exception thrown by " << m_algorithm->name() << endmsg;
+    log << MSG::FATAL << ".executeEvent(): Standard std::exception thrown by " << algName << endmsg;
     log << MSG::ERROR << Exception.what() << endmsg;
     eventfailed = true;
   } catch ( ... ) {
-    log << MSG::FATAL << ".executeEvent(): UNKNOWN Exception thrown by " << m_algorithm->name() << endmsg;
+    log << MSG::FATAL << ".executeEvent(): UNKNOWN Exception thrown by " << algName << endmsg;
     eventfailed = true;
   }
 
   // DP it is important to propagate the failure of an event.
   // We need to stop execution when this happens so that execute run can
   // then receive the FAILURE
-  m_aess->updateEventStatus( eventfailed, m_evtCtx );
+  m_aess->updateEventStatus( eventfailed, evtCtx );
+
+  // Release algorithm
+  StatusCode releaseAlgSC = m_scheduler->m_algResourcePool->releaseAlgorithm( algName, iAlgoPtr );
+  if ( !releaseAlgSC.isSuccess() ) {
+    log << MSG::ERROR << "[Event " << evtCtx.evt() << ", Slot " << evtCtx.slot() << "] "
+        << "Instance of algorithm " << algName << " could not be properly put back." << endmsg;
+  }
 
-  // update scheduler state
-  m_promote2ExecutedClosure();
+  // Create a lambda to update scheduler state
+  m_scheduler->m_actionsQueue.push(
+      [this, slotIndex, algIndex, eventContext = std::get<2>( queuePop )]() -> StatusCode {
+        return this->m_scheduler->promoteToExecuted( algIndex, slotIndex, eventContext );
+      } );
 
   Gaudi::Hive::setCurrentContextEvt( -1 );