Merge branch 'relocate-g4hive-code' into 'master'

Relocate G4AtlasMT code from G4HiveEx into G4Atlas packages

Closes ATLASSIM-2772

See merge request !4655

Simulation/G4Atlas/G4AtlasApps/share/jobOptions.G4AtlasMT.py

+#
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+#
+
+#
+# Job options file for multi-threaded Geant4 ATLAS detector simulation.
+# Run this by specifying the number of threads on the command line:
+# $ athena --threads=4 G4AtlasApps/jobOptions.G4AtlasMT.py
+#
+
+from AthenaCommon.Logging import log as msg
+
+from AthenaCommon.ConcurrencyFlags import jobproperties as jp
+nThreads = jp.ConcurrencyFlags.NumThreads()
+if (nThreads < 1) :
+   msg.fatal('numThreads must be >0. Did you set the --threads=N option?')
+   sys.exit(AthenaCommon.ExitCodes.CONFIGURATION_ERROR)
+
+# Message stream format
+msgFmt = "% F%40W%S%5W%e%s%7W%R%T %0W%M"
+svcMgr.MessageSvc.Format = msgFmt
+# svcMgr.MessageSvc.useColors = True
+# svcMgr.AthenaHiveEventLoopMgr.OutputLevel = DEBUG
+
+# Make the scheduler dump some information
+from AthenaCommon.AlgScheduler import AlgScheduler
+AlgScheduler.ShowControlFlow( True )
+AlgScheduler.ShowDataDependencies( True )
+
+# Thread pool service and G4 thread initialization
+from GaudiHive.GaudiHiveConf import ThreadPoolSvc
+svcMgr += ThreadPoolSvc("ThreadPoolSvc")
+svcMgr.ThreadPoolSvc.ThreadInitTools = ["G4ThreadInitTool"]
+
+
+# AthenaCommon flags
+from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
+athenaCommonFlags.PoolEvgenInput = ['/afs/cern.ch/atlas/offline/ProdData/15.6.11.3/mu_E200_eta0-60-10000.evgen.pool.root']
+athenaCommonFlags.PoolHitsOutput = "g4hive.hits.pool.root"
+athenaCommonFlags.EvtMax = 500
+
+# Global conditions tag
+from AthenaCommon.GlobalFlags import jobproperties
+jobproperties.Global.ConditionsTag = "OFLCOND-RUN12-SDR-21"
+
+# Detector flags
+from AthenaCommon.DetFlags import DetFlags
+DetFlags.ID_setOn()
+DetFlags.Calo_setOn()
+DetFlags.Muon_setOn()
+DetFlags.Lucid_setOff()
+DetFlags.Truth_setOn()
+
+# Simulation flags
+from G4AtlasApps.SimFlags import simFlags
+simFlags.load_atlas_flags()
+# Use the default layout
+simFlags.SimLayout.set_On()
+# Set the EtaPhi, VertexSpread and VertexRange checks on/off
+simFlags.EventFilter.set_Off()
+# Set the LAr parameterization
+#simFlags.LArParameterization = 2
+# Magnetic field
+simFlags.MagneticField.set_On()
+
+# Debug outputs of user actions
+#CfgGetter.getPublicTool('G4UA::AthenaTrackingActionTool').OutputLevel = DEBUG
+
+# Setup the algorithm sequence
+from AthenaCommon.AlgSequence import AlgSequence
+topSeq = AlgSequence()
+
+# SGInputLoader is a module in SGComps that will do a typeless StoreGate read
+# of data on disk, to preload it in the Whiteboard for other Alorithms to use.
+# It uses the same syntax as Algorithmic dependency declarations.
+from AthenaCommon import CfgMgr
+topSeq += CfgMgr.SGInputLoader(OutputLevel=INFO, ShowEventDump=False)
+topSeq.SGInputLoader.Load = [('McEventCollection', 'StoreGateSvc+GEN_EVENT')]
+
+# Add the beam effects algorithm
+from AthenaCommon.CfgGetter import getAlgorithm
+topSeq += getAlgorithm("BeamEffectsAlg", tryDefaultConfigurable=True)
+
+# Add the (python) G4 simulation service.
+# This will kickstart a lot of simulation setup.
+from G4AtlasApps.PyG4Atlas import PyG4AtlasSvc
+svcMgr += PyG4AtlasSvc()
+
+# Explicitly specify the data-flow dependencies of G4AtlasAlg and StreamHITS.
+# This is done like this because currently our VarHandles do not live in the
+# algorithm but rather in Geant4 components.
+# TODO: make this declaration more automatic
+topSeq.G4AtlasAlg.ExtraInputs =  [('McEventCollection','StoreGateSvc+BeamTruthEvent')]
+topSeq.G4AtlasAlg.ExtraOutputs = [('SiHitCollection','StoreGateSvc+SCT_Hits')]
+topSeq.StreamHITS.ExtraInputs += topSeq.G4AtlasAlg.ExtraOutputs
+
+# Increase verbosity of the output stream
+#topSeq.StreamHITS.OutputLevel = DEBUG
+
+# Disable alg filtering - doesn't work yet in multi-threading
+topSeq.StreamHITS.AcceptAlgs = []
+
+# Override algorithm cloning settings
+for alg in topSeq:
+    if alg.name() != 'StreamHITS':
+        alg.Cardinality = nThreads

Simulation/G4Atlas/G4AtlasTools/src/G4ThreadInitTool.cxx

+/*
+  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+*/
+
+// Main header include
+#include "G4ThreadInitTool.h"
+
+// Geant4 includes
+#include "G4WorkerRunManager.hh"
+#include "G4UImanager.hh"
+#include "G4MTRunManager.hh"
+#include "G4WorkerThread.hh"
+#include "G4UserWorkerThreadInitialization.hh"
+#include "G4RunManager.hh"
+#include "G4VUserActionInitialization.hh"
+#include "G4UserWorkerInitialization.hh"
+#include "G4AutoDelete.hh"
+
+// System includes
+#include <unistd.h>
+#include <sys/syscall.h>
+
+//-----------------------------------------------------------------------------
+// Constructor
+//-----------------------------------------------------------------------------
+G4ThreadInitTool::G4ThreadInitTool(const std::string& type,
+                                   const std::string& name,
+                                   const IInterface* parent)
+  : base_class(type, name, parent),
+    m_nInitThreads(0)
+{}
+
+//-----------------------------------------------------------------------------
+// Worker thread initialization.
+// This code is modeled after G4MTRunManagerKernel::StartThread.
+//-----------------------------------------------------------------------------
+void G4ThreadInitTool::initThread()
+{
+  ATH_MSG_INFO("==> tbb thread started with id: 0x" <<
+               std::hex << pthread_self() << std::dec);
+
+  // Define the G4 worker thread context and setup its cleanup mechanism.
+  auto wThreadContext = new G4WorkerThread;
+  G4AutoDelete::Register(wThreadContext);
+
+  // Assign the thread ID
+  static std::atomic_uint tid(0);
+  wThreadContext->SetThreadId( tid++ );
+  G4Threading::G4SetThreadId( wThreadContext->GetThreadId() );
+
+  // Setup thread-local geometry and physics
+  wThreadContext->BuildGeometryAndPhysicsVector();
+
+  // Retrieve the master thread run manager
+  G4MTRunManager* masterRM = G4MTRunManager::GetMasterRunManager();
+  // Worker thread initialization object
+  const G4UserWorkerThreadInitialization* workerInitializer =
+    masterRM->GetUserWorkerThreadInitialization();
+
+  // Random number setup.
+  // TODO: revisit this once MT AthRNGSvc is available.
+  const CLHEP::HepRandomEngine* masterEngine = masterRM->getMasterRandomEngine();
+  workerInitializer->SetupRNGEngine(masterEngine);
+
+  // Create the thread-local worker run manager (G4AtlasWorkerRunManager)
+  ATH_MSG_INFO("Creating worker RM");
+  G4WorkerRunManager* wrm = workerInitializer->CreateWorkerRunManager();
+  wrm->SetWorkerThread(wThreadContext);
+
+  // Share detector from master with worker.
+  ATH_MSG_INFO("Assigning detector construction");
+  const G4VUserDetectorConstruction* detector =
+    masterRM->GetUserDetectorConstruction();
+  // I don't want to const-cast here, but this is what they do in G4's
+  // StartThread function, so there is likely no alternative.
+  wrm->G4RunManager::SetUserInitialization
+    (const_cast<G4VUserDetectorConstruction*>(detector));
+  // Share physics list from master with worker.
+  const G4VUserPhysicsList* physicslist = masterRM->GetUserPhysicsList();
+  wrm->SetUserInitialization(const_cast<G4VUserPhysicsList*>(physicslist));
+
+  // Build thread-local user actions - NOT CURRENTLY USED.
+  if(masterRM->GetUserActionInitialization()) {
+    masterRM->GetNonConstUserActionInitialization()->Build();
+  }
+
+  // Start user worker initialization
+  if(masterRM->GetUserWorkerInitialization()) {
+    masterRM->GetUserWorkerInitialization()->WorkerStart();
+  }
+
+  // Initialize the worker run manager
+  ATH_MSG_INFO("Initializing worker RM");
+  wrm->Initialize();
+
+  // Copy the UI commands to the worker
+  std::vector<G4String> cmds = masterRM->GetCommandStack();
+  ATH_MSG_INFO (cmds.size() << " commands in UI stack");
+  G4UImanager* uimgr = G4UImanager::GetUIpointer();
+  for(const auto& it : cmds) {
+    ATH_MSG_INFO ("Adding command to worker: " << it);
+    uimgr->ApplyCommand(it);
+  }
+
+  // Atomic increment number of initialized threads
+  m_nInitThreads++;
+
+  ATH_MSG_INFO("==> tbb thread end of initThread with id: 0x" <<
+               std::hex << pthread_self() << std::dec);
+}
+
+//-----------------------------------------------------------------------------
+// Worker thread termination
+//-----------------------------------------------------------------------------
+void G4ThreadInitTool::terminateThread()
+{
+  // Atomic decrement number of initialized threads
+  m_nInitThreads--;
+}

Simulation/G4Atlas/G4AtlasTools/src/G4ThreadInitTool.h

+/*
+  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+*/
+
+#ifndef G4ATLASTOOLS_G4THREADINITTOOL_H
+#define G4ATLASTOOLS_G4THREADINITTOOL_H
+
+#include "AthenaBaseComps/AthAlgTool.h"
+#include "GaudiKernel/IThreadInitTool.h"
+
+#include <string>
+#include <atomic>
+
+/// @class G4ThreadInitTool
+/// @brief A tool which sets up the worker-thread-local workspace for Geant4.
+///
+/// @author Steve Farrell <Steven.Farrell@cern.ch>
+///
+class G4ThreadInitTool : virtual public extends1<AthAlgTool, IThreadInitTool>
+{
+
+  public:
+
+    /// Standard tool constructor
+    G4ThreadInitTool(const std::string&, const std::string&, const IInterface*);
+
+    /// Set up the Geant4 workspace for this worker thread
+    virtual void initThread() override final;
+
+    /// Tear down the Geant4 workspace for this worker thread
+    virtual void terminateThread() override final;
+
+    /// Counter used for barrier mechanism in thread initialization.
+    /// This number needs to be correctly reported to properly ensure
+    /// scheduling of thread-initialization tasks on every thread.
+    virtual unsigned int nInit() const override final {
+      return m_nInitThreads;
+    }
+
+  private:
+
+    /// Counter of threads that have been initialized
+    std::atomic_uint m_nInitThreads;
+
+}; // class G4ThreadInitTool
+
+#endif // G4ATLASTOOLS_G4THREADINITTOOL_H

Simulation/G4Atlas/G4AtlasTools/src/components/G4AtlasTools_entries.cxx

@@ -14,6 +14,7 @@
 #include "../FastSimulationMasterTool.h"
 #include "G4AtlasTools/GlobalFieldManagerTool.h"
 #include "G4AtlasTools/DetectorFieldManagerTool.h"
+#include "../G4ThreadInitTool.h"
 
 DECLARE_TOOL_FACTORY( PhysicsListToolBase )
 DECLARE_TOOL_FACTORY( DetectorGeometryBase )
@@ -29,3 +30,4 @@ DECLARE_TOOL_FACTORY( FastSimulationMasterTool )
 DECLARE_TOOL_FACTORY( AddPhysicsDecayTool )
 DECLARE_TOOL_FACTORY( GlobalFieldManagerTool )
 DECLARE_TOOL_FACTORY( DetectorFieldManagerTool )
+DECLARE_TOOL_FACTORY( G4ThreadInitTool )