diff --git a/Simulation/G4Sim/G4HiveEx/CMakeLists.txt b/Simulation/G4Sim/G4HiveEx/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..5433b70e524aab65f813c2cea6ba549903ab0a54 --- /dev/null +++ b/Simulation/G4Sim/G4HiveEx/CMakeLists.txt @@ -0,0 +1,27 @@ +################################################################################ +# Package: G4HiveEx +################################################################################ + +# Declare the package name: +atlas_subdir( G4HiveEx ) + +# Declare the package's dependencies: +atlas_depends_on_subdirs( PRIVATE + Control/AthenaBaseComps + Control/AthenaKernel + GaudiKernel + Generators/GeneratorObjects ) + +# External dependencies: +find_package( Geant4 ) +find_package( CLHEP ) + +# Component(s) in the package: +atlas_add_component( G4HiveEx + src/*.cxx + src/components/*.cxx + INCLUDE_DIRS ${GEANT4_INCLUDE_DIRS} ${CLHEP_INCLUDE_DIRS} + LINK_LIBRARIES ${GEANT4_LIBRARIES} ${CLHEP_LIBRARIES} AthenaKernel AthenaBaseComps GeneratorObjects ) + +# Install files from the package: +atlas_install_joboptions( share/*.py ) diff --git a/Simulation/G4Sim/G4HiveEx/cmt/requirements b/Simulation/G4Sim/G4HiveEx/cmt/requirements new file mode 100644 index 0000000000000000000000000000000000000000..a42ef04c5168855b1834d3745b31542c71efd375 --- /dev/null +++ b/Simulation/G4Sim/G4HiveEx/cmt/requirements @@ -0,0 +1,16 @@ +package G4HiveEx +# +use AtlasPolicy AtlasPolicy-* + +private +use GaudiInterface GaudiInterface-* External +use AthenaKernel AthenaKernel-* Control +use AthenaBaseComps AthenaBaseComps-* Control +use GeneratorObjects GeneratorObjects-* Generators +use Geant4 Geant4-* External +end_private + +library G4HiveEx *.cxx -s=components *.cxx +apply_pattern component_library + +apply_pattern declare_joboptions files="*.py" diff --git a/Simulation/G4Sim/G4HiveEx/share/G4HiveExOpts.py b/Simulation/G4Sim/G4HiveEx/share/G4HiveExOpts.py new file mode 100644 index 0000000000000000000000000000000000000000..3e27922337ae3afec636720d583d91f4268162cd --- /dev/null +++ b/Simulation/G4Sim/G4HiveEx/share/G4HiveExOpts.py @@ -0,0 +1,208 @@ +# +## G4HiveExOpts.py - an example job options for multi-threaded simulation +## in AthenaHive. This configuration can only be run if Hive is enabled. +## Run it by specifying the number of threads at the command line: +## $ athena --threads=4 ./G4HiveExOpts.py +# + +from AthenaCommon.ConcurrencyFlags import jobproperties as jp +nThreads = jp.ConcurrencyFlags.NumThreads() +if (nThreads < 1) : + from AthenaCommon.Logging import log as msg + msg.fatal('numThreads must be >0. Did you set the --threads=N option?') + sys.exit(AthenaCommon.ExitCodes.CONFIGURATION_ERROR) + +# +## Override Setup for Hive +# +msgFmt = "% F%40W%S%5W%e%s%7W%R%T %0W%M" +svcMgr.MessageSvc.Format = msgFmt +# svcMgr.MessageSvc.useColors = True + +# svcMgr.AthenaHiveEventLoopMgr.OutputLevel = DEBUG +# svcMgr.ForwardSchedulerSvc.OutputLevel = DEBUG + +# +## Override defaults. otherwise these are ALL EQUAL to the number of threads +## set with the command line opt --threads=N +# + +# numStores = 1 +# numAlgsInFlight = 1 + +# svcMgr.EventDataSvc.NSlots = numStores +# svcMgr.ForwardSchedulerSvc.MaxEventsInFlight = numStores +# svcMgr.ForwardSchedulerSvc.MaxAlgosInFlight = numAlgsInFlight + +# Thread pool service and initialization +from GaudiHive.GaudiHiveConf import ThreadPoolSvc +svcMgr += ThreadPoolSvc("ThreadPoolSvc") +svcMgr.ThreadPoolSvc.ThreadInitTools = ["G4InitTool"] + +# Algorithm resource pool +from GaudiHive.GaudiHiveConf import AlgResourcePool +svcMgr += AlgResourcePool( OutputLevel = INFO ); + + +# +## Uncomment following to avoid long waits when segfaulting, +## and add "Root.Stacktrace: no" to your .rootrc file +# +# import ROOT +# ROOT.SetSignalPolicy( ROOT.kSignalFast ) + +## Output threshold (DEBUG, INFO, WARNING, ERROR, FATAL) +#ServiceMgr.MessageSvc.OutputLevel = INFO + + + +###################################################################################### +# +## AthenaCommon flags +from AthenaCommon.AthenaCommonFlags import athenaCommonFlags +athenaCommonFlags.PoolEvgenInput = [ + '/afs/cern.ch/user/z/zmarshal/public/ForSteve/mc15_13TeV.424000.ParticleGun_single_mu_Pt100.evgen.EVNT.e3580/EVNT.04922446._000001.pool.root.1', + #'/afs/cern.ch/atlas/offline/ProdData/16.6.X/16.6.7.Y/ttbar_muplusjets-pythia6-7000.evgen.pool.root', +] + +# Dirty way to set number of events via command line +if 'evtMax' in dir(): pass +else: evtMax = -1 + +# check to see if we're running hybrid mp/mt +nProc = jp.ConcurrencyFlags.NumProcs() +if (nProc > 0) : + + # + ## For MP/Hive we need to set the chunk size + # + + from AthenaCommon.Logging import log as msg + if (evtMax == -1) : + msg.fatal('EvtMax must be >0 for hybrid configuration') + sys.exit(AthenaCommon.ExitCodes.CONFIGURATION_ERROR) + + if ( evtMax % nProc != 0 ) : + msg.warning('EvtMax[%s] is not divisible by nProcs[%s]: ' + + 'MP Workers will not process all requested events', + evtMax, nProc) + + chunkSize = int (evtMax / nProc) + + from AthenaMP.AthenaMPFlags import jobproperties as jps + jps.AthenaMPFlags.ChunkSize = chunkSize + + msg.info('AthenaMP workers will process %s events each', chunkSize) + +athenaCommonFlags.PoolHitsOutput = "atlasG4.hits.pool.root" +athenaCommonFlags.EvtMax = evtMax + +###################################################################################### +# +## Job options for Geant4 ATLAS detector simulations +# + +## Detector flags +from AthenaCommon.DetFlags import DetFlags +DetFlags.ID_setOn() +DetFlags.Calo_setOn() +DetFlags.Muon_setOn() +DetFlags.Lucid_setOff() +DetFlags.Truth_setOn() + +## Global conditions tag +from AthenaCommon.GlobalFlags import jobproperties +jobproperties.Global.ConditionsTag = "OFLCOND-MC12-SIM-00" + +## Simulation flags +from G4AtlasApps.SimFlags import simFlags +from G4AtlasApps import callbacks +simFlags.load_atlas_flags() +#simFlags.RandomSvc = 'AtDSFMTGenSvc' + +## Layout tags: see simFlags.SimLayout for allowed values +## Use the default layout: +simFlags.SimLayout.set_On() + +## Set the EtaPhi, VertexSpread and VertexRange checks on +simFlags.EventFilter.set_Off() + +## Set the LAr parameterization +#simFlags.LArParameterization = 2 + +## No magnetic field +simFlags.MagneticField.set_On() + +## Change the field stepper or use verbose G4 tracking +#from G4AtlasApps import callbacks +#simFlags.InitFunctions.add_function("postInit", callbacks.use_simplerunge_stepper) +#simFlags.InitFunctions.add_function("preInitG4", callbacks.use_verbose_tracking) + +# Debug output +#CfgGetter.getPublicTool('G4UA::AthenaTrackingActionTool').OutputLevel = DEBUG + +# Setup the algorithm sequence +from AthenaCommon.AlgSequence import AlgSequence +topSeq = AlgSequence() + +# Currently, Hive requires an algorithm to load the initial data into the +# whiteboard and kickstart the data dependency chain. This alg must be at the +# front of the AlgSequence. +from AthenaCommon import CfgMgr +topSeq += CfgMgr.SGInputLoader(OutputLevel = INFO, ShowEventDump=False) + +# Dependencies are either specified automatically via VarHandles (the preferred +# way), or via the ExtraInputs/ExtraOutputs properties. Data objects in the +# latter method are specified as vector of tuples as [(ClassID,'key'),...] or +# [('ClassName','key'),...] + +# SGInputLoader is a module in SGComps that will do a typeless StoreGate read +# of data on disk, to preload it in the Whiteboard for other Alorithms to use. +# Is uses the same syntax as Algorithmic dependency declarations +topSeq.SGInputLoader.Load = [('McEventCollection','GEN_EVENT')] + +# Add the beam effects algorithm +from AthenaCommon.CfgGetter import getAlgorithm +topSeq += getAlgorithm("BeamEffectsAlg", tryDefaultConfigurable=True) + +## Add the G4 simulation service +from G4AtlasApps.PyG4Atlas import PyG4AtlasSvc +svcMgr += PyG4AtlasSvc() + +# TODO: make this declaration more automatic +topSeq.G4AtlasAlg.ExtraInputs = [('McEventCollection','BeamTruthEvent')] +topSeq.G4AtlasAlg.ExtraOutputs = [('SiHitCollection','SCT_Hits')] + +topSeq.StreamHITS.ExtraInputs += topSeq.G4AtlasAlg.ExtraOutputs + +# Disable all of the LAr SDs because they are not yet thread-safe +sdMaster = ToolSvc.SensitiveDetectorMasterTool +larSDs = [sd for sd in sdMaster.SensitiveDetectors if sd.name().startswith('LAr')] +for sd in larSDs: sdMaster.SensitiveDetectors.remove(sd) + +# Increase verbosity of the output stream +#topSeq.StreamHITS.OutputLevel = DEBUG + +# Disable alg filtering - doesn't work in multi-threading +topSeq.StreamHITS.AcceptAlgs = [] + +# theAuditorSvc.Auditors = ["ChronoAuditor", "NameAuditor", "AlgContextAuditor"] + +# +## set which Algorithms can be cloned +# + +# set algCardinality = 1 to disable cloning for all Algs +algCardinality = jp.ConcurrencyFlags.NumThreads() + +if (algCardinality != 1): + for alg in topSeq: + name = alg.name() + if name in ["StreamHITS"]: + print 'Disabling cloning/cardinality for', name + # Don't clone these algs + alg.Cardinality = 1 + alg.IsClonable = False + else: + alg.Cardinality = algCardinality + alg.IsClonable = True diff --git a/Simulation/G4Sim/G4HiveEx/src/G4InitTool.cxx b/Simulation/G4Sim/G4HiveEx/src/G4InitTool.cxx new file mode 100644 index 0000000000000000000000000000000000000000..b1073dc7b4ccc57fbe1a1c6420bc78bf04b48ef0 --- /dev/null +++ b/Simulation/G4Sim/G4HiveEx/src/G4InitTool.cxx @@ -0,0 +1,130 @@ +/* + Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +*/ + +#include "unistd.h" +#include "sys/syscall.h" + +#include "G4InitTool.h" + +#include "G4WorkerRunManager.hh" + +#include "G4UImanager.hh" + +#include "G4MTRunManager.hh" +#include "G4WorkerThread.hh" +#include "G4UserWorkerThreadInitialization.hh" +#include "G4RunManager.hh" +#include "G4VUserActionInitialization.hh" +#include "G4UserWorkerInitialization.hh" +#include "G4AutoDelete.hh" + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + +G4InitTool::G4InitTool( const std::string& type, const std::string& name, + const IInterface* parent ) + : base_class(type, name, parent), + m_nInitThreads(0) + +{ +} + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + +void +G4InitTool::initThread() { + + ATH_MSG_INFO ("==> tbb thread started with id: 0x" << std::hex + << pthread_self() << std::dec); + + // The following code is modeled after G4MTRunManagerKernel::StartThread + + // Define the G4 worker thread context + auto wThreadContext = new G4WorkerThread; + G4AutoDelete::Register(wThreadContext); + + // Assign the thread id + // TODO: implement a better solution. +#ifdef SYS_gettid + // WARNING: not portable! + pid_t tid = syscall(SYS_gettid); + wThreadContext->SetThreadId( tid ); +#else + #error "SYS_gettid unavailable on this system" +#endif + //wThreadContext->SetThreadId( pthread_self() ); + G4int thisID = wThreadContext->GetThreadId(); + G4Threading::G4SetThreadId(thisID); + + // Setup up thread-local workspace + wThreadContext->BuildGeometryAndPhysicsVector(); + + // Retrieve the G4AtlasMTRunManager + auto masterRM = G4MTRunManager::GetMasterRunManager(); + // Worker thread initialization + auto workerInitializer = masterRM->GetUserWorkerThreadInitialization(); + + const CLHEP::HepRandomEngine* masterEngine = masterRM->getMasterRandomEngine(); + workerInitializer->SetupRNGEngine(masterEngine); + + // Create the thread-local worker run manager (G4AtlasWorkerRunManager) + ATH_MSG_INFO("Creating worker RM"); + auto wrm = workerInitializer->CreateWorkerRunManager(); + wrm->SetWorkerThread(wThreadContext); + + // Share detector from master with worker. + ATH_MSG_INFO("Assigning detector construction"); + const auto detector = masterRM->GetUserDetectorConstruction(); + wrm->G4RunManager::SetUserInitialization + (const_cast<G4VUserDetectorConstruction*>(detector)); + // Share physics list from master with worker. + const auto physicslist = masterRM->GetUserPhysicsList(); + wrm->SetUserInitialization(const_cast<G4VUserPhysicsList*>(physicslist)); + + // Build thread-local user actions - NOT CURRENTLY USED. + if(masterRM->GetUserActionInitialization()) { + masterRM->GetNonConstUserActionInitialization()->Build(); + } + + // Start user worker initialization + if(masterRM->GetUserWorkerInitialization()) { + masterRM->GetUserWorkerInitialization()->WorkerStart(); + } + + // Initialize the worker run manager + ATH_MSG_INFO("Initializing worker RM"); + wrm->Initialize(); + + // Copy the UI commands to the worker + std::vector<G4String> cmds = masterRM->GetCommandStack(); + ATH_MSG_INFO (cmds.size() << " commands in UI stack"); + G4UImanager* uimgr = G4UImanager::GetUIpointer(); + for(const auto& it : cmds) { + ATH_MSG_INFO ("Adding command to worker: " << it); + uimgr->ApplyCommand(it); + } + + // Atomic increment number of initialized threads + m_nInitThreads++; + + ATH_MSG_INFO ("==> tbb thread end of initThread with id: 0x" << std::hex << + pthread_self() << std::dec ); + +} + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + +void +G4InitTool::terminateThread() { + + ATH_MSG_INFO ("terminateThread ==> tbb thread 0x" + << std::hex << pthread_self() << std::dec ); + + // call the run termination method on the G4 side + G4RunManager::GetRunManager()->RunTermination(); + + m_nInitThreads--; + +} diff --git a/Simulation/G4Sim/G4HiveEx/src/G4InitTool.h b/Simulation/G4Sim/G4HiveEx/src/G4InitTool.h new file mode 100644 index 0000000000000000000000000000000000000000..935f358452099c4bfa090c600c8b5e8233b02dd4 --- /dev/null +++ b/Simulation/G4Sim/G4HiveEx/src/G4InitTool.h @@ -0,0 +1,30 @@ +/* + Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +*/ + +#ifndef G4HIVEEX_G4INITTOOL_H +#define G4HIVEEX_G4INITTOOL_H + +#include "AthenaBaseComps/AthAlgTool.h" +#include "GaudiKernel/IThreadInitTool.h" + +#include <string> +#include <atomic> + +class G4InitTool: virtual public extends1<AthAlgTool, IThreadInitTool> { + +public: + G4InitTool( const std::string&, const std::string&, const IInterface* ); + + // the magic method this tool provides + virtual void initThread(); + virtual void terminateThread(); + virtual unsigned int nInit() const { return m_nInitThreads; } + +private: + // Number of threads that have been initialized + std::atomic_uint m_nInitThreads; + +}; + +#endif diff --git a/Simulation/G4Sim/G4HiveEx/src/G4InputLoader.cxx b/Simulation/G4Sim/G4HiveEx/src/G4InputLoader.cxx new file mode 100644 index 0000000000000000000000000000000000000000..95c7cef177e9a19236ebde267d48ec6934d7e67d --- /dev/null +++ b/Simulation/G4Sim/G4HiveEx/src/G4InputLoader.cxx @@ -0,0 +1,42 @@ +/* + Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +*/ + +///////////////////////// -*- C++ -*- ///////////////////////////// +// G4InputLoader.cxx +// Implementation file for class G4InputLoader +/////////////////////////////////////////////////////////////////// + +#include "G4InputLoader.h" + +#include "GaudiKernel/Property.h" + +// #include "EventInfo/EventInfo.h" +#include "GeneratorObjects/McEventCollection.h" +#include "AthenaKernel/errorcheck.h" + +// #include "InDetIdentifier/PixelID.h" +// #include "InDetSimEvent/SiHitCollection.h" +// #include "InDetSimEvent/TRTUncompressedHitCollection.h" +// #include "TrackRecord/TrackRecordCollection.h" + + +G4InputLoader::G4InputLoader( const std::string& name, + ISvcLocator* pSvcLocator ) : + ::AthAlgorithm( name, pSvcLocator ), m_mcTruth("GEN_EVENT") +{ + declareProperty( "McTruth", m_mcTruth="GEN_EVENT" ); +} + + +StatusCode G4InputLoader::execute() +{ + ATH_MSG_DEBUG ("Executing " << name() << "..."); + + const McEventCollection* genEvt = 0; + CHECK(evtStore()->retrieve(genEvt, m_mcTruth)); + + ATH_MSG_DEBUG(evtStore()->dump()); + + return StatusCode::SUCCESS; +} diff --git a/Simulation/G4Sim/G4HiveEx/src/G4InputLoader.h b/Simulation/G4Sim/G4HiveEx/src/G4InputLoader.h new file mode 100644 index 0000000000000000000000000000000000000000..ae64f549ea98a7374c440a5d28630889f57eacbd --- /dev/null +++ b/Simulation/G4Sim/G4HiveEx/src/G4InputLoader.h @@ -0,0 +1,39 @@ +/* + Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +*/ + +///////////////////////// -*- C++ -*- ///////////////////////////// +// G4InputLoader.h +// Header file for class G4InputLoader +// Author: C.Leggett +/////////////////////////////////////////////////////////////////// +#ifndef G4HIVEEX_G4INPUTLOADER_H +#define G4HIVEEX_G4INPUTLOADER_H 1 + +// STL includes +#include <string> + +// FrameWork includes +#include "AthenaBaseComps/AthAlgorithm.h" + + +class G4InputLoader + : public ::AthAlgorithm +{ + public: + + G4InputLoader( const std::string& name, ISvcLocator* pSvcLocator ); + + // virtual StatusCode initialize(); + virtual StatusCode execute(); + // virtual StatusCode finalize(); + + private: + + std::string m_mcTruth; + + +}; + + +#endif diff --git a/Simulation/G4Sim/G4HiveEx/src/components/G4HiveEx_entries.cxx b/Simulation/G4Sim/G4HiveEx/src/components/G4HiveEx_entries.cxx new file mode 100644 index 0000000000000000000000000000000000000000..cae830fc555de5cc11aec03a8891f22bdc7d5ace --- /dev/null +++ b/Simulation/G4Sim/G4HiveEx/src/components/G4HiveEx_entries.cxx @@ -0,0 +1,17 @@ +/* + Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +*/ + +#include "GaudiKernel/DeclareFactoryEntries.h" +#include "../G4InputLoader.h" +#include "../G4InitTool.h" + +DECLARE_ALGORITHM_FACTORY(G4InputLoader) +DECLARE_TOOL_FACTORY( G4InitTool ) + + +DECLARE_FACTORY_ENTRIES(G4HiveEx) { + DECLARE_ALGORITHM(G4InputLoader) + DECLARE_TOOL(G4InitTool) + +} diff --git a/Simulation/G4Sim/G4HiveEx/src/components/G4HiveEx_load.cxx b/Simulation/G4Sim/G4HiveEx/src/components/G4HiveEx_load.cxx new file mode 100644 index 0000000000000000000000000000000000000000..53fc534024333e2db4cba75d7bf55fc5454f3a4e --- /dev/null +++ b/Simulation/G4Sim/G4HiveEx/src/components/G4HiveEx_load.cxx @@ -0,0 +1,8 @@ +/* + Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +*/ + +#include "GaudiKernel/LoadFactoryEntries.h" + +LOAD_FACTORY_ENTRIES(G4HiveEx) +