Merge branch 'ISF_TI_arttest' into 'master'

ISF art test and Transform integration

See merge request atlas/athena!38344

Simulation/BeamEffects/python/BeamEffectsAlgConfig.py

@@ -96,7 +96,7 @@ def makeLongBeamspotVertexPositioner(name="LongBeamspotVertexPositioner", **kwar
     return Simulation__LongBeamspotVertexPositioner(name, **kwargs)
 
 
-def BeamEffectsAlgBasicCfg(ConfigFlags, **kwargs):
+def BeamEffectsAlgCfg(ConfigFlags, **kwargs):
     """Return an accumulator and algorithm for beam effects, wihout output"""
     acc = ComponentAccumulator()
     alg = Simulation__BeamEffectsAlg(name="BeamEffectsAlg", **kwargs)
@@ -122,9 +122,9 @@ def BeamEffectsAlgBasicCfg(ConfigFlags, **kwargs):
     return acc
 
 
-def BeamEffectsAlgCfg(ConfigFlags, **kwargs):
+def BeamEffectsAlgOutputCfg(ConfigFlags, **kwargs):
     """Return an accumulator and algorithm for beam effects, with output"""
-    acc = BeamEffectsAlgBasicCfg(ConfigFlags, **kwargs)
+    acc = BeamEffectsAlgCfg(ConfigFlags, **kwargs)
     # Set to write HITS pool file
     alg = acc.getPrimary()
     ItemList = ["McEventCollection#" + alg.OutputMcEventCollection]

Simulation/G4Atlas/G4AtlasApps/python/SimConfigFlags.py

@@ -135,4 +135,7 @@ def createSimConfigFlags():
         return RunToTimestampDict
     scf.addFlag("Sim.RunToTimestampDict", lambda prevFlags: getRunToTimestampDict())
 
+    scf.addFlag("Sim.BeamPipeCut", 100.0)
+    scf.addFlag("Sim.TightMuonStepping", False)
+
     return scf

Simulation/ISF/ISF_Config/python/ISF_MainConfigNew.py

@@ -57,6 +57,10 @@ def Kernel_GenericSimulatorMTCfg(flags, name="ISF_Kernel_GenericSimulatorMT", **
     kwargs.setdefault("InputEvgenCollection", "BeamTruthEvent")
     kwargs.setdefault("OutputTruthCollection", "TruthEvent")
 
+    #Write MetaData container
+    from G4AtlasApps.G4Atlas_MetadataNew import writeSimulationParametersMetadata
+    acc.merge(writeSimulationParametersMetadata(flags))
+ 
     acc.addEventAlgo(CompFactory.ISF.SimKernelMT(name, **kwargs))
     return acc
 

Simulation/SimuJobTransforms/python/ISF_Skeleton.py

@@ -2,6 +2,7 @@
 
 import sys
 from PyJobTransforms.CommonRunArgsToFlags import commonRunArgsToFlags
+from OverlayConfiguration.OverlayHelpers import accFromFragment
 
 # based on https://acode-browser1.usatlas.bnl.gov/lxr/source/athena/Control/AthenaServices/python/Configurables.py#0247
 def EvtIdModifierSvc_add_modifier(svc,
@@ -53,7 +54,7 @@ def defaultSimulationFlags(ConfigFlags):
     ConfigFlags.GeoModel.Align.Dynamic = False
 
     #Frozen showers OFF = 0
-    ConfigFlags.Sim.LArParameterization = 2
+    # ConfigFlags.Sim.LArParameterization = 2
 
 
     #set the detector flags: - all on currently
@@ -158,14 +159,14 @@ def fromRunArgs(runArgs):
     defaultSimulationFlags(ConfigFlags)
 
     # Pre-exec
-    #simFlags error?
-    # if hasattr(runArgs, 'preExec') and runArgs.preExec != 'NONE' and runArgs.preExec:
-    #     for cmd in runArgs.preExec:
-    #         exec(cmd)
+    if hasattr(runArgs, 'preExec') and runArgs.preExec != 'NONE' and runArgs.preExec:
+        for cmd in runArgs.preExec:
+            exec(cmd)
 
     # Pre-include
-    # if hasattr(runArgs, 'preInclude') and runArgs.preInclude:
-    #     raise ValueError('preInclude not supported')
+    if hasattr(runArgs, 'preInclude') and runArgs.preInclude:
+        for fragment in runArgs.preInclude:
+            accFromFragment(fragment, ConfigFlags)
     
     # Lock flags
     ConfigFlags.lock()
@@ -211,21 +212,89 @@ def fromRunArgs(runArgs):
     from TileGeoG4SD.TileGeoG4SDToolConfig import TileGeoG4SDCalcCfg
     cfg.merge(TileGeoG4SDCalcCfg(ConfigFlags))
 
+    #Add to item list 
+    #TODO - make a separate function (combine with G4AtlasAlg one?)
+    ItemList = ["EventInfo#*",
+                "McEventCollection#TruthEvent",
+                "JetCollection#*"]
+
+    if ConfigFlags.Sim.IncludeParentsInG4Event:
+        ItemList += ["McEventCollection#GEN_EVENT"]
+
+    ItemList += ["xAOD::JetContainer#*",
+                 "xAOD::JetAuxContainer#*"]
+
+    if ConfigFlags.Detector.SimulateID:
+        ItemList += ["SiHitCollection#*",
+                     "TRTUncompressedHitCollection#*",
+                     "TrackRecordCollection#CaloEntryLayer"]
+
+    if ConfigFlags.Detector.SimulateITk:
+        ItemList += ["SiHitCollection#*",
+                     "TrackRecordCollection#CaloEntryLayer"]
+
+    if ConfigFlags.Detector.SimulateCalo:
+        ItemList += ["CaloCalibrationHitContainer#*",
+                     "LArHitContainer#*",
+                     "TileHitVector#*",
+                     "TrackRecordCollection#MuonEntryLayer"]
+
+    if ConfigFlags.Detector.SimulateMuon:
+        ItemList += ["RPCSimHitCollection#*",
+                     "TGCSimHitCollection#*",
+                     "MDTSimHitCollection#*",
+                     "TrackRecordCollection#MuonExitLayer"]
+        if ConfigFlags.Detector.GeometryCSC:
+            ItemList += ["CSCSimHitCollection#*"]
+        if ConfigFlags.Detector.GeometrysTGC:
+            ItemList += ["sTGCSimHitCollection#*"]
+        if ConfigFlags.Detector.GeometryMM:
+            ItemList += ["MMSimHitCollection#*"]
+
+    if ConfigFlags.Detector.SimulateLucid:
+        ItemList += ["LUCID_SimHitCollection#*"]
+
+    if ConfigFlags.Detector.SimulateFwdRegion:
+        ItemList += ["SimulationHitCollection#*"]
+
+    if ConfigFlags.Detector.SimulateZDC:
+        ItemList += ["ZDC_SimPixelHit_Collection#*",
+                     "ZDC_SimStripHit_Collection#*"]
+
+    if ConfigFlags.Detector.SimulateALFA:
+        ItemList += ["ALFA_HitCollection#*",
+                     "ALFA_ODHitCollection#*"]
+
+    if ConfigFlags.Detector.SimulateAFP:
+        ItemList += ["AFP_TDSimHitCollection#*",
+                     "AFP_SIDSimHitCollection#*"]
+
+    # TimingAlg
+    ItemList += ["RecoTimingObj#EVNTtoHITS_timings"]
+
+    from OutputStreamAthenaPool.OutputStreamConfig import OutputStreamCfg
+    cfg.merge( OutputStreamCfg(ConfigFlags,"HITS", ItemList=ItemList, disableEventTag=True) )
+
+    # FIXME hack because deduplication is broken
+    PoolAttributes = ["TREE_BRANCH_OFFSETTAB_LEN = '100'"]
+    PoolAttributes += ["DatabaseName = '" + ConfigFlags.Output.HITSFileName + "'; ContainerName = 'TTree=CollectionTree'; TREE_AUTO_FLUSH = '1'"]
+    cfg.getService("AthenaPoolCnvSvc").PoolAttributes += PoolAttributes
+
+
     # Post-include
-    # if hasattr(runArgs, 'postInclude') and runArgs.postInclude:
-    #     # from OverlayConfiguration.OverlayHelpers import accFromFragment
-    #     for fragment in runArgs.postInclude:
-    #         cfg.merge(accFromFragment(fragment, ConfigFlags))
+    if hasattr(runArgs, 'postInclude') and runArgs.postInclude:
+        for fragment in runArgs.postInclude:
+            cfg.merge(accFromFragment(fragment, ConfigFlags))
 
     # Post-exec
-    # if hasattr(runArgs, 'postExec') and runArgs.postExec != 'NONE' and runArgs.postExec:
-    #     for cmd in runArgs.postExec:
-    #         exec(cmd)
+    if hasattr(runArgs, 'postExec') and runArgs.postExec != 'NONE' and runArgs.postExec:
+        for cmd in runArgs.postExec:
+            exec(cmd)
 
     import time
     tic = time.time()
     # Run the final accumulator
-    sc = cfg.run()
+    sc = cfg.run(maxEvents=evtMax)
     log.info("Run ISF_MainConfigNew_Test in " + str(time.time()-tic) + " seconds")
     
     sys.exit(not sc.isSuccess())

Simulation/SimuJobTransforms/python/prepostincludes.py

+"""pre- and post-includes for New style transform integrations
+
+Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+"""
+
+def preInclude_FrozenShowersFCalOnly(ConfigFlags):
+    """based on preInclude.FrozenShowersFCalOnly.py
+    #==============================================================
+    # SimulationJobOptions/preInclude.FrozenShowersFCalOnly.py
+    # - Turns on GFlash shower parametrization for FCAL
+    #==============================================================
+    """
+    ConfigFlags.Sim.LArParameterization=3
+    ConfigFlags.Sim.CalibrationRun = 'Off' #   simFlags.CalibrationRun.set_Off()
+
+def preInclude_BeamPipeKill(ConfigFlags):
+    """based on share/BeamPipeKill.py"""
+    ConfigFlags.Sim.BeamPipeCut = 0.
+    ConfigFlags.Sim.BeamPipeSimMode = "FastSim"
+
+# Note - is this flag used?
+def preInclude_TightMuonStepping(ConfigFlags):
+    """ previously EVNTtoHITS:simFlags.TightMuonStepping=True'"""
+    ConfigFlags.Sim.TightMuonStepping = True

Simulation/Tests/ISF_ValidationMT/test/test_MC16_FullG4_ttbar_OldvsNew.sh

+#!/bin/sh
+#
+# art-description: Run MT and ST simulation outside ISF, reading ttbar events, writing HITS, using MC16 geometry and conditions
+# art-include: master/Athena
+
+# art-type: grid
+# art-output: log.*
+# art-output: test.OLD.HITS.pool.root
+# art-output: test.NEW.HITS.pool.root
+export ATHENA_CORE_NUMBER=8
+
+Sim_tf.py \
+--CA \
+--multithreaded \
+--inputEVNTFile "/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/SimCoreTests/valid1.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.evgen.EVNT.e4993.EVNT.08166201._000012.pool.root.1" \
+--outputHITSFile "test.NEW.HITS.pool.root" \
+--maxEvents 1 \
+--geometryVersion 'default:ATLAS-R2-2016-01-00-01' \
+--conditionsTag 'default:OFLCOND-MC16-SDR-14' \
+--DataRunNumber '284500' \
+--physicsList 'FTFP_BERT_ATL' \
+--truthStrategy 'MC15aPlus' \
+--simulator 'FullG4MT' \
+--postInclude 'PyJobTransforms.prepostincludes.postInclude_UseFrontier' \
+--preInclude 'SimuJobTransforms.prepostincludes.preInclude_BeamPipeKill,SimuJobTransforms.prepostincludes.preInclude_FrozenShowersFCalOnly,SimuJobTransforms.prepostincludes.preInclude_TightMuonStepping' \
+--imf False
+
+rc=$?
+mv log.EVNTtoHITS log.FullG4MTAthenaCA
+echo  "art-result: $rc FullG4MTAthenaCA"
+rc2=-9999
+if [ $rc -eq 0 ]
+then
+    ArtPackage=$1
+    ArtJobName=$2
+    Sim_tf.py \
+    --multithreaded \
+    --inputEVNTFile "/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/SimCoreTests/valid1.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.evgen.EVNT.e4993.EVNT.08166201._000012.pool.root.1" \
+  --outputHITSFile "test.OLD.HITS.pool.root" \
+  --maxEvents 1 \
+  --geometryVersion 'default:ATLAS-R2-2016-01-00-01' \
+  --conditionsTag 'default:OFLCOND-MC16-SDR-14' \
+  --DataRunNumber '284500' \
+  --physicsList 'FTFP_BERT_ATL' \
+  --truthStrategy 'MC15aPlus' \
+  --simulator 'FullG4MT' \
+  --postInclude 'default:PyJobTransforms/UseFrontier.py' \
+  --preInclude 'EVNTtoHITS:SimulationJobOptions/preInclude.BeamPipeKill.py,SimulationJobOptions/preInclude.FrozenShowersFCalOnly.py' \
+  --preExec 'EVNTtoHITS:simFlags.TightMuonStepping=True' \
+  --imf False
+    mv log.EVNTtoHITS log.FullG4MTAthenaOLD
+    rc2=$?
+    
+fi
+echo  "art-result: $rc2 FullG4MTAthenaOLD"
+rc4=-9999
+if [ $rc2 -eq 0 ]
+then
+    acmd.py diff-root test.OLD.HITS.pool.root test.NEW.HITS.pool.root --error-mode resilient --mode=semi-detailed --order-trees --ignore-leaves RecoTimingObj_p1_EVNTtoHITS_timings index_ref
+    rc4=$?
+fi
+echo  "art-result: $rc4 FullG4MT_OLDvsCA"

Tools/PyJobTransforms/python/prepostincludes.py

+"""pre- and post-includes for New style transform integrations
+
+Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+"""
+from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator
+import os
+from AthenaConfiguration.ComponentFactory import CompFactory
+
+def postInclude_UseFrontier(ConfigFlags):
+    """based on share/UseFrontier.py
+    ## @brief Switch database to using FRONTIER, but with a fallback
+    # to DBRelease if FRONTIER_SERVER is undefined (e.g., on HPC)
+
+    # Move from RecJobTransforms to PyJobTransforms to enable use
+    # in simulation ATN and KV jobs
+    # $Id: UseFrontier.py 605683 2014-07-09 17:22:17Z graemes $
+    """
+
+    #check ConfigFlags are locked for postExec
+    if not ConfigFlags.locked():
+        raise RuntimeError('Running a postExec before locking ConfigFlags')
+
+    acc = ComponentAccumulator()
+    if(os.environ.get('FRONTIER_SERVER')):
+        print('UseFrontier.py: Enabling FRONTIER DB access')
+        acc.addService(CompFactory.DBReplicaSvc(COOLSQLiteVetoPattern="DBRelease"))
+    else:
+        print('UseFrontier.py: Using default DB access')
+
+    return acc
\ No newline at end of file