From abcac5207f28c370efe662951aa3acc04889874b Mon Sep 17 00:00:00 2001 From: Martina Javurkova Date: Fri, 17 Dec 2021 12:29:23 +0100 Subject: [PATCH 1/3] Adding a common Overlay skeleton to FastChain transform (and removing the obsolete OverlayPool skeleton), and adding new ART tests running FastChain with MCOverlay for MC16a and MC20a --- .../test_fastchain_MCOverlay_mc16a_ttbar.sh | 29 +++++++++++++++++++ .../test_fastchain_MCOverlay_mc20a_ttbar.sh | 26 +++++++++++++++++ .../scripts/FastChain_tf.py | 24 ++++++++++----- 3 files changed, 71 insertions(+), 8 deletions(-) create mode 100755 Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc16a_ttbar.sh create mode 100755 Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc20a_ttbar.sh diff --git a/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc16a_ttbar.sh b/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc16a_ttbar.sh new file mode 100755 index 00000000000..92e8020c050 --- /dev/null +++ b/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc16a_ttbar.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# +# art-description: Run FastChain with MC+MC Overlay for MC16a, ttbar +# art-type: grid +# art-include: master/Athena + +# art-output: *.root +# art-output: log.* +# art-output: mem.summary.* +# art-output: mem.full.* +# art-output: runargs.* + +events=25 +HITS_File="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/OverlayMonitoringRTT/valid1.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.simul.HITS.e4993_s3091/HITS.10504490._000425.pool.root.1" +RDO_BKG_File="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/OverlayMonitoringRTT/PileupPremixing/22.0/v4/RDO.merged-pileup-MT.100events.pool.root" + +FastChain_tf.py \ + --inputHITSFile ${HITS_File} \ + --inputRDO_BKGFile ${RDO_BKG_File} \ + --outputRDOFile MC_plus_MC.RDO.pool.root \ + --maxEvents ${events} \ + --skipEvents 10 \ + --digiSeedOffset1 511 \ + --digiSeedOffset2 727 \ + --conditionsTag OFLCOND-MC16-SDR-20-01 \ + --geometryVersion ATLAS-R2-2016-01-00-01 \ + --preExec 'from LArROD.LArRODFlags import larRODFlags;larRODFlags.NumberOfCollisions.set_Value_and_Lock(20);larRODFlags.nSamples.set_Value_and_Lock(4);larRODFlags.doOFCPileupOptimization.set_Value_and_Lock(True);larRODFlags.firstSample.set_Value_and_Lock(0);larRODFlags.useHighestGainAutoCorr.set_Value_and_Lock(True); from LArDigitization.LArDigitizationFlags import jobproperties;jobproperties.LArDigitizationFlags.useEmecIwHighGain.set_Value_and_Lock(False);' \ + --postExec 'all:CfgMgr.MessageSvc().setError+=["HepMcParticleLink"]' \ + --imf False diff --git a/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc20a_ttbar.sh b/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc20a_ttbar.sh new file mode 100755 index 00000000000..bdec6b4718d --- /dev/null +++ b/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc20a_ttbar.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# art-description: Run FastChain with MC+MC Overlay without reco for MC20a, ttbar +# art-type: grid +# art-include: master/Athena + +# art-output: *.root +# art-output: log.* +# art-output: mem.summary.* +# art-output: mem.full.* +# art-output: runargs.* + +events=25 +HITS_File="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/Tier0ChainTests/mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.simul.HITS.e6337_s3681/HITS.25836812._004813.pool.root.1" +RDO_BKG_File="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/OverlayTests/PresampledPileUp/22.0/Run2/large/mc20_13TeV.900149.PG_single_nu_Pt50.digit.RDO.e8307_s3482_s3136_d1713/RDO.26811885._035498.pool.root.1" + +FastChain_tf.py \ + --inputHITSFile ${HITS_File} \ + --inputRDO_BKGFile ${RDO_BKG_File} \ + --outputRDOFile MC_plus_MC.RDO.pool.root \ + --maxEvents ${events} \ + --skipEvents 10 --digiSeedOffset1 511 --digiSeedOffset2 727 \ + --conditionsTag OFLCOND-MC16-SDR-RUN2-09 \ + --geometryVersion ATLAS-R2-2016-01-00-01 \ + --preInclude 'all:Campaigns/MC20a.py' \ + --imf False diff --git a/Tools/FullChainTransforms/scripts/FastChain_tf.py b/Tools/FullChainTransforms/scripts/FastChain_tf.py index 205b8eb9683..85e617b8c3e 100755 --- a/Tools/FullChainTransforms/scripts/FastChain_tf.py +++ b/Tools/FullChainTransforms/scripts/FastChain_tf.py @@ -49,11 +49,17 @@ def getTransform(): substep = 'simdigi', tryDropAndReload = False, perfMonFile = 'ntuple.pmon.gz', inData=['NULL','EVNT'], outData=['RDO','NULL'] )) + # Overlay - from EventOverlayJobTransforms.overlayTransformUtils import addOverlay_PoolArguments - executorSet.add(athenaExecutor(name = 'OverlayPool', skeletonFile = 'EventOverlayJobTransforms/skeleton.OverlayPool_tf.py', - substep = 'overlayPOOL', tryDropAndReload = False, perfMonFile = 'ntuple.pmon.gz', - inData = [('HITS', 'RDO_BKG')], outData = ['RDO', 'RDO_SGNL'])) + from OverlayConfiguration.OverlayTransformHelpers import addOverlayArguments + executorSet.add(athenaExecutor(name='Overlay', + skeletonFile='OverlayConfiguration/skeleton_LegacyOverlay.py', + skeletonCA='OverlayConfiguration.OverlaySkeleton', + substep='overlay', + tryDropAndReload=False, + perfMonFile='ntuple.pmon.gz', + inData=['RDO_BKG', 'BS_SKIM', 'HITS'], + outData=['RDO', 'RDO_SGNL'])) trf = transform(executor = executorSet, description = 'Fast chain ATLAS transform with ISF simulation, digitisation' ' and reconstruction. Inputs can be EVNT, with outputs of RDO, ESD, AOD or DPDs.' @@ -77,8 +83,8 @@ def getTransform(): addTrackRecordArgs(trf.parser) addFastChainTrfArgs(trf.parser) - # Overlay - addOverlay_PoolArguments(trf.parser) + # Overlay arguments + addOverlayArguments(trf.parser) # Add PhysVal addPhysValidationFiles(trf.parser) @@ -108,10 +114,12 @@ def addFastChainTrfArgs(parser): parser.add_argument('--preDigiInclude',type=argFactory(argList),nargs='+', help='preInclude before digitisation step', group='FastChain') - parser.defineArgGroup('EventOverlayPool', 'Event Overlay Options') + parser.defineArgGroup('MCOverlay', 'MC Overlay Options') parser.add_argument('--inputRDO_BKGFile', nargs='+', type=argFactory(argRDOFile, io='input'), - help='Input RAW RDO for pileup overlay', group='EventOverlayPool') + help='Input background RDO for MC+MC overlay', + group='MCOverlay') + if __name__ == '__main__': main() -- GitLab From 4c82112672de5d9b966852b86fd752b4cf8f45ac Mon Sep 17 00:00:00 2001 From: Martina Javurkova Date: Mon, 20 Dec 2021 11:24:50 +0100 Subject: [PATCH 2/3] Adding PHYSVAL step --- .../test_fastchain_MCOverlay_mc16a_ttbar.sh | 48 +++++++++++++++--- .../test_fastchain_MCOverlay_mc20a_ttbar.sh | 49 ++++++++++++++++--- 2 files changed, 84 insertions(+), 13 deletions(-) diff --git a/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc16a_ttbar.sh b/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc16a_ttbar.sh index 92e8020c050..b81caacd1ac 100755 --- a/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc16a_ttbar.sh +++ b/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc16a_ttbar.sh @@ -5,25 +5,59 @@ # art-include: master/Athena # art-output: *.root -# art-output: log.* -# art-output: mem.summary.* -# art-output: mem.full.* -# art-output: runargs.* events=25 HITS_File="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/OverlayMonitoringRTT/valid1.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.simul.HITS.e4993_s3091/HITS.10504490._000425.pool.root.1" RDO_BKG_File="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/OverlayMonitoringRTT/PileupPremixing/22.0/v4/RDO.merged-pileup-MT.100events.pool.root" +RDO_File="MC_plus_MC.RDO.pool.root" +AOD_File="MC_plus_MC.AOD.pool.root" +NTUP_File="MC_plus_MC.NTUP.pool.root" FastChain_tf.py \ --inputHITSFile ${HITS_File} \ --inputRDO_BKGFile ${RDO_BKG_File} \ - --outputRDOFile MC_plus_MC.RDO.pool.root \ + --outputRDOFile ${RDO_File} \ --maxEvents ${events} \ - --skipEvents 10 \ + --skipEvents 0 \ --digiSeedOffset1 511 \ --digiSeedOffset2 727 \ --conditionsTag OFLCOND-MC16-SDR-20-01 \ --geometryVersion ATLAS-R2-2016-01-00-01 \ --preExec 'from LArROD.LArRODFlags import larRODFlags;larRODFlags.NumberOfCollisions.set_Value_and_Lock(20);larRODFlags.nSamples.set_Value_and_Lock(4);larRODFlags.doOFCPileupOptimization.set_Value_and_Lock(True);larRODFlags.firstSample.set_Value_and_Lock(0);larRODFlags.useHighestGainAutoCorr.set_Value_and_Lock(True); from LArDigitization.LArDigitizationFlags import jobproperties;jobproperties.LArDigitizationFlags.useEmecIwHighGain.set_Value_and_Lock(False);' \ - --postExec 'all:CfgMgr.MessageSvc().setError+=["HepMcParticleLink"]' \ + --postExec 'from AthenaCommon.ConfigurationShelve import saveToAscii;saveToAscii("config.txt")' 'all:CfgMgr.MessageSvc().setError+=["HepMcParticleLink"]'\ --imf False + + +rc=$? +echo "art-result: ${rc} HITStoRDO" + + +rc1=999 +rc2=999 +if [ ${rc} -eq 0 ] +then + # Reconstruction + Reco_tf.py --inputRDOFile ${RDO_File} --maxEvents '-1' \ + --autoConfiguration=everything \ + --outputAODFile ${AOD_File} \ + --steering 'doRDO_TRIG' \ + --athenaopts "all:--threads=1" \ + --postExec 'RAWtoESD:from AthenaCommon.ConfigurationShelve import saveToAscii;saveToAscii("RAWtoESD_config.txt")' \ + --imf False + + rc2=$? + if [ ${rc2} -eq 0 ] + then + # NTUP prod. + Reco_tf.py --inputAODFile ${AOD_File} --maxEvents '-1' \ + --outputNTUP_PHYSVALFile ${NTUP_File} \ + --ignoreErrors True \ + --validationFlags 'doInDet' \ + --valid 'True' + rc3=$? + fi +fi + + +echo "art-result: ${rc2} RDOtoAOD" +echo "art-result: ${rc3} AODtoNTUP" diff --git a/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc20a_ttbar.sh b/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc20a_ttbar.sh index bdec6b4718d..ceb22c70b3b 100755 --- a/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc20a_ttbar.sh +++ b/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc20a_ttbar.sh @@ -5,22 +5,59 @@ # art-include: master/Athena # art-output: *.root -# art-output: log.* -# art-output: mem.summary.* -# art-output: mem.full.* -# art-output: runargs.* events=25 HITS_File="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/Tier0ChainTests/mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.simul.HITS.e6337_s3681/HITS.25836812._004813.pool.root.1" RDO_BKG_File="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/OverlayTests/PresampledPileUp/22.0/Run2/large/mc20_13TeV.900149.PG_single_nu_Pt50.digit.RDO.e8307_s3482_s3136_d1713/RDO.26811885._035498.pool.root.1" +RDO_File="MC_plus_MC.RDO.pool.root" +AOD_File="MC_plus_MC.AOD.pool.root" +NTUP_File="MC_plus_MC.NTUP.pool.root" FastChain_tf.py \ --inputHITSFile ${HITS_File} \ --inputRDO_BKGFile ${RDO_BKG_File} \ - --outputRDOFile MC_plus_MC.RDO.pool.root \ + --outputRDOFile ${RDO_File} \ --maxEvents ${events} \ - --skipEvents 10 --digiSeedOffset1 511 --digiSeedOffset2 727 \ + --skipEvents 0 \ + --digiSeedOffset1 511 \ + --digiSeedOffset2 727 \ --conditionsTag OFLCOND-MC16-SDR-RUN2-09 \ --geometryVersion ATLAS-R2-2016-01-00-01 \ --preInclude 'all:Campaigns/MC20a.py' \ + --postExec 'from AthenaCommon.ConfigurationShelve import saveToAscii;saveToAscii("config.txt")' \ --imf False + + +rc=$? +echo "art-result: ${rc} HITStoRDO" + + +rc1=999 +rc2=999 +if [ ${rc} -eq 0 ] +then + # Reconstruction + Reco_tf.py --inputRDOFile ${RDO_File} --maxEvents '-1' \ + --autoConfiguration=everything \ + --outputAODFile ${AOD_File} \ + --steering 'doRDO_TRIG' \ + --athenaopts "all:--threads=1" \ + --postExec 'RAWtoESD:from AthenaCommon.ConfigurationShelve import saveToAscii;saveToAscii("RAWtoESD_config.txt")' \ + --imf False + + rc2=$? + if [ ${rc2} -eq 0 ] + then + # NTUP prod. + Reco_tf.py --inputAODFile ${AOD_File} --maxEvents '-1' \ + --outputNTUP_PHYSVALFile ${NTUP_File} \ + --ignoreErrors True \ + --validationFlags 'doInDet' \ + --valid 'True' + rc3=$? + fi +fi + + +echo "art-result: ${rc2} RDOtoAOD" +echo "art-result: ${rc3} AODtoNTUP" -- GitLab From 66d09e51a776e8e6c5c582708bead72bd7a12706 Mon Sep 17 00:00:00 2001 From: Martina Javurkova Date: Mon, 20 Dec 2021 11:27:17 +0100 Subject: [PATCH 3/3] Adding config files to art-output --- .../test/test_fastchain_MCOverlay_mc16a_ttbar.sh | 2 ++ .../test/test_fastchain_MCOverlay_mc20a_ttbar.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc16a_ttbar.sh b/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc16a_ttbar.sh index b81caacd1ac..0c817c81510 100755 --- a/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc16a_ttbar.sh +++ b/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc16a_ttbar.sh @@ -5,6 +5,8 @@ # art-include: master/Athena # art-output: *.root +# art-output: config.txt +# art-output: RAWtoESD_config.txt events=25 HITS_File="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/OverlayMonitoringRTT/valid1.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.simul.HITS.e4993_s3091/HITS.10504490._000425.pool.root.1" diff --git a/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc20a_ttbar.sh b/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc20a_ttbar.sh index ceb22c70b3b..efceb6407cb 100755 --- a/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc20a_ttbar.sh +++ b/Simulation/FastSimulation/FastChainPileup/test/test_fastchain_MCOverlay_mc20a_ttbar.sh @@ -5,6 +5,8 @@ # art-include: master/Athena # art-output: *.root +# art-output: config.txt +# art-output: RAWtoESD_config.txt events=25 HITS_File="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/Tier0ChainTests/mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.simul.HITS.e6337_s3681/HITS.25836812._004813.pool.root.1" -- GitLab