diff --git a/AtlasTest/CITest/Athena.cmake b/AtlasTest/CITest/Athena.cmake index c3fc97005a6124d3d949abad3b35399ac70b8ee6..bab0df63c8f14bd7818e95d7b778991698ac047e 100644 --- a/AtlasTest/CITest/Athena.cmake +++ b/AtlasTest/CITest/Athena.cmake @@ -132,6 +132,10 @@ atlas_add_citest( DerivationRun3Data_PHYSLITE SCRIPT RunWorkflowTests_Run3.py --CI -d -w Derivation --tag data_PHYSLITE --threads 4 PROPERTIES PROCESSORS 4 ) +atlas_add_citest( DerivationRun3Data_Train + SCRIPT RunWorkflowTests_Run3.py --CI -d -w Derivation --tag data_PHYS_PHYSLITE --threads 4 --no-output-checks + PROPERTIES PROCESSORS 4 ) + atlas_add_citest( DerivationRun3MC_PHYS SCRIPT RunWorkflowTests_Run3.py --CI -d -w Derivation --tag mc_PHYS --threads 4 PROPERTIES PROCESSORS 4 ) @@ -140,6 +144,14 @@ atlas_add_citest( DerivationRun3MC_PHYSLITE SCRIPT RunWorkflowTests_Run3.py --CI -d -w Derivation --tag mc_PHYSLITE --threads 4 PROPERTIES PROCESSORS 4 ) +atlas_add_citest( DerivationRun3MCAF3_PHYS + SCRIPT RunWorkflowTests_Run3.py --CI -d -w Derivation --tag af3_PHYS --threads 4 + PROPERTIES PROCESSORS 4 ) + +atlas_add_citest( DerivationRun3MCAF3_PHYSLITE + SCRIPT RunWorkflowTests_Run3.py --CI -d -w Derivation --tag af3_PHYSLITE --threads 4 + PROPERTIES PROCESSORS 4 ) + atlas_add_citest( RecoRun4MC_DAODPHYS SCRIPT RunWorkflowTests_Run4.py --CI -d -w Derivation -e '--maxEvents 5 --inputAODFile=../../RecoRun4MC/run_q447/myAOD.pool.root' --no-output-checks # go two levels up as the test runs in a subfolder DEPENDS_SUCCESS RecoRun4MC ) diff --git a/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/PHYSLITE.py b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/PHYSLITE.py index dd599c74bd2c4f12a7b7a7d31108b27297bd00d1..7a2b7608a247c27a472db8fa022cf64df9aca469 100755 --- a/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/PHYSLITE.py +++ b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/PHYSLITE.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2023 CERN for the benefit of the ATLAS collaboration +# Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration #!/usr/bin/env python #==================================================================== # DAOD_PHYSLITE.py @@ -21,39 +21,41 @@ def CPAlgorithmsCfg(flags): logPLCPAlgCfg = logging.getLogger('PLCPAlgCfg') logPLCPAlgCfg.info('****************** STARTING PHYSLITE CPAlgorithmsCfg *****************') + forceEGammaFullSimConfig = False + if not flags.Sim.ISF.Simulator.isFullSim(): + logPLCPAlgCfg.warning("Forcing full simulation configuration for EGamma algorithms. This is not recommended for fast simulation but no recommendations available yet.") + forceEGammaFullSimConfig = True from AnalysisAlgorithmsConfig.ConfigFactory import ConfigFactory from AnalysisAlgorithmsConfig.ConfigSequence import ConfigSequence configSeq = ConfigSequence () # create factory object to build block configurations - config = ConfigFactory() - # get function to make configs - makeConfig = config.makeConfig + factory = ConfigFactory() # Set up the systematics loader/handler algorithm: - subConfig = makeConfig ('CommonServices') + subConfig = factory.makeConfig ('CommonServices') subConfig.setOptionValue ('.runSystematics', False) configSeq += subConfig # Create a pile-up analysis config if flags.Input.isMC: # setup config and lumicalc files for pile-up tool - configSeq += makeConfig ('PileupReweighting') + configSeq += factory.makeConfig ('PileupReweighting') # set up the muon analysis algorithm config (must come before electrons and photons to allow FSR collection): logPLCPAlgCfg.info('Do Muons') - subConfig = makeConfig ('Muons', containerName='AnalysisMuons') + subConfig = factory.makeConfig ('Muons', containerName='AnalysisMuons') subConfig.setOptionValue ('.trackSelection', False) configSeq += subConfig - subConfig = makeConfig ('Muons.WorkingPoint', containerName='AnalysisMuons', + subConfig = factory.makeConfig ('Muons.WorkingPoint', containerName='AnalysisMuons', selectionName='loose') subConfig.setOptionValue ('.quality', 'Loose') subConfig.setOptionValue ('.isolation', 'NonIso') configSeq += subConfig - subConfig = makeConfig ('Thinning', containerName='AnalysisMuons') + subConfig = factory.makeConfig ('Thinning', containerName='AnalysisMuons') subConfig.setOptionValue ('.selectionName', 'loose') subConfig.setOptionValue ('.deepCopy', True) subConfig.setOptionValue ('.noUniformSelection', True) @@ -63,39 +65,41 @@ def CPAlgorithmsCfg(flags): logPLCPAlgCfg.info('Do Electrons') - subConfig = makeConfig ('Electrons', containerName='AnalysisElectrons') + subConfig = factory.makeConfig ('Electrons', containerName='AnalysisElectrons') + subConfig.setOptionValue ('.forceFullSimConfig', forceEGammaFullSimConfig) subConfig.setOptionValue ('.trackSelection', False) subConfig.setOptionValue ('.isolationCorrection', True) subConfig.setOptionValue ('.minPt', 0.) configSeq += subConfig - subConfig = makeConfig ('Electrons.WorkingPoint', containerName='AnalysisElectrons', + subConfig = factory.makeConfig ('Electrons.WorkingPoint', containerName='AnalysisElectrons', selectionName='loose') subConfig.setOptionValue ('.likelihoodWP', 'LooseLHElectron') subConfig.setOptionValue ('.isolationWP', 'NonIso') subConfig.setOptionValue ('.doFSRSelection', True) subConfig.setOptionValue ('.noEffSF', True) configSeq += subConfig - subConfig = makeConfig ('Thinning', containerName='AnalysisElectrons') + subConfig = factory.makeConfig ('Thinning', containerName='AnalysisElectrons') subConfig.setOptionValue ('.selectionName', 'loose') subConfig.setOptionValue ('.deepCopy', True) subConfig.setOptionValue ('.noUniformSelection', True) configSeq += subConfig # So SiHit electrons - should come after the standard selection in order to avoid keeping the same electrons twice - subConfig = makeConfig ('Electrons', containerName='AnalysisSiHitElectrons') + subConfig = factory.makeConfig ('Electrons', containerName='AnalysisSiHitElectrons') + subConfig.setOptionValue ('.forceFullSimConfig', forceEGammaFullSimConfig) subConfig.setOptionValue ('.trackSelection', False) subConfig.setOptionValue ('.isolationCorrection', True) subConfig.setOptionValue ('.minPt', 0.) subConfig.setOptionValue ('.postfix', 'SiHit') configSeq += subConfig - subConfig = makeConfig ('Electrons.WorkingPoint', containerName='AnalysisSiHitElectrons', selectionName='SiHits') + subConfig = factory.makeConfig ('Electrons.WorkingPoint', containerName='AnalysisSiHitElectrons', selectionName='SiHits') subConfig.setOptionValue ('.likelihoodWP', 'SiHitElectron') subConfig.setOptionValue ('.isolationWP', 'NonIso') subConfig.setOptionValue ('.doFSRSelection', True) # needed to veto FSR electrons subConfig.setOptionValue ('.noEffSF', True) subConfig.setOptionValue ('.postfix', 'SiHit') configSeq += subConfig - subConfig = makeConfig ('Thinning', containerName='AnalysisSiHitElectrons') + subConfig = factory.makeConfig ('Thinning', containerName='AnalysisSiHitElectrons') subConfig.setOptionValue ('.selectionName', 'SiHits') subConfig.setOptionValue ('.deepCopy', True) subConfig.setOptionValue ('.noUniformSelection', True) @@ -105,11 +109,12 @@ def CPAlgorithmsCfg(flags): logPLCPAlgCfg.info('Do Photons') - subConfig = makeConfig ('Photons', containerName='AnalysisPhotons') + subConfig = factory.makeConfig ('Photons', containerName='AnalysisPhotons') + subConfig.setOptionValue ('.forceFullSimConfig', forceEGammaFullSimConfig) subConfig.setOptionValue ('.recomputeIsEM', False) subConfig.setOptionValue ('.minPt', 0.) configSeq += subConfig - subConfig = makeConfig ('Photons.WorkingPoint', containerName='AnalysisPhotons', + subConfig = factory.makeConfig ('Photons.WorkingPoint', containerName='AnalysisPhotons', selectionName='loose') subConfig.setOptionValue ('.qualityWP', 'Loose') subConfig.setOptionValue ('.isolationWP', 'NonIso') @@ -117,7 +122,7 @@ def CPAlgorithmsCfg(flags): subConfig.setOptionValue ('.recomputeIsEM', False) subConfig.setOptionValue ('.noEffSF', True) configSeq += subConfig - subConfig = makeConfig ('Thinning', containerName='AnalysisPhotons') + subConfig = factory.makeConfig ('Thinning', containerName='AnalysisPhotons') subConfig.setOptionValue ('.selectionName', 'loose') subConfig.setOptionValue ('.deepCopy', True) subConfig.setOptionValue ('.noUniformSelection', True) @@ -127,13 +132,13 @@ def CPAlgorithmsCfg(flags): # set up the tau analysis algorithm config: # Commented for now due to use of public tools - subConfig = makeConfig ('TauJets', containerName='AnalysisTauJets') + subConfig = factory.makeConfig ('TauJets', containerName='AnalysisTauJets') configSeq += subConfig - subConfig = makeConfig ('TauJets.WorkingPoint', containerName='AnalysisTauJets', + subConfig = factory.makeConfig ('TauJets.WorkingPoint', containerName='AnalysisTauJets', selectionName='baseline') subConfig.setOptionValue ('.quality', 'Baseline') configSeq += subConfig - subConfig = makeConfig ('Thinning', containerName='AnalysisTauJets') + subConfig = factory.makeConfig ('Thinning', containerName='AnalysisTauJets') subConfig.setOptionValue ('.selectionName', 'baseline') subConfig.setOptionValue ('.deepCopy', True) subConfig.setOptionValue ('.noUniformSelection', True) @@ -141,24 +146,24 @@ def CPAlgorithmsCfg(flags): # set up the jet analysis algorithm config: jetContainer = 'AntiKt4EMPFlowJets' - subConfig = makeConfig ('Jets', containerName='AnalysisJets', + subConfig = factory.makeConfig ('Jets', containerName='AnalysisJets', jetCollection=jetContainer) subConfig.setOptionValue ('.runFJvtUpdate', False) subConfig.setOptionValue ('.runFJvtSelection', False) subConfig.setOptionValue ('.runJvtSelection', False) configSeq += subConfig - subConfig = makeConfig ('Thinning', containerName='AnalysisJets') + subConfig = factory.makeConfig ('Thinning', containerName='AnalysisJets') subConfig.setOptionValue ('.deepCopy', True) subConfig.setOptionValue ('.noUniformSelection', True) configSeq += subConfig largeRjetContainer='AntiKt10UFOCSSKSoftDropBeta100Zcut10Jets' - subConfig = makeConfig ('Jets', containerName='AnalysisLargeRJets', + subConfig = factory.makeConfig ('Jets', containerName='AnalysisLargeRJets', jetCollection=largeRjetContainer) subConfig.setOptionValue ('.runGhostMuonAssociation', False) subConfig.setOptionValue ('.postfix', 'largeR_jets' ) configSeq += subConfig - subConfig = makeConfig ('Thinning', containerName='AnalysisLargeRJets') + subConfig = factory.makeConfig ('Thinning', containerName='AnalysisLargeRJets') subConfig.setOptionValue ('.deepCopy', True) subConfig.setOptionValue ('.noUniformSelection', True) configSeq += subConfig diff --git a/Tools/PROCTools/data/af3_PHYS_Run2_DAOD_PHYS_diff-interest-list.txt b/Tools/PROCTools/data/af3_PHYS_Run2_DAOD_PHYS_diff-interest-list.txt new file mode 120000 index 0000000000000000000000000000000000000000..e78ed44af611ddcb2272d5102c48e43a201ad95a --- /dev/null +++ b/Tools/PROCTools/data/af3_PHYS_Run2_DAOD_PHYS_diff-interest-list.txt @@ -0,0 +1 @@ +mc_PHYS_Run2_DAOD_PHYS_diff-interest-list.txt \ No newline at end of file diff --git a/Tools/PROCTools/data/af3_PHYS_Run3_DAOD_PHYS_diff-interest-list.txt b/Tools/PROCTools/data/af3_PHYS_Run3_DAOD_PHYS_diff-interest-list.txt new file mode 120000 index 0000000000000000000000000000000000000000..e78ed44af611ddcb2272d5102c48e43a201ad95a --- /dev/null +++ b/Tools/PROCTools/data/af3_PHYS_Run3_DAOD_PHYS_diff-interest-list.txt @@ -0,0 +1 @@ +mc_PHYS_Run2_DAOD_PHYS_diff-interest-list.txt \ No newline at end of file diff --git a/Tools/WorkflowTestRunner/python/Inputs.py b/Tools/WorkflowTestRunner/python/Inputs.py index 6e360338705907113be60bbd45e21f8eb92fc332..3708e78c1cde12ae43886275ed8f45902dd66003 100644 --- a/Tools/WorkflowTestRunner/python/Inputs.py +++ b/Tools/WorkflowTestRunner/python/Inputs.py @@ -58,5 +58,6 @@ input_AOD = { WorkflowRun.Run3: { "data": "/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/CampaignInputs/data22/AOD/data22_13p6TeV.00431906.physics_Main.merge.AOD.r13928_p5279/1000events.AOD.30220215._001367.pool.root.1", "mc": "/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/CampaignInputs/mc21/AOD/mc21_13p6TeV.601229.PhPy8EG_A14_ttbar_hdamp258p75_SingleLep.recon.AOD.e8453_s3873_r13829/1000events.AOD.29787656._000153.pool.root.1", + "af3": "/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/CampaignInputs/mc23/AOD/mc23_13p6TeV.601229.PhPy8EG_A14_ttbar_hdamp258p75_SingleLep.recon.AOD.e8514_a911_s4114_r15224/1000events.AOD.36216231._001858.pool.root.1", }, } diff --git a/Tools/WorkflowTestRunner/python/References.py b/Tools/WorkflowTestRunner/python/References.py index bf7a8c1a8c7840449b296a1a0844fbccfc8cacbd..9245b24cf2048ce0db4cd9b872f21bfd51ab919c 100644 --- a/Tools/WorkflowTestRunner/python/References.py +++ b/Tools/WorkflowTestRunner/python/References.py @@ -32,4 +32,5 @@ references_map = { "data_PHYS_Run3": "v15", "mc_PHYS_Run2": "v19", "mc_PHYS_Run3": "v19", + "af3_PHYS_Run3": "v1", } diff --git a/Tools/WorkflowTestRunner/python/StandardTests.py b/Tools/WorkflowTestRunner/python/StandardTests.py index 9ea62fed64a36fe13f5de3ef87471982f6d923ad..6e2fd7fca64cdaba523ff9b12814b1d7660e70d7 100644 --- a/Tools/WorkflowTestRunner/python/StandardTests.py +++ b/Tools/WorkflowTestRunner/python/StandardTests.py @@ -174,7 +174,7 @@ class DerivationTest(WorkflowTest): def __init__(self, ID: str, run: WorkflowRun, type: WorkflowType, steps: List[str], setup: TestSetup, extra_args: str = "") -> None: test_def = ID.split("_") data_type = test_def[0].lower() - format = test_def[1].upper() + formats = [format.upper() for format in test_def[1:-1]] threads = 0 if setup.custom_threads is not None: @@ -186,14 +186,15 @@ class DerivationTest(WorkflowTest): flush = 80 extra_args += f" --maxEvents {events}" - extra_args += f" --preExec 'ConfigFlags.Output.TreeAutoFlush={{\"DAOD_{format}\": {flush}}}'" + format_flush = ", ".join([f"\"DAOD_{format}\": {flush}" for format in formats]) + extra_args += f" --preExec 'flags.Output.TreeAutoFlush={{{format_flush}}}'" if "inputAODFile" not in extra_args: extra_args += f" --inputAODFile {input_AOD[run][data_type]}" # could also use p5503 self.command = \ (f"ATHENA_CORE_NUMBER={threads} Derivation_tf.py --CA" - f" --formats {format}" + f" --formats {' '.join(formats)}" " --multiprocess --multithreadedFileValidation True" " --athenaMPMergeTargetSize 'DAOD_*:0'" " --sharedWriter True" @@ -203,12 +204,11 @@ class DerivationTest(WorkflowTest): # skip performance checks for now self.skip_performance_checks = True - enable_checks = True if format == "PHYS" else False - if enable_checks: - self.output_checks = [ - FrozenTier0PolicyCheck(setup, f"DAOD_{format}", 10), - MetadataCheck(setup, f"DAOD_{format}"), - ] + self.output_checks = [] + for format in formats: + if format == "PHYS": + self.output_checks.append(FrozenTier0PolicyCheck(setup, f"DAOD_{format}", 10)) + self.output_checks.append(MetadataCheck(setup, f"DAOD_{format}")) super().__init__(ID, run, type, steps, setup)