Merge branch 'trig-flags-edmver' into 'master'

Improve TriggerConfigFlags.EDMDecodingVersion auto-configuration

See merge request !38381

Trigger/TrigValidation/TrigValTools/share/TrigValInputs.json

@@ -20,6 +20,13 @@
             "/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigAnalysisTest/data12_8TeV.00209109.physics_JetTauEtmiss.merge.RAW._lb0186._SFO-1._0001.1"
         ]
     },
+    "data_run3": {
+        "source": "data",
+        "format": "BS",
+        "paths": [
+            "/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/data20_test/data_test.00374861.physics_Main.daq.RAW._lb0004._SFO-1._0001.data"
+        ]
+    },
     "data_cos": {
         "source": "data",
         "format": "BS",

Trigger/TriggerCommon/TriggerJobOpts/CMakeLists.txt

@@ -15,3 +15,7 @@ atlas_add_test( TriggerConfigFlagsTest
 atlas_add_test( TriggerConfigTest
    SCRIPT python -m TriggerJobOpts.TriggerConfig
    POST_EXEC_SCRIPT nopost.sh )
+
+atlas_add_test( TriggerConfigFlags_AutoConfTest
+   SCRIPT test_TriggerFlags_autoconf.py
+   POST_EXEC_SCRIPT nopost.sh )

Trigger/TriggerCommon/TriggerJobOpts/python/TriggerConfigFlags.py

@@ -46,69 +46,90 @@ def createTriggerFlags():
     # Enable calorimeters
     flags.addFlag('Trigger.doCalo', True)
 
-    # if 1, Run1 decoding version is set; if 2, Run2; if 3, Run 3 
+    # if 1, Run1 decoding version is set; if 2, Run2; if 3, Run 3
     def EDMDecodingVersion(flags):
-        log.debug("Attempting to determine EDMDecodingVersion.")
-        version = 3
+        '''
+        Determine Trigger EDM version based on the input file. For ByteStream, Run-3 EDM is indicated
+        by HLT ROD version > 1.0, for both Runs 1 and 2 the HLT ROD version was 0.0 and the run number
+        is used to disambiguate between them. For POOL inputs, the EDM version is determined based on
+        finding a characteristic HLT navigation collection in the file.
+        '''
+        _log = logging.getLogger('TriggerConfigFlags.EDMDecodingVersion')
+        _log.debug("Attempting to determine EDMDecodingVersion")
+        default_version = 3
         if flags.Input.Format=="BS":
-            log.debug("EDMDecodingVersion: Input format is ByteStream")
+            _log.debug("Input format is ByteStream")
             inputFileName = flags.Input.Files[0]
             if not inputFileName and flags.Common.isOnline():
-                log.debug("EDMDecodingVersion: Online reconstruction, no input file. Return default version, i.e. AthenaMT.")
-                return version
+                _log.info("Online reconstruction, no input file. Return default EDMDecodingVersion=%d", default_version)
+                return default_version
 
-            log.debug("EDMDecodingVersion: Checking ROD version.")
+            _log.debug("Checking ROD version")
             import eformat
             from libpyeformat_helper import SubDetector
             bs = eformat.istream(inputFileName)
 
-            rodVersionM = -1                                                                                                
-            rodVersionL = -1                                                                                                
-            # Find the first HLT ROBFragment in the first event                                                             
-            for robf in bs[0]:                                                                                              
-                if robf.rob_source_id().subdetector_id()==SubDetector.TDAQ_HLT:                                             
-                    rodVersionM = robf.rod_minor_version() >> 8                                                             
-                    rodVersionL = robf.rod_minor_version() & 0xFF                                                           
-                    log.debug("EDMDecodingVersion: HLT ROD minor version from input file is {:d}.{:d}".format(rodVersionM, rodVersionL))
-                    break                                                                                                   
-
-            if rodVersionM >= 1:
-                version = 3
-                return version
-            log.info("EDMDecodingVersion: Could not determine ROD version -- falling back to run-number-based determination")
-
-            # Use run number to determine decoding version
+            rodVersionM = -1
+            rodVersionL = -1
+            # Find the first HLT ROBFragment in the first event
+            for robf in bs[0]:
+                if robf.rob_source_id().subdetector_id()==SubDetector.TDAQ_HLT:
+                    rodVersionM = robf.rod_minor_version() >> 8
+                    rodVersionL = robf.rod_minor_version() & 0xFF
+                    _log.debug("HLT ROD minor version from input file is %d.%d", rodVersionM, rodVersionL)
+                    break
+
+            # Case 1: failed to read ROD version
+            if rodVersionM < 0 or rodVersionL < 0:
+                _log.warning("Cannot determine HLT ROD version from input file, falling back to run-number-based decision")
+            # Case 2: ROD version indicating Run 3
+            elif rodVersionM >= 1:
+                _log.info("Determined EDMDecodingVersion to be 3, because running on BS file with HLT ROD version %d.%d",
+                          rodVersionM, rodVersionL)
+                return 3
+            # Case 3: ROD version indicating Run 1 or 2 - use run number to disambiguate
             runNumber = flags.Input.RunNumber[0]
-            log.debug("EDMDecodingVersion: Read run number {}.".format(runNumber))
+            _log.debug("Read run number %s", runNumber)
 
             boundary_run12 = 230000
             boundary_run23 = 368000
 
-            if runNumber <= 0:
-                log.warning("EDMDecodingVersion: Cannot determine decoding version because run number {} is invalid. Leaving the default version.".format(runNumber))
+            if not runNumber or runNumber <= 0:
+                _log.warning("Cannot determine EDM version because run number %s is invalid. "
+                             "Return default EDMDecodingVersion=%d", runNumber, default_version)
+                return default_version
             elif runNumber < boundary_run12:
                 # Run-1 data
-                version = 1
+                _log.info("Determined EDMDecodingVersion to be 1 based on BS file run number (runNumber < %d)",
+                          boundary_run12)
+                return 1
             elif runNumber < boundary_run23:
                 # Run-2 data
-                version = 2
+                _log.info("Determined EDMDecodingVersion to be 2 based on BS file run number (%d < runNumber < %d)",
+                          boundary_run12, boundary_run23)
+                return 2
             else:
                 # Run-3 data
-                version = 3
+                _log.info("Determined EDMDecodingVersion to be 3 based on BS file run number (runNumber > %d)",
+                          boundary_run23)
+                return 3
         else:
-            log.debug("EDMDecodingVersion: Input format is POOL -- determine from input file collections.")
-            # POOL files: decide based on HLT output type present in file
+            # POOL files: decide based on HLT output type present in the file
+            _log.debug("EDMDecodingVersion: Input format is POOL -- determine from input file collections")
             if "HLTResult_EF" in flags.Input.Collections:
-                version = 1
+                _log.info("Determined EDMDecodingVersion to be 1, because HLTResult_EF found in POOL file")
+                return 1
             elif "TrigNavigation" in flags.Input.Collections:
-                version = 2
+                _log.info("Determined EDMDecodingVersion to be 2, because TrigNavigation found in POOL file")
+                return 2
             elif "HLTNav_Summary" in flags.Input.Collections:
-                version = 3
-            elif flags.Input.Format == "POOL":
-                # If running Trigger on RDO input (without previous trigger result), choose Run-3
-                version = 3
-        log.info("Determined EDMDecodingVersion to be {}.".format({1:"Run 1", 2:"Run 2", 3:"AthenaMT"}[version]))
-        return version
+                _log.info("Determined EDMDecodingVersion to be 3, because HLTNav_Summary found in POOL file")
+                return 3
+
+        _log.warning("Could not determine EDM version from the input file. Return default EDMDecodingVersion=%d",
+                     default_version)
+        return default_version
+
     flags.addFlag('Trigger.EDMDecodingVersion', lambda prevFlags: EDMDecodingVersion(prevFlags))
                      
     # enables additional algorithms colecting MC truth infrmation  (this is only used by IDso maybe we need Trigger.ID.doTruth only?)

Trigger/TriggerCommon/TriggerJobOpts/test/test_TriggerFlags_autoconf.py

@@ -7,29 +7,26 @@
 from AthenaCommon.Logging import logging
 log=logging.getLogger('test_TriggerFlags_autoconf')
 
-import os, glob
-artdir = "/eos/atlas/atlascerngroupdisk/data-art/build-output/master/Athena/x86_64-centos7-gcc8-opt/"
-nightly = os.listdir(artdir)[-2] # don't take the latest; this may still be running
-nightlydir = artdir+nightly+"/"
-
-def getRun2MCFile():
-    filen = "myAOD.pool.root"
-    artdir_21 = "/eos/atlas/atlascerngroupdisk/data-art/grid-output/21.0/Athena/x86_64-slc6-gcc62-opt/"
-    nightly_21 = os.listdir(artdir_21)[-2] # don't take the latest; this may still be running
-    if not os.path.isfile(filen):
-        tarball = glob.glob(artdir_21+nightly_21+"/Tier0ChainTests/test_q220/user.artprod.*.EXT1.*.tar")[0]
-        os.system("tar -xf {} {}".format(tarball,filen))
-    return filen
+import os
+def get_file_from_art(package, test):
+    artdir = "/eos/atlas/atlascerngroupdisk/data-art/build-output/master/Athena/x86_64-centos7-gcc8-opt/"
+    if not os.path.isdir(artdir):
+        log.warning('Cannot access %s', artdir)
+        return None
+    nightly = os.listdir(artdir)[-2] # don't take the latest; this may still be running
+    nightlydir = artdir + nightly + "/"
+    return nightlydir + package + "/" + test + "/AOD.pool.root"
 
+from TrigValTools.TrigValSteering import Input
 inputfiles = {
-    "Run1_Data": "/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigAnalysisTest/data12_8TeV.00209109.physics_JetTauEtmiss.merge.RAW._lb0186._SFO-1._0001.1",
-    "Run2_Data": "/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/data18_13TeV.00360026.physics_EnhancedBias.merge.RAW._lb0151._SFO-1._0001.1",
-    "Run3_Data": nightlydir+"/TrigP1Test/test_trigP1_v1PhysP1_T0Mon_build/data18_13TeV.00360026.physics_Main.unknown.RAW._lb0151._athenaHLT._0001.data",
+    "Run1_Data": Input.get_input('data_run1').paths[0],
+    "Run2_Data": Input.get_input('data').paths[0],
+    "Run3_Data": Input.get_input('data_run3').paths[0],
     #
-    "Run2_MC_r22": nightlydir+"TrigAnalysisTest/test_trigAna_RDOtoAOD_build/AOD.pool.root",
-    "Run2_MC_r21": getRun2MCFile(),
+    "Run2_MC_r22": get_file_from_art("TrigAnalysisTest", "test_trigAna_RDOtoAOD_v7Primaries_build"),
+    "Run2_MC_r21": '/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/Tier0ChainTests/q221/21.0/v3/myAOD.pool.root',
     #
-    "Run3_MC": nightlydir+"TrigAnalysisTest/test_trigAna_RDOtoAOD_mt1_build/AOD.pool.root",
+    "Run3_MC": get_file_from_art("TrigAnalysisTest", "test_trigAna_RDOtoAOD_v1Dev_build")
 }
 
 from TriggerJobOpts.TriggerConfigFlags import createTriggerFlags
@@ -41,6 +38,11 @@ tcf_log.setLevel(DEBUG)
 def test_TriggerFlags(sample):
 
     from AthenaConfiguration.AllConfigFlags import ConfigFlags
+    input_file = inputfiles[sample]
+    if not input_file or not os.path.isfile(input_file):
+        log.warning('Skipping %s because cannot access the input file', sample)
+        return
+
     acf = ConfigFlags.clone()
     acf.Input.Files = [inputfiles[sample]]
     acf.addFlagsCategory("Trigger", createTriggerFlags)
@@ -59,7 +61,7 @@ def test_TriggerFlags(sample):
 if __name__=="__main__":
     log.info("Testing TriggerConfigFlags autoconfiguration")
     import sys
-    for sample in sorted(inputfiles.keys()):
+    for sample in inputfiles:
         run, source = sample.split('_',1)
         log.info("{} {} input file: {}".format(run, source, inputfiles[sample]))
         try: