TriggerMenuMT: Stop producing L1 menu files during build

The main change is to no longer produce the L1 menu files (.xml, .json)
during the build and not install them any longer as part of the release.
This technique was inherited from `TriggerMenuXML`, but no current job
actually uses these pre-generated files. This simplifies the cmake
configuration greatly and `TriggerMenuMT` is no longer forced to run at
the end of the build.

To ensure menus are working, add a unit test for each L1 menu.

Some unrelated package cleanup:
- Delete unused `testMenuMT.sh`, `trigermenuMT_test.py`, `generateMenuMT.py`.
- Delete `generateMT.py` and instead inline the flags in `test_HLTmenu.sh`.
- Move `menu_config_tests.py` to `python/` directory.

Trigger/TriggerCommon/TriggerMenuMT/CMakeLists.txt

-################################################################################
-# Package: TriggerMenuMT
-################################################################################
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
 
 # Declare the package name:
 atlas_subdir( TriggerMenuMT )
 
-# Declare the package's dependencies: 
-atlas_depends_on_subdirs( PRIVATE
-                          Trigger/TriggerCommon/TriggerJobOpts)
-
 # External dependencies:
 find_package( six )
 
-atlas_add_test( flake8
-   SCRIPT flake8 --select=ATL,F,E7,E9,W6,E101 --enable-extension=ATL900,ATL901,ATL902
-   ${CMAKE_CURRENT_SOURCE_DIR}/python ${CMAKE_CURRENT_SOURCE_DIR}/scripts
-   POST_EXEC_SCRIPT nopost.sh )
-
-#----------------------------------
-# Function to build trigger menu:
-function( atlas_build_lvl1_trigger_menu menu )
-
-   # Don't do anything in release recompilation dryrun mode. In all other
-   # modes, proceed as usual.
-   if( ATLAS_RELEASE_RECOMPILE_DRYRUN )
-      return()
-   endif()
-
-   # Command to build trigger menu. The idea is that ${menu}.stamp gets
-   # created as the last command, should the menu generation succeed such that 
-   # after a successful menu generation it wouldn't be attempted again.
-   # In order for the installation step to not try to re-generate
-   # the menu in case it was the generation itself that failed, another
-   # stamp file, ${menu}.attempted.stamp is created as the first command.
-   # The menu is then only generated as part of the installation step if
-   # this ${menu}.attempted.stamp file doesn't even exist.
-   
-   add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${menu}.stamp
-      COMMAND ${CMAKE_COMMAND} -E touch
-      ${CMAKE_CURRENT_BINARY_DIR}/${menu}.attempted.stamp
-      COMMAND ${CMAKE_COMMAND} -E make_directory
-      ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu}
-      COMMAND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/atlas_build_run.sh
-      ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/generateL1MenuMT.sh -r ${CMAKE_PROJECT_VERSION} ${menu}
-      ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu}
-      COMMAND ${CMAKE_COMMAND} -E make_directory
-      ${CMAKE_XML_OUTPUT_DIRECTORY}/TriggerMenuMT
-      COMMAND ${CMAKE_COMMAND} -E copy_directory
-      ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu}/
-      ${CMAKE_XML_OUTPUT_DIRECTORY}/TriggerMenuMT
-      COMMAND ${CMAKE_COMMAND} -E touch
-      ${CMAKE_CURRENT_BINARY_DIR}/${menu}.stamp
-      DEPENDS "Package_$<JOIN:$<TARGET_PROPERTY:ATLAS_PACKAGES_TARGET,ATLAS_PACKAGES>,;Package_>" )
-
-   # Create custom target and add it to package dependencies
-   add_custom_target( build_menu_${menu} ALL SOURCES
-      ${CMAKE_CURRENT_BINARY_DIR}/${menu}.stamp )
-
-   # In case the file generation failed, because it wasn't even attempted
-   # (failure in another package), then try to run the generation as part
-   # of the installation. Note that apparently chaining commands inside a
-   # single execute_process(...) call doesn't work correctly during installation
-   # for some reason. Hence it's taken apart into 3 separate calls.
-   install( CODE "if( NOT EXISTS
-                     ${CMAKE_CURRENT_BINARY_DIR}/${menu}.attempted.stamp )
-                     message( WARNING \"Generating trigger menu ${menu}\"
-                              \" during the installation\" )
-                     execute_process( COMMAND ${CMAKE_COMMAND} -E touch
-                        ${CMAKE_CURRENT_BINARY_DIR}/${menu}.attempted.stamp )
-                     execute_process(
-                        COMMAND ${CMAKE_COMMAND} -E make_directory
-                        ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu} )
-                     execute_process(
-                        COMMAND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/atlas_build_run.sh
-                        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/generateL1MenuMT.sh
-                        -r ${CMAKE_PROJECT_VERSION} ${menu} ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu} )
-                  endif()" )
-
-   # Install the generated XML files. Note that this installation rule is
-   # executed after the previous code. So by this time the files should be
-   # in place, if they could be produced.
-   install( DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu}/
-      DESTINATION ${CMAKE_INSTALL_XMLDIR}/TriggerMenuMT
-      USE_SOURCE_PERMISSIONS
-      FILES_MATCHING PATTERN "*.xml" )
-
-   # Create a target that will depend on all the other targets, and will
-   # print the "right message" at the end of the build. Notice that
-   # we can't rely on the Package_TriggerMenuXML target here, since
-   # the XML generation depends on all package targets being ready before
-   # it could start. So it would cause a circular dependency to make the
-   # menu targets be dependencies of the package target.
-   if( NOT TARGET TriggerMenuMTMain )
-      add_custom_target( TriggerMenuMTMain ALL
-         COMMAND ${CMAKE_COMMAND} -E echo
-         "TriggerMenuMT: Package build succeeded" )
-   endif()
-   add_dependencies( TriggerMenuMTMain build_menu_${menu} )
-
-endfunction ( atlas_build_lvl1_trigger_menu )
-
-
-
-#----------------------------------
 # Install files from the package:
 atlas_install_python_modules( python/*.py
                               python/LVL1MenuConfig
                               python/L1
                               python/HLTMenuConfig
-			      POST_BUILD_CMD ${ATLAS_FLAKE8})
-atlas_install_joboptions( share/*.py )
-
-atlas_install_scripts( scripts/generateMenuMT.py )
-atlas_install_scripts( scripts/generateL1MenuMT.sh )
-atlas_install_scripts( scripts/generateLVL1MenuMT.py )
-atlas_install_scripts( scripts/generateL1TopoMenuMT.py )
-atlas_install_scripts( scripts/test_HLTmenu.sh )
-atlas_install_scripts( scripts/generateL1MenuRun3.py )
-atlas_install_scripts( scripts/verify_menu_config.py )
-
-atlas_install_xmls( data/*.dtd data/*.xml )
-
-atlas_add_test( generateMenuMT_newJO SCRIPT python -m TriggerMenuMT.HLTMenuConfig.Menu.LS2_v1_newJO
-                PROPERTIES TIMEOUT 500 
-              )
-
-atlas_add_test( ViewCFTest     SCRIPT python -m unittest -v TriggerMenuMT.HLTMenuConfig.Test.ViewCFTest POST_EXEC_SCRIPT nopost.sh )
-
-atlas_add_test( generateMenuMT SCRIPT bash test_HLTmenu.sh
-                PROPERTIES TIMEOUT 4000 
-              )
-
-#----------------------------------
-# List of menus to be created:
-atlas_build_lvl1_trigger_menu( LS2_v1 )
-atlas_build_lvl1_trigger_menu( Physics_pp_run3_v1 )
-atlas_build_lvl1_trigger_menu( PhysicsP1_pp_run3_v1 )
-atlas_build_lvl1_trigger_menu( MC_pp_run3_v1 )
-atlas_build_lvl1_trigger_menu( Cosmic_run3_v1 )
-atlas_build_lvl1_trigger_menu( PhysicsP1_HI_run3_v1 )
-atlas_build_lvl1_trigger_menu( Dev_HI_run3_v1 )
+		                      POST_BUILD_CMD ${ATLAS_FLAKE8})
 
+atlas_install_scripts( scripts/generateMenuMT.py
+                       scripts/generateLVL1MenuMT.py
+                       scripts/generateL1TopoMenuMT.py
+                       scripts/generateL1MenuRun3.py
+                       scripts/verify_menu_config.py
+                       POST_BUILD_CMD ${ATLAS_FLAKE8} )
 
+atlas_install_scripts( scripts/generateL1MenuMT.sh )
+atlas_install_xmls( data/*.xml )
 
+# Tests in the package:
+atlas_add_test( flake8
+                SCRIPT flake8 --select=ATL,F,E7,E9,W6,E101 --enable-extension=ATL900,ATL901,ATL902
+                ${CMAKE_CURRENT_SOURCE_DIR}/python ${CMAKE_CURRENT_SOURCE_DIR}/scripts
+                POST_EXEC_SCRIPT nopost.sh )
+
+atlas_add_test( ViewCFTest
+                SCRIPT python -m unittest -v TriggerMenuMT.HLTMenuConfig.Test.ViewCFTest
+                POST_EXEC_SCRIPT nopost.sh )
+
+atlas_add_test( generateMenuMT_newJO
+                SCRIPT python -m TriggerMenuMT.HLTMenuConfig.Menu.LS2_v1_newJO
+                PROPERTIES TIMEOUT 500 )
+
+atlas_add_test( generateMenuMT
+                SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/scripts/test_HLTmenu.sh
+                PROPERTIES TIMEOUT 4000 )
+
+# Test L1 trigger menus:
+function( atlas_test_lvl1_trigger_menu menu )
+   atlas_add_test( "L1_${menu}"
+                   SCRIPT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/generateL1MenuMT.sh
+                   -r ${CMAKE_PROJECT_VERSION} ${menu} ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu} )
+endfunction()
+
+atlas_test_lvl1_trigger_menu( LS2_v1 )
+atlas_test_lvl1_trigger_menu( Physics_pp_run3_v1 )
+atlas_test_lvl1_trigger_menu( PhysicsP1_pp_run3_v1 )
+atlas_test_lvl1_trigger_menu( MC_pp_run3_v1 )
+atlas_test_lvl1_trigger_menu( Cosmic_run3_v1 )
+atlas_test_lvl1_trigger_menu( PhysicsP1_HI_run3_v1 )
+atlas_test_lvl1_trigger_menu( Dev_HI_run3_v1 )

Trigger/TriggerCommon/TriggerMenuMT/scripts/generateL1MenuMT.sh

@@ -36,6 +36,8 @@ menu=$1
 dest=$2
 if [ -z "$dest" ]; then
     dest=$PWD
+else
+    mkdir -p $dest
 fi
 
 # Get absolute path name

Trigger/TriggerCommon/TriggerMenuMT/scripts/generateMenuMT.py

-# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
-
-# import flags
-doWriteRDOTrigger = False
-doWriteBS = False
-include("TriggerJobOpts/runHLT_standalone.py")  # noqa: F821
-
-#################################
-# Configure L1Decoder
-#################################
-
-# provide a minimal menu information
-from AthenaCommon.GlobalFlags import globalflags
-from AthenaCommon.AlgSequence import AlgSequence
-topSequence = AlgSequence()
-if globalflags.InputFormat.is_bytestream():
-   from AthenaCommon.Constants import DEBUG
-   topSequence.L1Decoder.ctpUnpacker.OutputLevel=DEBUG  # noqa: ATL900
-
-
-
-# this is a temporary hack to include new test chains
-# very hacky indeed, the L1 part should be in in the GenerateMenuMT part
-
-from TriggerJobOpts.TriggerFlags import TriggerFlags
-TriggerFlags.triggerMenuSetup = "LS2_v1"
-
-from TriggerMenuMT.HLTMenuConfig.Menu.GenerateMenuMT import GenerateMenuMT
-g = GenerateMenuMT()
-
-
-
-
-
-##########################################
-# Menu and CF construction
-##########################################
-g.generateMT()
-
-
-
-

Trigger/TriggerCommon/TriggerMenuMT/scripts/testMenuMT.sh

-#!/usr/bin/env sh
-
-help() {
-    echo "Syntax: testMenuMT.sh [-r VERSION] MENU [DEST]"
-}
-
-if [ $# -lt 1 ]; then
-    help
-    exit 0
-fi
-
-while true; do
-    case "$1" in
-        -r)
-            release=$2
-            shift 2
-            ;;
-        -h)
-            help
-            exit 0
-            ;;
-        -*)
-            echo "Invalid parameter"
-            exit 1
-            ;;
-        *)
-            break
-    esac
-done
-
-menu=$1
-dest=$2
-if [ -z "$dest" ]; then
-    dest=$PWD
-fi
-
-# Get absolute path name
-dest=`cd $dest; pwd`
-
-if [ -z "$release" ]; then
-    release=${AtlasVersion}    # for interactive use
-fi
-
-# Temporary run directroy and cleanup traps in case of termination
-rundir=`mktemp -t -d tmxml.${menu}.XXXXXXXXXX`
-TRAPINT() {
-    rm -rf $rundir
-    return 130 # 128+SIGINT
-}
-TRAPTERM() {
-    rm -rf $rundir
-    return 143 # 128+SIGTERM
-}
-
-
-## menu generation starts here
-echo "generateHLTMenuMT: Building menu: ${menu} for ${release}"
-logfiletopo=topo${menu}.log
-logfilelvl1=lvl1${menu}.log
-logfilehlt=hlt${menu}.log
-cd $rundir
-
-generateLVL1MenuMT.py $menu &> $logfiletopo 
-generateL1TopoMenuMT.py $menu &> $logfilelvl1 
-generateMenuMT.py $menu &> $logfilehlt
-
-cp L1Topoconfig_*.xml ${dest}
-cp LVL1config_*.xml ${dest}
-#cp $logfilelvl1 $logfiletopo ${dest}
-
-if [[ -e outputLVL1config.xml ]]; then
-    cp outputLVL1config.xml ${dest}/LVL1config_${menu}_${release}.xml
-fi
-
-#this gives some more sensitivity to hidden problems
-grep --colour ERROR ${dest}/$logfile
-grep --colour -A 100 "Shortened traceback" ${dest}/$logfile
-
-rm -rf $rundir
-
-# Do not return real athena exit code as we want to pretend everything was fine
-#unset PYTHONDONTWRITEBYTECODE

Trigger/TriggerCommon/TriggerMenuMT/scripts/test_HLTmenu.sh

@@ -2,6 +2,6 @@
 
 set -e
 
-athena --config-only=config.pkl -l DEBUG --threads=1 --skipEvents=10 --evtMax=20 --filesInput="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/data17_13TeV.00327265.physics_EnhancedBias.merge.RAW._lb0100._SFO-1._0001.1" TriggerMenuMT/generateMT.py;
+athena --config-only=config.pkl -l DEBUG --threads=1 --skipEvents=10 --evtMax=20 --filesInput="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/data17_13TeV.00327265.physics_EnhancedBias.merge.RAW._lb0100._SFO-1._0001.1" -c "setMenu='LS2_v1';doWriteBS=False;from TriggerJobOpts.TriggerFlags import TriggerFlags;TriggerFlags.generateMenuDiagnostics=True" TriggerJobOpts/runHLT_standalone.py
 
 verify_menu_config.py -f $PWD || { echo "ERROR: Menu verification failed" ; exit 1; }

Trigger/TriggerCommon/TriggerMenuMT/scripts/verify_menu_config.py

@@ -15,7 +15,7 @@ import sys
 import argparse
 
 from AthenaCommon.Logging import logging
-from menu_config_tests import TriggerLevel, menu_tests
+from TriggerMenuMT.menu_config_tests import TriggerLevel, menu_tests
 
 log = logging.getLogger("TriggerMenuConfigTest")
 

Trigger/TriggerCommon/TriggerMenuMT/share/generateMT.py

-# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
-
-setMenu             = "LS2_v1"
-doWriteRDOTrigger   = False
-doWriteBS           = False
-endJobAfterGenerate = False
-from TriggerJobOpts.TriggerFlags import TriggerFlags
-TriggerFlags.generateMenuDiagnostics = True
-include("TriggerJobOpts/runHLT_standalone.py")
-

Trigger/TriggerCommon/TriggerMenuMT/share/trigermenuMT_test.py

-#
-# Minimal example to build a trigger menu
-#
-from TriggerMenuMT.LVL1MenuConfig.TriggerConfigLVL1 import TriggerConfigLVL1            
-from TriggerMenuMT.LVL1MenuConfig.LVL1.Lvl1Flags import Lvl1Flags
-from TriggerMenuMT.LVL1MenuConfig.LVL1.Lvl1MenuItems import LVL1MenuItem
-from TriggerMenuMT.L1.Config.TriggerTypeDef import TT
-from TriggerMenuMT.LVL1MenuConfig.LVL1.Lvl1Condition import ThrCondition, Lvl1InternalTrigger
-from TriggerMenuMT.L1.Base.Logic import Logic
-
-# Define L1 menu thresholds and items
-Lvl1Flags.RemapThresholdsAsListed = True
-Lvl1Flags.thresholds = ['EM3','MU4']
-Lvl1Flags.items = ['L1_EM3','L1_MU4']
-
-l1menu = TriggerConfigLVL1( outputFile = 'l1menu.xml', 
-                            menuName = 'menutest',
-                            topoMenu = None )
-
-LVL1MenuItem.l1configForRegistration = l1menu
-
-# Register L1 menu
-bgrp = Logic(Lvl1InternalTrigger('BGRP0')) & Logic(Lvl1InternalTrigger('BGRP1'))
-
-EM3 = l1menu.registerThr('EM3','EM').addThrValue(3)
-LVL1MenuItem('L1_EM3'       ).setLogic( ThrCondition(EM3) & bgrp ).setTriggerType( TT.calo )
-
-MU4 = l1menu.registerThr('MU4','MUON').addThrValue(4)
-LVL1MenuItem('L1_MU4'       ).setLogic( ThrCondition(MU4) & bgrp ).setTriggerType( TT.muon )
-
-# Generate and write L1 menu
-l1menu.generateMenu()        
-l1menu.writeXML()
-
-
-from TriggerMenu.menu.TriggerPythonConfig import TriggerPythonConfig
-from TriggerMenu.menu.HLTObjects import HLTChain
-hltmenu = TriggerPythonConfig('hltmenu.xml')
-
-from TrigSteeringTest.TrigSteeringTestConf import PESA__dummyAlgo
-defalgo = PESA__dummyAlgo('Em3')
-hltmenu.addSequence("EM3" , defalgo, "em3" )
-
-chain = HLTChain( chain_name="HLT_e3", chain_counter="1", lower_chain_name="EM3", level = "HLT", prescale="1", pass_through="1")
-chain.addHLTSignature( "em3" ).addHLTSignature( "em3" )
-chain.addTriggerTypeBit('4')
-chain.addStreamTag('electrons', prescale='1', obeyLB="yes")
-chain.addStreamTag('IDCalibration', prescale='1', obeyLB="no", type="calibration")
-chain.addGroup("electrons")
-hltmenu.addHLTChain(chain)
-
-hltmenu.writeConfigFiles();
-
-import sys
-sys.exit(0)