From 15ab2d7d5441f261c611cf6faa34190376c4b0f4 Mon Sep 17 00:00:00 2001
From: Frank Winklmeier <frank.winklmeier@cern.ch>
Date: Tue, 16 Jun 2020 11:49:23 +0200
Subject: [PATCH] TriggerMenuMT: Stop producing L1 menu files during build
The main change is to no longer produce the L1 menu files (.xml, .json)
during the build and not install them any longer as part of the release.
This technique was inherited from `TriggerMenuXML`, but no current job
actually uses these pre-generated files. This simplifies the cmake
configuration greatly and `TriggerMenuMT` is no longer forced to run at
the end of the build.
To ensure menus are working, add a unit test for each L1 menu.
Some unrelated package cleanup:
- Delete unused `testMenuMT.sh`, `trigermenuMT_test.py`, `generateMenuMT.py`.
- Delete `generateMT.py` and instead inline the flags in `test_HLTmenu.sh`.
- Move `menu_config_tests.py` to `python/` directory.
---
.../TriggerMenuMT/CMakeLists.txt | 172 +++++-------------
.../{scripts => python}/menu_config_tests.py | 0
.../TriggerMenuMT/scripts/generateL1MenuMT.sh | 2 +
.../TriggerMenuMT/scripts/generateMenuMT.py | 42 -----
.../TriggerMenuMT/scripts/testMenuMT.sh | 82 ---------
.../TriggerMenuMT/scripts/test_HLTmenu.sh | 2 +-
.../scripts/verify_menu_config.py | 2 +-
.../TriggerMenuMT/share/generateMT.py | 10 -
.../TriggerMenuMT/share/trigermenuMT_test.py | 55 ------
9 files changed, 46 insertions(+), 321 deletions(-)
rename Trigger/TriggerCommon/TriggerMenuMT/{scripts => python}/menu_config_tests.py (100%)
delete mode 100644 Trigger/TriggerCommon/TriggerMenuMT/scripts/generateMenuMT.py
delete mode 100755 Trigger/TriggerCommon/TriggerMenuMT/scripts/testMenuMT.sh
delete mode 100755 Trigger/TriggerCommon/TriggerMenuMT/share/generateMT.py
delete mode 100644 Trigger/TriggerCommon/TriggerMenuMT/share/trigermenuMT_test.py
diff --git a/Trigger/TriggerCommon/TriggerMenuMT/CMakeLists.txt b/Trigger/TriggerCommon/TriggerMenuMT/CMakeLists.txt
index 1a23c5bb63e..70014723b4f 100644
--- a/Trigger/TriggerCommon/TriggerMenuMT/CMakeLists.txt
+++ b/Trigger/TriggerCommon/TriggerMenuMT/CMakeLists.txt
@@ -1,145 +1,57 @@
-################################################################################
-# Package: TriggerMenuMT
-################################################################################
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
# Declare the package name:
atlas_subdir( TriggerMenuMT )
-# Declare the package's dependencies:
-atlas_depends_on_subdirs( PRIVATE
- Trigger/TriggerCommon/TriggerJobOpts)
-
# External dependencies:
find_package( six )
-atlas_add_test( flake8
- SCRIPT flake8 --select=ATL,F,E7,E9,W6,E101 --enable-extension=ATL900,ATL901,ATL902
- ${CMAKE_CURRENT_SOURCE_DIR}/python ${CMAKE_CURRENT_SOURCE_DIR}/scripts
- POST_EXEC_SCRIPT nopost.sh )
-
-#----------------------------------
-# Function to build trigger menu:
-function( atlas_build_lvl1_trigger_menu menu )
-
- # Don't do anything in release recompilation dryrun mode. In all other
- # modes, proceed as usual.
- if( ATLAS_RELEASE_RECOMPILE_DRYRUN )
- return()
- endif()
-
- # Command to build trigger menu. The idea is that ${menu}.stamp gets
- # created as the last command, should the menu generation succeed such that
- # after a successful menu generation it wouldn't be attempted again.
- # In order for the installation step to not try to re-generate
- # the menu in case it was the generation itself that failed, another
- # stamp file, ${menu}.attempted.stamp is created as the first command.
- # The menu is then only generated as part of the installation step if
- # this ${menu}.attempted.stamp file doesn't even exist.
-
- add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${menu}.stamp
- COMMAND ${CMAKE_COMMAND} -E touch
- ${CMAKE_CURRENT_BINARY_DIR}/${menu}.attempted.stamp
- COMMAND ${CMAKE_COMMAND} -E make_directory
- ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu}
- COMMAND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/atlas_build_run.sh
- ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/generateL1MenuMT.sh -r ${CMAKE_PROJECT_VERSION} ${menu}
- ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu}
- COMMAND ${CMAKE_COMMAND} -E make_directory
- ${CMAKE_XML_OUTPUT_DIRECTORY}/TriggerMenuMT
- COMMAND ${CMAKE_COMMAND} -E copy_directory
- ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu}/
- ${CMAKE_XML_OUTPUT_DIRECTORY}/TriggerMenuMT
- COMMAND ${CMAKE_COMMAND} -E touch
- ${CMAKE_CURRENT_BINARY_DIR}/${menu}.stamp
- DEPENDS "Package_$<JOIN:$<TARGET_PROPERTY:ATLAS_PACKAGES_TARGET,ATLAS_PACKAGES>,;Package_>" )
-
- # Create custom target and add it to package dependencies
- add_custom_target( build_menu_${menu} ALL SOURCES
- ${CMAKE_CURRENT_BINARY_DIR}/${menu}.stamp )
-
- # In case the file generation failed, because it wasn't even attempted
- # (failure in another package), then try to run the generation as part
- # of the installation. Note that apparently chaining commands inside a
- # single execute_process(...) call doesn't work correctly during installation
- # for some reason. Hence it's taken apart into 3 separate calls.
- install( CODE "if( NOT EXISTS
- ${CMAKE_CURRENT_BINARY_DIR}/${menu}.attempted.stamp )
- message( WARNING \"Generating trigger menu ${menu}\"
- \" during the installation\" )
- execute_process( COMMAND ${CMAKE_COMMAND} -E touch
- ${CMAKE_CURRENT_BINARY_DIR}/${menu}.attempted.stamp )
- execute_process(
- COMMAND ${CMAKE_COMMAND} -E make_directory
- ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu} )
- execute_process(
- COMMAND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/atlas_build_run.sh
- ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/generateL1MenuMT.sh
- -r ${CMAKE_PROJECT_VERSION} ${menu} ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu} )
- endif()" )
-
- # Install the generated XML files. Note that this installation rule is
- # executed after the previous code. So by this time the files should be
- # in place, if they could be produced.
- install( DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu}/
- DESTINATION ${CMAKE_INSTALL_XMLDIR}/TriggerMenuMT
- USE_SOURCE_PERMISSIONS
- FILES_MATCHING PATTERN "*.xml" )
-
- # Create a target that will depend on all the other targets, and will
- # print the "right message" at the end of the build. Notice that
- # we can't rely on the Package_TriggerMenuXML target here, since
- # the XML generation depends on all package targets being ready before
- # it could start. So it would cause a circular dependency to make the
- # menu targets be dependencies of the package target.
- if( NOT TARGET TriggerMenuMTMain )
- add_custom_target( TriggerMenuMTMain ALL
- COMMAND ${CMAKE_COMMAND} -E echo
- "TriggerMenuMT: Package build succeeded" )
- endif()
- add_dependencies( TriggerMenuMTMain build_menu_${menu} )
-
-endfunction ( atlas_build_lvl1_trigger_menu )
-
-
-
-#----------------------------------
# Install files from the package:
atlas_install_python_modules( python/*.py
python/LVL1MenuConfig
python/L1
python/HLTMenuConfig
- POST_BUILD_CMD ${ATLAS_FLAKE8})
-atlas_install_joboptions( share/*.py )
-
-atlas_install_scripts( scripts/generateMenuMT.py )
-atlas_install_scripts( scripts/generateL1MenuMT.sh )
-atlas_install_scripts( scripts/generateLVL1MenuMT.py )
-atlas_install_scripts( scripts/generateL1TopoMenuMT.py )
-atlas_install_scripts( scripts/test_HLTmenu.sh )
-atlas_install_scripts( scripts/generateL1MenuRun3.py )
-atlas_install_scripts( scripts/verify_menu_config.py )
-
-atlas_install_xmls( data/*.dtd data/*.xml )
-
-atlas_add_test( generateMenuMT_newJO SCRIPT python -m TriggerMenuMT.HLTMenuConfig.Menu.LS2_v1_newJO
- PROPERTIES TIMEOUT 500
- )
-
-atlas_add_test( ViewCFTest SCRIPT python -m unittest -v TriggerMenuMT.HLTMenuConfig.Test.ViewCFTest POST_EXEC_SCRIPT nopost.sh )
-
-atlas_add_test( generateMenuMT SCRIPT bash test_HLTmenu.sh
- PROPERTIES TIMEOUT 4000
- )
-
-#----------------------------------
-# List of menus to be created:
-atlas_build_lvl1_trigger_menu( LS2_v1 )
-atlas_build_lvl1_trigger_menu( Physics_pp_run3_v1 )
-atlas_build_lvl1_trigger_menu( PhysicsP1_pp_run3_v1 )
-atlas_build_lvl1_trigger_menu( MC_pp_run3_v1 )
-atlas_build_lvl1_trigger_menu( Cosmic_run3_v1 )
-atlas_build_lvl1_trigger_menu( PhysicsP1_HI_run3_v1 )
-atlas_build_lvl1_trigger_menu( Dev_HI_run3_v1 )
+ POST_BUILD_CMD ${ATLAS_FLAKE8})
+atlas_install_scripts( scripts/generateMenuMT.py
+ scripts/generateLVL1MenuMT.py
+ scripts/generateL1TopoMenuMT.py
+ scripts/generateL1MenuRun3.py
+ scripts/verify_menu_config.py
+ POST_BUILD_CMD ${ATLAS_FLAKE8} )
+atlas_install_scripts( scripts/generateL1MenuMT.sh )
+atlas_install_xmls( data/*.xml )
+# Tests in the package:
+atlas_add_test( flake8
+ SCRIPT flake8 --select=ATL,F,E7,E9,W6,E101 --enable-extension=ATL900,ATL901,ATL902
+ ${CMAKE_CURRENT_SOURCE_DIR}/python ${CMAKE_CURRENT_SOURCE_DIR}/scripts
+ POST_EXEC_SCRIPT nopost.sh )
+
+atlas_add_test( ViewCFTest
+ SCRIPT python -m unittest -v TriggerMenuMT.HLTMenuConfig.Test.ViewCFTest
+ POST_EXEC_SCRIPT nopost.sh )
+
+atlas_add_test( generateMenuMT_newJO
+ SCRIPT python -m TriggerMenuMT.HLTMenuConfig.Menu.LS2_v1_newJO
+ PROPERTIES TIMEOUT 500 )
+
+atlas_add_test( generateMenuMT
+ SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/scripts/test_HLTmenu.sh
+ PROPERTIES TIMEOUT 4000 )
+
+# Test L1 trigger menus:
+function( atlas_test_lvl1_trigger_menu menu )
+ atlas_add_test( "L1_${menu}"
+ SCRIPT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/generateL1MenuMT.sh
+ -r ${CMAKE_PROJECT_VERSION} ${menu} ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu} )
+endfunction()
+
+atlas_test_lvl1_trigger_menu( LS2_v1 )
+atlas_test_lvl1_trigger_menu( Physics_pp_run3_v1 )
+atlas_test_lvl1_trigger_menu( PhysicsP1_pp_run3_v1 )
+atlas_test_lvl1_trigger_menu( MC_pp_run3_v1 )
+atlas_test_lvl1_trigger_menu( Cosmic_run3_v1 )
+atlas_test_lvl1_trigger_menu( PhysicsP1_HI_run3_v1 )
+atlas_test_lvl1_trigger_menu( Dev_HI_run3_v1 )
diff --git a/Trigger/TriggerCommon/TriggerMenuMT/scripts/menu_config_tests.py b/Trigger/TriggerCommon/TriggerMenuMT/python/menu_config_tests.py
similarity index 100%
rename from Trigger/TriggerCommon/TriggerMenuMT/scripts/menu_config_tests.py
rename to Trigger/TriggerCommon/TriggerMenuMT/python/menu_config_tests.py
diff --git a/Trigger/TriggerCommon/TriggerMenuMT/scripts/generateL1MenuMT.sh b/Trigger/TriggerCommon/TriggerMenuMT/scripts/generateL1MenuMT.sh
index 705824041c5..41b3b9679a7 100755
--- a/Trigger/TriggerCommon/TriggerMenuMT/scripts/generateL1MenuMT.sh
+++ b/Trigger/TriggerCommon/TriggerMenuMT/scripts/generateL1MenuMT.sh
@@ -36,6 +36,8 @@ menu=$1
dest=$2
if [ -z "$dest" ]; then
dest=$PWD
+else
+ mkdir -p $dest
fi
# Get absolute path name
diff --git a/Trigger/TriggerCommon/TriggerMenuMT/scripts/generateMenuMT.py b/Trigger/TriggerCommon/TriggerMenuMT/scripts/generateMenuMT.py
deleted file mode 100644
index c98b0ae0c8c..00000000000
--- a/Trigger/TriggerCommon/TriggerMenuMT/scripts/generateMenuMT.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
-
-# import flags
-doWriteRDOTrigger = False
-doWriteBS = False
-include("TriggerJobOpts/runHLT_standalone.py") # noqa: F821
-
-#################################
-# Configure L1Decoder
-#################################
-
-# provide a minimal menu information
-from AthenaCommon.GlobalFlags import globalflags
-from AthenaCommon.AlgSequence import AlgSequence
-topSequence = AlgSequence()
-if globalflags.InputFormat.is_bytestream():
- from AthenaCommon.Constants import DEBUG
- topSequence.L1Decoder.ctpUnpacker.OutputLevel=DEBUG # noqa: ATL900
-
-
-
-# this is a temporary hack to include new test chains
-# very hacky indeed, the L1 part should be in in the GenerateMenuMT part
-
-from TriggerJobOpts.TriggerFlags import TriggerFlags
-TriggerFlags.triggerMenuSetup = "LS2_v1"
-
-from TriggerMenuMT.HLTMenuConfig.Menu.GenerateMenuMT import GenerateMenuMT
-g = GenerateMenuMT()
-
-
-
-
-
-##########################################
-# Menu and CF construction
-##########################################
-g.generateMT()
-
-
-
-
diff --git a/Trigger/TriggerCommon/TriggerMenuMT/scripts/testMenuMT.sh b/Trigger/TriggerCommon/TriggerMenuMT/scripts/testMenuMT.sh
deleted file mode 100755
index c26ea2bbdf2..00000000000
--- a/Trigger/TriggerCommon/TriggerMenuMT/scripts/testMenuMT.sh
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/usr/bin/env sh
-
-help() {
- echo "Syntax: testMenuMT.sh [-r VERSION] MENU [DEST]"
-}
-
-if [ $# -lt 1 ]; then
- help
- exit 0
-fi
-
-while true; do
- case "$1" in
- -r)
- release=$2
- shift 2
- ;;
- -h)
- help
- exit 0
- ;;
- -*)
- echo "Invalid parameter"
- exit 1
- ;;
- *)
- break
- esac
-done
-
-menu=$1
-dest=$2
-if [ -z "$dest" ]; then
- dest=$PWD
-fi
-
-# Get absolute path name
-dest=`cd $dest; pwd`
-
-if [ -z "$release" ]; then
- release=${AtlasVersion} # for interactive use
-fi
-
-# Temporary run directroy and cleanup traps in case of termination
-rundir=`mktemp -t -d tmxml.${menu}.XXXXXXXXXX`
-TRAPINT() {
- rm -rf $rundir
- return 130 # 128+SIGINT
-}
-TRAPTERM() {
- rm -rf $rundir
- return 143 # 128+SIGTERM
-}
-
-
-## menu generation starts here
-echo "generateHLTMenuMT: Building menu: ${menu} for ${release}"
-logfiletopo=topo${menu}.log
-logfilelvl1=lvl1${menu}.log
-logfilehlt=hlt${menu}.log
-cd $rundir
-
-generateLVL1MenuMT.py $menu &> $logfiletopo
-generateL1TopoMenuMT.py $menu &> $logfilelvl1
-generateMenuMT.py $menu &> $logfilehlt
-
-cp L1Topoconfig_*.xml ${dest}
-cp LVL1config_*.xml ${dest}
-#cp $logfilelvl1 $logfiletopo ${dest}
-
-if [[ -e outputLVL1config.xml ]]; then
- cp outputLVL1config.xml ${dest}/LVL1config_${menu}_${release}.xml
-fi
-
-#this gives some more sensitivity to hidden problems
-grep --colour ERROR ${dest}/$logfile
-grep --colour -A 100 "Shortened traceback" ${dest}/$logfile
-
-rm -rf $rundir
-
-# Do not return real athena exit code as we want to pretend everything was fine
-#unset PYTHONDONTWRITEBYTECODE
diff --git a/Trigger/TriggerCommon/TriggerMenuMT/scripts/test_HLTmenu.sh b/Trigger/TriggerCommon/TriggerMenuMT/scripts/test_HLTmenu.sh
index bce1f232c39..19dfbd9ff9c 100755
--- a/Trigger/TriggerCommon/TriggerMenuMT/scripts/test_HLTmenu.sh
+++ b/Trigger/TriggerCommon/TriggerMenuMT/scripts/test_HLTmenu.sh
@@ -2,6 +2,6 @@
set -e
-athena --config-only=config.pkl -l DEBUG --threads=1 --skipEvents=10 --evtMax=20 --filesInput="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/data17_13TeV.00327265.physics_EnhancedBias.merge.RAW._lb0100._SFO-1._0001.1" TriggerMenuMT/generateMT.py;
+athena --config-only=config.pkl -l DEBUG --threads=1 --skipEvents=10 --evtMax=20 --filesInput="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/data17_13TeV.00327265.physics_EnhancedBias.merge.RAW._lb0100._SFO-1._0001.1" -c "setMenu='LS2_v1';doWriteBS=False;from TriggerJobOpts.TriggerFlags import TriggerFlags;TriggerFlags.generateMenuDiagnostics=True" TriggerJobOpts/runHLT_standalone.py
verify_menu_config.py -f $PWD || { echo "ERROR: Menu verification failed" ; exit 1; }
diff --git a/Trigger/TriggerCommon/TriggerMenuMT/scripts/verify_menu_config.py b/Trigger/TriggerCommon/TriggerMenuMT/scripts/verify_menu_config.py
index d4e512a10bf..91a79a1ce3e 100755
--- a/Trigger/TriggerCommon/TriggerMenuMT/scripts/verify_menu_config.py
+++ b/Trigger/TriggerCommon/TriggerMenuMT/scripts/verify_menu_config.py
@@ -15,7 +15,7 @@ import sys
import argparse
from AthenaCommon.Logging import logging
-from menu_config_tests import TriggerLevel, menu_tests
+from TriggerMenuMT.menu_config_tests import TriggerLevel, menu_tests
log = logging.getLogger("TriggerMenuConfigTest")
diff --git a/Trigger/TriggerCommon/TriggerMenuMT/share/generateMT.py b/Trigger/TriggerCommon/TriggerMenuMT/share/generateMT.py
deleted file mode 100755
index 9f5039fe5a8..00000000000
--- a/Trigger/TriggerCommon/TriggerMenuMT/share/generateMT.py
+++ /dev/null
@@ -1,10 +0,0 @@
-# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
-
-setMenu = "LS2_v1"
-doWriteRDOTrigger = False
-doWriteBS = False
-endJobAfterGenerate = False
-from TriggerJobOpts.TriggerFlags import TriggerFlags
-TriggerFlags.generateMenuDiagnostics = True
-include("TriggerJobOpts/runHLT_standalone.py")
-
diff --git a/Trigger/TriggerCommon/TriggerMenuMT/share/trigermenuMT_test.py b/Trigger/TriggerCommon/TriggerMenuMT/share/trigermenuMT_test.py
deleted file mode 100644
index 9f64d82acfc..00000000000
--- a/Trigger/TriggerCommon/TriggerMenuMT/share/trigermenuMT_test.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#
-# Minimal example to build a trigger menu
-#
-from TriggerMenuMT.LVL1MenuConfig.TriggerConfigLVL1 import TriggerConfigLVL1
-from TriggerMenuMT.LVL1MenuConfig.LVL1.Lvl1Flags import Lvl1Flags
-from TriggerMenuMT.LVL1MenuConfig.LVL1.Lvl1MenuItems import LVL1MenuItem
-from TriggerMenuMT.L1.Config.TriggerTypeDef import TT
-from TriggerMenuMT.LVL1MenuConfig.LVL1.Lvl1Condition import ThrCondition, Lvl1InternalTrigger
-from TriggerMenuMT.L1.Base.Logic import Logic
-
-# Define L1 menu thresholds and items
-Lvl1Flags.RemapThresholdsAsListed = True
-Lvl1Flags.thresholds = ['EM3','MU4']
-Lvl1Flags.items = ['L1_EM3','L1_MU4']
-
-l1menu = TriggerConfigLVL1( outputFile = 'l1menu.xml',
- menuName = 'menutest',
- topoMenu = None )
-
-LVL1MenuItem.l1configForRegistration = l1menu
-
-# Register L1 menu
-bgrp = Logic(Lvl1InternalTrigger('BGRP0')) & Logic(Lvl1InternalTrigger('BGRP1'))
-
-EM3 = l1menu.registerThr('EM3','EM').addThrValue(3)
-LVL1MenuItem('L1_EM3' ).setLogic( ThrCondition(EM3) & bgrp ).setTriggerType( TT.calo )
-
-MU4 = l1menu.registerThr('MU4','MUON').addThrValue(4)
-LVL1MenuItem('L1_MU4' ).setLogic( ThrCondition(MU4) & bgrp ).setTriggerType( TT.muon )
-
-# Generate and write L1 menu
-l1menu.generateMenu()
-l1menu.writeXML()
-
-
-from TriggerMenu.menu.TriggerPythonConfig import TriggerPythonConfig
-from TriggerMenu.menu.HLTObjects import HLTChain
-hltmenu = TriggerPythonConfig('hltmenu.xml')
-
-from TrigSteeringTest.TrigSteeringTestConf import PESA__dummyAlgo
-defalgo = PESA__dummyAlgo('Em3')
-hltmenu.addSequence("EM3" , defalgo, "em3" )
-
-chain = HLTChain( chain_name="HLT_e3", chain_counter="1", lower_chain_name="EM3", level = "HLT", prescale="1", pass_through="1")
-chain.addHLTSignature( "em3" ).addHLTSignature( "em3" )
-chain.addTriggerTypeBit('4')
-chain.addStreamTag('electrons', prescale='1', obeyLB="yes")
-chain.addStreamTag('IDCalibration', prescale='1', obeyLB="no", type="calibration")
-chain.addGroup("electrons")
-hltmenu.addHLTChain(chain)
-
-hltmenu.writeConfigFiles();
-
-import sys
-sys.exit(0)
--
GitLab