Revert "Merge branch 'menumt_cleanup' into 'master'"

This reverts merge request !33843

Trigger/TriggerCommon/TriggerMenuMT/CMakeLists.txt

-# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+################################################################################
+# Package: TriggerMenuMT
+################################################################################
 
 # Declare the package name:
 atlas_subdir( TriggerMenuMT )
 
+# Declare the package's dependencies: 
+atlas_depends_on_subdirs( PRIVATE
+                          Trigger/TriggerCommon/TriggerJobOpts)
+
 # External dependencies:
 find_package( six )
 
+atlas_add_test( flake8
+   SCRIPT flake8 --select=ATL,F,E7,E9,W6,E101 --enable-extension=ATL900,ATL901,ATL902
+   ${CMAKE_CURRENT_SOURCE_DIR}/python ${CMAKE_CURRENT_SOURCE_DIR}/scripts
+   POST_EXEC_SCRIPT nopost.sh )
+
+#----------------------------------
+# Function to build trigger menu:
+function( atlas_build_lvl1_trigger_menu menu )
+
+   # Don't do anything in release recompilation dryrun mode. In all other
+   # modes, proceed as usual.
+   if( ATLAS_RELEASE_RECOMPILE_DRYRUN )
+      return()
+   endif()
+
+   # Command to build trigger menu. The idea is that ${menu}.stamp gets
+   # created as the last command, should the menu generation succeed such that 
+   # after a successful menu generation it wouldn't be attempted again.
+   # In order for the installation step to not try to re-generate
+   # the menu in case it was the generation itself that failed, another
+   # stamp file, ${menu}.attempted.stamp is created as the first command.
+   # The menu is then only generated as part of the installation step if
+   # this ${menu}.attempted.stamp file doesn't even exist.
+   
+   add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${menu}.stamp
+      COMMAND ${CMAKE_COMMAND} -E touch
+      ${CMAKE_CURRENT_BINARY_DIR}/${menu}.attempted.stamp
+      COMMAND ${CMAKE_COMMAND} -E make_directory
+      ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu}
+      COMMAND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/atlas_build_run.sh
+      ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/generateL1MenuMT.sh -r ${CMAKE_PROJECT_VERSION} ${menu}
+      ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu}
+      COMMAND ${CMAKE_COMMAND} -E make_directory
+      ${CMAKE_XML_OUTPUT_DIRECTORY}/TriggerMenuMT
+      COMMAND ${CMAKE_COMMAND} -E copy_directory
+      ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu}/
+      ${CMAKE_XML_OUTPUT_DIRECTORY}/TriggerMenuMT
+      COMMAND ${CMAKE_COMMAND} -E touch
+      ${CMAKE_CURRENT_BINARY_DIR}/${menu}.stamp
+      DEPENDS "Package_$<JOIN:$<TARGET_PROPERTY:ATLAS_PACKAGES_TARGET,ATLAS_PACKAGES>,;Package_>" )
+
+   # Create custom target and add it to package dependencies
+   add_custom_target( build_menu_${menu} ALL SOURCES
+      ${CMAKE_CURRENT_BINARY_DIR}/${menu}.stamp )
+
+   # In case the file generation failed, because it wasn't even attempted
+   # (failure in another package), then try to run the generation as part
+   # of the installation. Note that apparently chaining commands inside a
+   # single execute_process(...) call doesn't work correctly during installation
+   # for some reason. Hence it's taken apart into 3 separate calls.
+   install( CODE "if( NOT EXISTS
+                     ${CMAKE_CURRENT_BINARY_DIR}/${menu}.attempted.stamp )
+                     message( WARNING \"Generating trigger menu ${menu}\"
+                              \" during the installation\" )
+                     execute_process( COMMAND ${CMAKE_COMMAND} -E touch
+                        ${CMAKE_CURRENT_BINARY_DIR}/${menu}.attempted.stamp )
+                     execute_process(
+                        COMMAND ${CMAKE_COMMAND} -E make_directory
+                        ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu} )
+                     execute_process(
+                        COMMAND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/atlas_build_run.sh
+                        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/generateL1MenuMT.sh
+                        -r ${CMAKE_PROJECT_VERSION} ${menu} ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu} )
+                  endif()" )
+
+   # Install the generated XML files. Note that this installation rule is
+   # executed after the previous code. So by this time the files should be
+   # in place, if they could be produced.
+   install( DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu}/
+      DESTINATION ${CMAKE_INSTALL_XMLDIR}/TriggerMenuMT
+      USE_SOURCE_PERMISSIONS
+      FILES_MATCHING PATTERN "*.xml" )
+
+   # Create a target that will depend on all the other targets, and will
+   # print the "right message" at the end of the build. Notice that
+   # we can't rely on the Package_TriggerMenuXML target here, since
+   # the XML generation depends on all package targets being ready before
+   # it could start. So it would cause a circular dependency to make the
+   # menu targets be dependencies of the package target.
+   if( NOT TARGET TriggerMenuMTMain )
+      add_custom_target( TriggerMenuMTMain ALL
+         COMMAND ${CMAKE_COMMAND} -E echo
+         "TriggerMenuMT: Package build succeeded" )
+   endif()
+   add_dependencies( TriggerMenuMTMain build_menu_${menu} )
+
+endfunction ( atlas_build_lvl1_trigger_menu )
+
+
+
+#----------------------------------
 # Install files from the package:
 atlas_install_python_modules( python/*.py
                               python/LVL1MenuConfig
                               python/L1
                               python/HLTMenuConfig
-		                      POST_BUILD_CMD ${ATLAS_FLAKE8})
-
-atlas_install_scripts( scripts/generateMenuMT.py
-                       scripts/generateLVL1MenuMT.py
-                       scripts/generateL1TopoMenuMT.py
-                       scripts/generateL1MenuRun3.py
-                       scripts/verify_menu_config.py
-                       POST_BUILD_CMD ${ATLAS_FLAKE8} )
+			      POST_BUILD_CMD ${ATLAS_FLAKE8})
+atlas_install_joboptions( share/*.py )
 
+atlas_install_scripts( scripts/generateMenuMT.py )
 atlas_install_scripts( scripts/generateL1MenuMT.sh )
-atlas_install_xmls( data/*.xml )
+atlas_install_scripts( scripts/generateLVL1MenuMT.py )
+atlas_install_scripts( scripts/generateL1TopoMenuMT.py )
+atlas_install_scripts( scripts/test_HLTmenu.sh )
+atlas_install_scripts( scripts/generateL1MenuRun3.py )
+atlas_install_scripts( scripts/verify_menu_config.py )
+
+atlas_install_xmls( data/*.dtd data/*.xml )
+
+atlas_add_test( generateMenuMT_newJO SCRIPT python -m TriggerMenuMT.HLTMenuConfig.Menu.LS2_v1_newJO
+                PROPERTIES TIMEOUT 500 
+              )
+
+atlas_add_test( ViewCFTest     SCRIPT python -m unittest -v TriggerMenuMT.HLTMenuConfig.Test.ViewCFTest POST_EXEC_SCRIPT nopost.sh )
+
+atlas_add_test( generateMenuMT SCRIPT bash test_HLTmenu.sh
+                PROPERTIES TIMEOUT 4000 
+              )
+
+#----------------------------------
+# List of menus to be created:
+atlas_build_lvl1_trigger_menu( LS2_v1 )
+atlas_build_lvl1_trigger_menu( Physics_pp_run3_v1 )
+atlas_build_lvl1_trigger_menu( PhysicsP1_pp_run3_v1 )
+atlas_build_lvl1_trigger_menu( MC_pp_run3_v1 )
+atlas_build_lvl1_trigger_menu( Cosmic_run3_v1 )
+atlas_build_lvl1_trigger_menu( PhysicsP1_HI_run3_v1 )
+atlas_build_lvl1_trigger_menu( Dev_HI_run3_v1 )
+
+
 
-# Tests in the package:
-atlas_add_test( flake8
-                SCRIPT flake8 --select=ATL,F,E7,E9,W6,E101 --enable-extension=ATL900,ATL901,ATL902
-                ${CMAKE_CURRENT_SOURCE_DIR}/python ${CMAKE_CURRENT_SOURCE_DIR}/scripts
-                POST_EXEC_SCRIPT nopost.sh )
-
-atlas_add_test( ViewCFTest
-                SCRIPT python -m unittest -v TriggerMenuMT.HLTMenuConfig.Test.ViewCFTest
-                POST_EXEC_SCRIPT nopost.sh )
-
-atlas_add_test( generateMenuMT_newJO
-                SCRIPT python -m TriggerMenuMT.HLTMenuConfig.Menu.LS2_v1_newJO
-                PROPERTIES TIMEOUT 500 )
-
-atlas_add_test( generateMenuMT
-                SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/scripts/test_HLTmenu.sh
-                PROPERTIES TIMEOUT 4000 )
-
-# Test L1 trigger menus:
-function( atlas_test_lvl1_trigger_menu menu )
-   atlas_add_test( "L1_${menu}"
-                   SCRIPT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/generateL1MenuMT.sh
-                   -r ${CMAKE_PROJECT_VERSION} ${menu} ${CMAKE_CURRENT_BINARY_DIR}/Menus/${menu} )
-endfunction()
-
-atlas_test_lvl1_trigger_menu( LS2_v1 )
-atlas_test_lvl1_trigger_menu( Physics_pp_run3_v1 )
-atlas_test_lvl1_trigger_menu( PhysicsP1_pp_run3_v1 )
-atlas_test_lvl1_trigger_menu( MC_pp_run3_v1 )
-atlas_test_lvl1_trigger_menu( Cosmic_run3_v1 )
-atlas_test_lvl1_trigger_menu( PhysicsP1_HI_run3_v1 )
-atlas_test_lvl1_trigger_menu( Dev_HI_run3_v1 )

Trigger/TriggerCommon/TriggerMenuMT/scripts/generateL1MenuMT.sh

@@ -36,8 +36,6 @@ menu=$1
 dest=$2
 if [ -z "$dest" ]; then
     dest=$PWD
-else
-    mkdir -p $dest
 fi
 
 # Get absolute path name

Trigger/TriggerCommon/TriggerMenuMT/scripts/generateMenuMT.py

+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
+# import flags
+doWriteRDOTrigger = False
+doWriteBS = False
+include("TriggerJobOpts/runHLT_standalone.py")  # noqa: F821
+
+#################################
+# Configure L1Decoder
+#################################
+
+# provide a minimal menu information
+from AthenaCommon.GlobalFlags import globalflags
+from AthenaCommon.AlgSequence import AlgSequence
+topSequence = AlgSequence()
+if globalflags.InputFormat.is_bytestream():
+   from AthenaCommon.Constants import DEBUG
+   topSequence.L1Decoder.ctpUnpacker.OutputLevel=DEBUG  # noqa: ATL900
+
+
+
+# this is a temporary hack to include new test chains
+# very hacky indeed, the L1 part should be in in the GenerateMenuMT part
+
+from TriggerJobOpts.TriggerFlags import TriggerFlags
+TriggerFlags.triggerMenuSetup = "LS2_v1"
+
+from TriggerMenuMT.HLTMenuConfig.Menu.GenerateMenuMT import GenerateMenuMT
+g = GenerateMenuMT()
+
+
+
+
+
+##########################################
+# Menu and CF construction
+##########################################
+g.generateMT()
+
+
+
+

Trigger/TriggerCommon/TriggerMenuMT/scripts/testMenuMT.sh

+#!/usr/bin/env sh
+
+help() {
+    echo "Syntax: testMenuMT.sh [-r VERSION] MENU [DEST]"
+}
+
+if [ $# -lt 1 ]; then
+    help
+    exit 0
+fi
+
+while true; do
+    case "$1" in
+        -r)
+            release=$2
+            shift 2
+            ;;
+        -h)
+            help
+            exit 0
+            ;;
+        -*)
+            echo "Invalid parameter"
+            exit 1
+            ;;
+        *)
+            break
+    esac
+done
+
+menu=$1
+dest=$2
+if [ -z "$dest" ]; then
+    dest=$PWD
+fi
+
+# Get absolute path name
+dest=`cd $dest; pwd`
+
+if [ -z "$release" ]; then
+    release=${AtlasVersion}    # for interactive use
+fi
+
+# Temporary run directroy and cleanup traps in case of termination
+rundir=`mktemp -t -d tmxml.${menu}.XXXXXXXXXX`
+TRAPINT() {
+    rm -rf $rundir
+    return 130 # 128+SIGINT
+}
+TRAPTERM() {
+    rm -rf $rundir
+    return 143 # 128+SIGTERM
+}
+
+
+## menu generation starts here
+echo "generateHLTMenuMT: Building menu: ${menu} for ${release}"
+logfiletopo=topo${menu}.log
+logfilelvl1=lvl1${menu}.log
+logfilehlt=hlt${menu}.log
+cd $rundir
+
+generateLVL1MenuMT.py $menu &> $logfiletopo 
+generateL1TopoMenuMT.py $menu &> $logfilelvl1 
+generateMenuMT.py $menu &> $logfilehlt
+
+cp L1Topoconfig_*.xml ${dest}
+cp LVL1config_*.xml ${dest}
+#cp $logfilelvl1 $logfiletopo ${dest}
+
+if [[ -e outputLVL1config.xml ]]; then
+    cp outputLVL1config.xml ${dest}/LVL1config_${menu}_${release}.xml
+fi
+
+#this gives some more sensitivity to hidden problems
+grep --colour ERROR ${dest}/$logfile
+grep --colour -A 100 "Shortened traceback" ${dest}/$logfile
+
+rm -rf $rundir
+
+# Do not return real athena exit code as we want to pretend everything was fine
+#unset PYTHONDONTWRITEBYTECODE

Trigger/TriggerCommon/TriggerMenuMT/scripts/test_HLTmenu.sh

@@ -2,6 +2,6 @@
 
 set -e
 
-athena --config-only=config.pkl -l DEBUG --threads=1 --skipEvents=10 --evtMax=20 --filesInput="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/data17_13TeV.00327265.physics_EnhancedBias.merge.RAW._lb0100._SFO-1._0001.1" -c "setMenu='LS2_v1';doWriteBS=False;from TriggerJobOpts.TriggerFlags import TriggerFlags;TriggerFlags.generateMenuDiagnostics=True" TriggerJobOpts/runHLT_standalone.py
+athena --config-only=config.pkl -l DEBUG --threads=1 --skipEvents=10 --evtMax=20 --filesInput="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/data17_13TeV.00327265.physics_EnhancedBias.merge.RAW._lb0100._SFO-1._0001.1" TriggerMenuMT/generateMT.py;
 
 verify_menu_config.py -f $PWD || { echo "ERROR: Menu verification failed" ; exit 1; }

Trigger/TriggerCommon/TriggerMenuMT/scripts/verify_menu_config.py

@@ -15,7 +15,7 @@ import sys
 import argparse
 
 from AthenaCommon.Logging import logging
-from TriggerMenuMT.menu_config_tests import TriggerLevel, menu_tests
+from menu_config_tests import TriggerLevel, menu_tests
 
 log = logging.getLogger("TriggerMenuConfigTest")
 

Trigger/TriggerCommon/TriggerMenuMT/share/generateMT.py

+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
+setMenu             = "LS2_v1"
+doWriteRDOTrigger   = False
+doWriteBS           = False
+endJobAfterGenerate = False
+from TriggerJobOpts.TriggerFlags import TriggerFlags
+TriggerFlags.generateMenuDiagnostics = True
+include("TriggerJobOpts/runHLT_standalone.py")
+

Trigger/TriggerCommon/TriggerMenuMT/share/trigermenuMT_test.py

+#
+# Minimal example to build a trigger menu
+#
+from TriggerMenuMT.LVL1MenuConfig.TriggerConfigLVL1 import TriggerConfigLVL1            
+from TriggerMenuMT.LVL1MenuConfig.LVL1.Lvl1Flags import Lvl1Flags
+from TriggerMenuMT.LVL1MenuConfig.LVL1.Lvl1MenuItems import LVL1MenuItem
+from TriggerMenuMT.L1.Config.TriggerTypeDef import TT
+from TriggerMenuMT.LVL1MenuConfig.LVL1.Lvl1Condition import ThrCondition, Lvl1InternalTrigger
+from TriggerMenuMT.L1.Base.Logic import Logic
+
+# Define L1 menu thresholds and items
+Lvl1Flags.RemapThresholdsAsListed = True
+Lvl1Flags.thresholds = ['EM3','MU4']
+Lvl1Flags.items = ['L1_EM3','L1_MU4']
+
+l1menu = TriggerConfigLVL1( outputFile = 'l1menu.xml', 
+                            menuName = 'menutest',
+                            topoMenu = None )
+
+LVL1MenuItem.l1configForRegistration = l1menu
+
+# Register L1 menu
+bgrp = Logic(Lvl1InternalTrigger('BGRP0')) & Logic(Lvl1InternalTrigger('BGRP1'))
+
+EM3 = l1menu.registerThr('EM3','EM').addThrValue(3)
+LVL1MenuItem('L1_EM3'       ).setLogic( ThrCondition(EM3) & bgrp ).setTriggerType( TT.calo )
+
+MU4 = l1menu.registerThr('MU4','MUON').addThrValue(4)
+LVL1MenuItem('L1_MU4'       ).setLogic( ThrCondition(MU4) & bgrp ).setTriggerType( TT.muon )
+
+# Generate and write L1 menu
+l1menu.generateMenu()        
+l1menu.writeXML()
+
+
+from TriggerMenu.menu.TriggerPythonConfig import TriggerPythonConfig
+from TriggerMenu.menu.HLTObjects import HLTChain
+hltmenu = TriggerPythonConfig('hltmenu.xml')
+
+from TrigSteeringTest.TrigSteeringTestConf import PESA__dummyAlgo
+defalgo = PESA__dummyAlgo('Em3')
+hltmenu.addSequence("EM3" , defalgo, "em3" )
+
+chain = HLTChain( chain_name="HLT_e3", chain_counter="1", lower_chain_name="EM3", level = "HLT", prescale="1", pass_through="1")
+chain.addHLTSignature( "em3" ).addHLTSignature( "em3" )
+chain.addTriggerTypeBit('4')
+chain.addStreamTag('electrons', prescale='1', obeyLB="yes")
+chain.addStreamTag('IDCalibration', prescale='1', obeyLB="no", type="calibration")
+chain.addGroup("electrons")
+hltmenu.addHLTChain(chain)
+
+hltmenu.writeConfigFiles();
+
+import sys
+sys.exit(0)