Skip to content
Snippets Groups Projects
Commit a557a927 authored by Adam Edward Barton's avatar Adam Edward Barton :speech_balloon:
Browse files

Merge branch 'athenaHLT_ctest' into 'master' 

athenaHLT: allow concurrent execution and other improvements

See merge request atlas/athena!21489
parents a9a7ac0a 456b0d1d
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 61 additions and 52 deletions
HLT/Trigger/TrigControl/TrigCommon/CMakeLists.txt
+ 4
− 4
View file @ a557a927
......@@ -24,8 +24,8 @@ atlas_add_test( test_AthHLT
POST_EXEC_SCRIPT nopost.sh
PROPERTIES TIMEOUT 300 )
#atlas_add_test( athenaHLT_jo_noopts
# SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/share/test_athenaHLT.sh dummy.py)
atlas_add_test( athenaHLT_jo_noopts
SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/share/test_athenaHLT.sh dummy.py)
#atlas_add_test( athenaHLT_jo_allopts
# SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/share/test_athenaHLT.sh --number-of-events 3 --skip-events 7 -l ERROR,FATAL -c 'x=1' -c 'y=2' -C 'x=2' -o out.data --threads 2 --nprocs 4 --timeout 777 --python-setup mysetup.py --partition mypart dummy.py)
atlas_add_test( athenaHLT_jo_allopts
SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/share/test_athenaHLT.sh --number-of-events 3 --skip-events 7 -l ERROR,FATAL -c 'x=1' -c 'y=2' -C 'x=2' -o out.data --threads 2 --nprocs 4 --timeout 777 --oh-monitoring --python-setup mysetup.py --partition mypart dummy.py)
HLT/Trigger/TrigControl/TrigCommon/bin/athenaHLT.py
+ 4
− 6
View file @ a557a927
......@@ -141,18 +141,16 @@ def set_athena_flags(args):
jp.ConcurrencyFlags.NumProcs = args.nprocs
def HLTMPPy_cfgdict(args):
"""
Create the configuration dictionary as expected by HLTMPPy as defined in
https://gitlab.cern.ch/atlas-tdaq-software/HLTMPPU/blob/master/python/HLTMPPy/runner.py
"""
"""Create the configuration dictionary as expected by HLTMPPy as defined in
https://gitlab.cern.ch/atlas-tdaq-software/HLTMPPU/blob/master/python/HLTMPPy/runner.py"""
cdict = {}
cdict['HLTMPPU'] = {
'application_name' : 'athenaHLT',
'application_name' : 'athenaHLT-%d' % os.getpid(), # unique name required to avoid interference
'extra_params' : None,
'interactive' : args.interactive,
'log_root' : os.getcwd(),
'log_name' : ('' if args.unique_log_files else 'worker'),
'log_name' : ('' if args.unique_log_files else 'athenaHLT:'),
'module' : 'HLTMPPU',
'num_forks' : args.nprocs,
'num_threads' : args.threads,
......
HLT/Trigger/TrigControl/TrigCommon/python/AthHLT.py
+ 21
− 3
View file @ a557a927
# Copyright (C) 2002-2018 CERN for the benefit of the ATLAS collaboration
# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
#
# Utilities used in athenaHLT.py
#
......@@ -23,8 +23,20 @@ class CondDB:
@memoize
def get_sor_params(run_number):
import cPickle as pickle
cool_cache = 'AthHLT.sor.pkl'
log.info('Reading SOR record for run %s from COOL' % run_number)
try:
# Try to load the SOR record from the file cache
d = pickle.load(open(cool_cache, 'rb'))
if d['RunNumber'] != run_number:
raise Exception('Cache does not contain current run')
log.info('Reading cached SOR record for run %s from %s' % (run_number, cool_cache))
return d
except Exception as e:
d = {}
log.verbose('Could not read SOR reacord from cache: %s' % e)
log.info('Reading SOR record for run %s from COOL' % run_number)
from CoolConvUtilities import AtlCoolLib
cdb = CondDB(run_number)
......@@ -41,7 +53,13 @@ def get_sor_params(run_number):
return None # This can happen for unknown run numbers
payload = sor.payload()
return {k: payload[k] for k in payload}
d = {k: payload[k] for k in payload}
try:
pickle.dump(d, open(cool_cache, 'wb'))
except Exception:
log.info('Could not store SOR record in cache %s' % cool_cache)
return d
#
......
HLT/Trigger/TrigControl/TrigCommon/share/athenaHLT_jo_allopts.ref
+ 13
− 18
View file @ a557a927
<Configuration>
<HLTMPPUApplication>
<childLogName>athenaHLT:</childLogName>
<numForks>4</numForks>
<childLogRoot>/scratch/fwinkl/athenaHLT/HLT/Trigger/TrigControl/TrigCommon/unitTestRun</childLogRoot>
<InfoServiceLibrary>MonSvcInfoService</InfoServiceLibrary>
<softTimeoutFraction>0.9</softTimeoutFraction>
<HLTImplementationLibraries>
<library>TrigServices</library>
<library>TrigPSC</library>
</HLTImplementationLibraries>
<UID>HLTMPPy</UID>
<HardTimeout>777</HardTimeout>
<finalizeTimeout>120</finalizeTimeout>
<numberOfEventSlots>2</numberOfEventSlots>
<DataSourceLibrary>DFFileBackend</DataSourceLibrary>
<DataSourceLibrary>DFDcmEmuBackend</DataSourceLibrary>
<numberOfAthenaMTThreads>2</numberOfAthenaMTThreads>
<DataSource>
<HLTDFFileBackend>
<preload>False</preload>
<numEvents>3</numEvents>
<start_id>1</start_id>
<library>DFFileBackend</library>
<library>DFDcmEmuBackend</library>
<fileOffset>-1</fileOffset>
<UID>DataSource-is-DCM</UID>
<compressionLevel>2</compressionLevel>
......@@ -30,7 +29,7 @@
<fileList>
<file>dummy.data</file>
</fileList>
<extraL1Robs/>
<extraL1Robs />
</HLTDFFileBackend>
</DataSource>
<InfoService>
......@@ -42,7 +41,7 @@
<UID>HltpuConfigurationRuleBundle</UID>
<Rules>
<ConfigurationRule>
<ExcludeFilter></ExcludeFilter>
<ExcludeFilter />
<Name>Dumm</Name>
<UID>HltpuOHRule</UID>
<Parameters>
......@@ -57,7 +56,7 @@
<IncludeFilter>.*</IncludeFilter>
</ConfigurationRule>
<ConfigurationRule>
<ExcludeFilter></ExcludeFilter>
<ExcludeFilter />
<Name>DummDumm</Name>
<UID>HltpuISRule</UID>
<Parameters>
......@@ -78,7 +77,6 @@
</HLTMPPUApplication>
<Partition>
<UID>mypart</UID>
<LogRoot>/scratch/fwinkl/athenaHLT/HLT/Trigger/TrigControl/TrigCommon/unitTestRun</LogRoot>
<TriggerConfiguration>
<TriggerConfiguration>
<L1TriggerConfiguration>
......@@ -99,24 +97,21 @@
<preCommand>x=1</preCommand>
<preCommand>y=2</preCommand>
<preCommand>_run_number=327265</preCommand>
<preCommand>include('TrigServices/OfflineTHistSvc.py')</preCommand>
</preCommands>
<showInclude>false</showInclude>
<postCommands>
<postCommand>x=2</postCommand>
</postCommands>
<UID>HLTImplementationJobOptions-1</UID>
<evtSel>NONE</evtSel>
<HLTCommonParameters>
<HLTCommonParameters>
<factoryName></factoryName>
<dllName></dllName>
<messageSvcType>TrigMessageSvc</messageSvcType>
<jobOptionsSvcType>JobOptionsSvc</jobOptionsSvcType>
</HLTCommonParameters>
</HLTCommonParameters>
<logLevels>
<logLevel>INFO</logLevel>
<logLevel>ERROR</logLevel>
<logLevel>FATAL</logLevel>
</logLevels>
<pythonSetupFile>mysetup.py</pythonSetupFile>
<jobOptionsPath>dummy.py</jobOptionsPath>
......@@ -128,8 +123,8 @@
<Name>dummy</Name>
<Server>TRIGGERDB</Server>
<Alias>TRIGGERDB</Alias>
<User></User>
<Password></Password>
<User />
<Password />
<Type>Coral</Type>
<SuperMasterKey>0</SuperMasterKey>
</TriggerDBConnection>
......@@ -137,7 +132,7 @@
</TriggerConfiguration>
</TriggerConfiguration>
</Partition>
<ROS2ROBS/>
<ROS2ROBS />
</Configuration>
<RunParams>
<totalTime>0</totalTime>
......@@ -145,12 +140,12 @@
<timeEOR>1/1/70 01:00:00</timeEOR>
<det_mask>00000000000000000001fffffffffff7</det_mask>
<beam_type>0</beam_type>
<filename_tag></filename_tag>
<filename_tag />
<max_events>0</max_events>
<run_number>327265</run_number>
<timeSOR>22/11/18 11:02:32.703856</timeSOR>
<trigger_type>0</trigger_type>
<T0_project_tag></T0_project_tag>
<T0_project_tag />
<beam_energy>0</beam_energy>
<recording_enabled>0</recording_enabled>
</RunParams>
......
HLT/Trigger/TrigControl/TrigCommon/share/athenaHLT_jo_noopts.ref
+ 13
− 18
View file @ a557a927
<Configuration>
<HLTMPPUApplication>
<childLogName>athenaHLT:</childLogName>
<numForks>1</numForks>
<childLogRoot>/scratch/fwinkl/athenaHLT/HLT/Trigger/TrigControl/TrigCommon/unitTestRun</childLogRoot>
<InfoServiceLibrary>MonSvcInfoService</InfoServiceLibrary>
<softTimeoutFraction>0.9</softTimeoutFraction>
<HLTImplementationLibraries>
<library>TrigServices</library>
<library>TrigPSC</library>
</HLTImplementationLibraries>
<UID>HLTMPPy</UID>
<HardTimeout>36000</HardTimeout>
<finalizeTimeout>120</finalizeTimeout>
<numberOfEventSlots>1</numberOfEventSlots>
<DataSourceLibrary>DFFileBackend</DataSourceLibrary>
<DataSourceLibrary>DFDcmEmuBackend</DataSourceLibrary>
<numberOfAthenaMTThreads>1</numberOfAthenaMTThreads>
<DataSource>
<HLTDFFileBackend>
<preload>False</preload>
<numEvents>-1</numEvents>
<start_id>1</start_id>
<library>DFFileBackend</library>
<library>DFDcmEmuBackend</library>
<fileOffset>-1</fileOffset>
<UID>DataSource-is-DCM</UID>
<compressionLevel>2</compressionLevel>
......@@ -29,7 +28,7 @@
<fileList>
<file>dummy.data</file>
</fileList>
<extraL1Robs/>
<extraL1Robs />
</HLTDFFileBackend>
</DataSource>
<InfoService>
......@@ -41,7 +40,7 @@
<UID>HltpuConfigurationRuleBundle</UID>
<Rules>
<ConfigurationRule>
<ExcludeFilter></ExcludeFilter>
<ExcludeFilter />
<Name>Dumm</Name>
<UID>HltpuOHRule</UID>
<Parameters>
......@@ -56,7 +55,7 @@
<IncludeFilter>.*</IncludeFilter>
</ConfigurationRule>
<ConfigurationRule>
<ExcludeFilter></ExcludeFilter>
<ExcludeFilter />
<Name>DummDumm</Name>
<UID>HltpuISRule</UID>
<Parameters>
......@@ -77,7 +76,6 @@
</HLTMPPUApplication>
<Partition>
<UID>athenaHLT</UID>
<LogRoot>/scratch/fwinkl/athenaHLT/HLT/Trigger/TrigControl/TrigCommon/unitTestRun</LogRoot>
<TriggerConfiguration>
<TriggerConfiguration>
<L1TriggerConfiguration>
......@@ -96,15 +94,12 @@
</libraries>
<preCommands>
<preCommand>_run_number=327265</preCommand>
<preCommand>include('TrigServices/OfflineTHistSvc.py')</preCommand>
</preCommands>
<postCommands/>
<showInclude>false</showInclude>
<postCommands />
<UID>HLTImplementationJobOptions-1</UID>
<evtSel>NONE</evtSel>
<HLTCommonParameters>
<HLTCommonParameters>
<factoryName></factoryName>
<dllName></dllName>
<messageSvcType>TrigMessageSvc</messageSvcType>
<jobOptionsSvcType>JobOptionsSvc</jobOptionsSvcType>
</HLTCommonParameters>
......@@ -123,8 +118,8 @@
<Name>dummy</Name>
<Server>TRIGGERDB</Server>
<Alias>TRIGGERDB</Alias>
<User></User>
<Password></Password>
<User />
<Password />
<Type>Coral</Type>
<SuperMasterKey>0</SuperMasterKey>
</TriggerDBConnection>
......@@ -132,7 +127,7 @@
</TriggerConfiguration>
</TriggerConfiguration>
</Partition>
<ROS2ROBS/>
<ROS2ROBS />
</Configuration>
<RunParams>
<totalTime>0</totalTime>
......@@ -140,12 +135,12 @@
<timeEOR>1/1/70 01:00:00</timeEOR>
<det_mask>00000000000000000001fffffffffff7</det_mask>
<beam_type>0</beam_type>
<filename_tag></filename_tag>
<filename_tag />
<max_events>0</max_events>
<run_number>327265</run_number>
<timeSOR>22/11/18 11:02:32.703856</timeSOR>
<trigger_type>0</trigger_type>
<T0_project_tag></T0_project_tag>
<T0_project_tag />
<beam_energy>0</beam_energy>
<recording_enabled>0</recording_enabled>
</RunParams>
......
HLT/Trigger/TrigControl/TrigCommon/share/test_athenaHLT.sh
+ 6
− 3
View file @ a557a927
#!/usr/bin/env bash
# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
#
# Configuration testing script for athenaHLT.py
#
......@@ -9,8 +10,8 @@ test_options=$@
trap cleanup INT TERM EXIT
function cleanup {
sid=`ps -o sess= -p $$` # our own session id
pkill -9 --parent 1 --session $sid ipc_server
pkill -9 --parent 1 --session $sid is_server
pkill -9 -P 1 -s $sid ipc_server
pkill -9 -P 1 -s $sid is_server
}
# We only test the configuration stage, so these dummy values are just fine.
......@@ -22,4 +23,6 @@ detmask="00000000000000000001fffffffffff7"
# Run only config stage (exit immediately via interactive mode) and filter final ptree
# If there was a failure, the exit code will be non-zero
echo "e" | athenaHLT.py --file ${file} --detector-mask ${detmask} --run-number ${run} --sor-time ${sortime} --interactive ${test_options} | sed -n '/<Configuration>/,/<\/Magnets>/p;/<\/Magnets>/q' | grep '<'
log=test_athenaHLT-${BASHPID}
echo "e" | athenaHLT.py --file ${file} --detector-mask ${detmask} --run-number ${run} --sor-time ${sortime} --interactive ${test_options} &> $log
cat $log | sed -n '/<Configuration>/,/<\/Magnets>/p;/<\/Magnets>/q' | grep '<' | grep -v 'LogRoot'
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment