From fbb1c483db4ab439dd774e64f131f47fb07577c1 Mon Sep 17 00:00:00 2001
From: Rafal Bielski <rafal.bielski@cern.ch>
Date: Fri, 21 Feb 2020 18:20:41 +0100
Subject: [PATCH] Improve histogram post-processing
Define common handling of the case when OH monitoring is used and adapt the affected tests.
Make the HistMerge step print a warning and exit if input files don't exist.
---
.../TrigP1Test/python/TrigP1TestSteps.py | 40 ++++++++++++++++++-
.../test_trigP1_FullMenu_preload_build.py | 9 ++---
...st_trigP1_HelloWorld_OHMonitoring_build.py | 9 ++---
...test_trigP1_HelloWorld_runStopRun_build.py | 5 ++-
.../test_trigP1_v1PhysP1_preload_build.py | 9 ++---
.../test_trigP1_v1PhysP1_runStopRun_build.py | 5 ++-
.../python/TrigValSteering/CheckSteps.py | 7 ++++
7 files changed, 61 insertions(+), 23 deletions(-)
diff --git a/Trigger/TrigValidation/TrigP1Test/python/TrigP1TestSteps.py b/Trigger/TrigValidation/TrigP1Test/python/TrigP1TestSteps.py
index d858ae6f6ac..73c798c8d3d 100644
--- a/Trigger/TrigValidation/TrigP1Test/python/TrigP1TestSteps.py
+++ b/Trigger/TrigValidation/TrigP1Test/python/TrigP1TestSteps.py
@@ -7,7 +7,7 @@ Definitions of additional validation steps in Trigger ART tests relevant only fo
The main common check steps are defined in the TrigValSteering.CheckSteps module.
'''
-from TrigValTools.TrigValSteering import Step
+from TrigValTools.TrigValSteering import Step, CheckSteps
import os
import inspect
@@ -83,3 +83,41 @@ class TrigBSDumpGrepStep(Step.Step):
if self.auto_report_result:
self.report_result()
return self.result, cmd
+
+
+class ExtractExpertMonitoring(CheckSteps.InputDependentStep):
+ '''
+ Step which extracts the EXPERT directory from an online monitoring file
+ produced by OH server into an offline-like expert-monitoring.root
+ '''
+ def __init__(self, name='ExtractExpertMonitoring'):
+ super(ExtractExpertMonitoring, self).__init__(name)
+ self.input_file = None
+ self.path_prefix = None
+ self.executable = 'rootcp'
+ self.args = '--recreate -r'
+ self.output_stream = Step.Step.OutputStream.STDOUT_ONLY
+
+ def configure(self, test):
+ self.args += ' {:s}:{:s}/HLT-Histogramming/*/EXPERT/* expert-monitoring.root'.format(self.input_file, self.path_prefix or '')
+ super(ExtractExpertMonitoring, self).configure(test)
+
+
+def default_check_steps_OHMon(test, hist_path):
+ steps = []
+ # Extract expert-monitoring.root file from OH server output
+ extract_hist = ExtractExpertMonitoring()
+ hist_path_split = hist_path.split(':')
+ if len(hist_path_split) > 1:
+ extract_hist.input_file = hist_path_split[0]
+ extract_hist.path_prefix = hist_path_split[1]
+ else:
+ extract_hist.input_file = hist_path
+ steps.append(extract_hist)
+ # Default check steps
+ steps.extend(CheckSteps.default_check_steps(test))
+ # Remove histogram merging step
+ matches = [step for step in steps if step.name == 'HistMerge']
+ for hm_step in matches:
+ steps.remove(hm_step)
+ return steps
diff --git a/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_FullMenu_preload_build.py b/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_FullMenu_preload_build.py
index f6014a0db48..a0cc1558ada 100755
--- a/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_FullMenu_preload_build.py
+++ b/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_FullMenu_preload_build.py
@@ -4,7 +4,8 @@
# art-type: build
# art-include: master/Athena
-from TrigValTools.TrigValSteering import Input, Test, Step, ExecStep, CheckSteps
+from TrigValTools.TrigValSteering import Input, Test, Step, ExecStep
+from TrigP1Test import TrigP1TestSteps
# Input file
f = Input.get_input('data').paths[0].encode('ascii', 'ignore')
@@ -45,11 +46,7 @@ ex.perfmon = False # Cannot use PerfMon with -M
test = Test.Test()
test.art_type = 'build'
test.exec_steps = [ex_rm, ex_bs, ex]
-test.check_steps = CheckSteps.default_check_steps(test)
-
-# Overwrite default histogram file name for checks
-for step in [test.get_step(name) for name in ['HistCount', 'RootComp', 'ChainDump']]:
- step.input_file = 'r0000999999_athenaHLT_HLT-Histogramming.root'
+test.check_steps = TrigP1TestSteps.default_check_steps_OHMon(test, 'r0000999999_athenaHLT_HLT-Histogramming.root:run_999999/lb_-1')
import sys
sys.exit(test.run())
diff --git a/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_HelloWorld_OHMonitoring_build.py b/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_HelloWorld_OHMonitoring_build.py
index e4cebd77352..ae8a5a6cf96 100755
--- a/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_HelloWorld_OHMonitoring_build.py
+++ b/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_HelloWorld_OHMonitoring_build.py
@@ -6,7 +6,8 @@
# Skipping art-output which has no effect for build tests.
# If you create a grid version, check art-output in existing grid tests.
-from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps
+from TrigValTools.TrigValSteering import Test, ExecStep
+from TrigP1Test import TrigP1TestSteps
ex = ExecStep.ExecStep()
ex.type = 'athenaHLT'
@@ -18,11 +19,7 @@ ex.perfmon = False # perfmon currently not fully supported with athenaHLT -M
test = Test.Test()
test.art_type = 'build'
test.exec_steps = [ex]
-test.check_steps = CheckSteps.default_check_steps(test)
-
-# Overwrite default histogram file name for checks
-for step in [test.get_step(name) for name in ['HistCount', 'RootComp', 'ChainDump']]:
- step.input_file = 'r0000327265_athenaHLT_HLT-Histogramming.root'
+test.check_steps = TrigP1TestSteps.default_check_steps_OHMon(test, 'r0000360026_athenaHLT_HLT-Histogramming.root:run_360026/lb_-1')
import sys
sys.exit(test.run())
diff --git a/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_HelloWorld_runStopRun_build.py b/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_HelloWorld_runStopRun_build.py
index b967221b0b4..c7c34da1203 100755
--- a/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_HelloWorld_runStopRun_build.py
+++ b/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_HelloWorld_runStopRun_build.py
@@ -6,7 +6,8 @@
# Skipping art-output which has no effect for build tests.
# If you create a grid version, check art-output in existing grid tests.
-from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps
+from TrigValTools.TrigValSteering import Test, ExecStep
+from TrigP1Test import TrigP1TestSteps
ex = ExecStep.ExecStep()
ex.type = 'athenaHLT'
@@ -21,7 +22,7 @@ ex.job_options += ' < `find_data.py run-stop-run-saveHist.trans`'
test = Test.Test()
test.art_type = 'build'
test.exec_steps = [ex]
-test.check_steps = CheckSteps.default_check_steps(test)
+test.check_steps = TrigP1TestSteps.default_check_steps_OHMon(test, 'run_2.root')
# Extra merging pattern for logs produced with -ul option
logmerge = test.get_step("LogMerge")
diff --git a/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_v1PhysP1_preload_build.py b/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_v1PhysP1_preload_build.py
index 876e5d68ea5..9e403f5a01c 100755
--- a/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_v1PhysP1_preload_build.py
+++ b/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_v1PhysP1_preload_build.py
@@ -4,7 +4,8 @@
# art-type: build
# art-include: master/Athena
-from TrigValTools.TrigValSteering import Input, Test, Step, ExecStep, CheckSteps
+from TrigValTools.TrigValSteering import Input, Test, Step, ExecStep
+from TrigP1Test import TrigP1TestSteps
# Input file
f = Input.get_input('data').paths[0].encode('ascii', 'ignore')
@@ -46,11 +47,7 @@ ex.perfmon = False # Cannot use PerfMon with -M
test = Test.Test()
test.art_type = 'build'
test.exec_steps = [ex_rm, ex_bs, ex]
-test.check_steps = CheckSteps.default_check_steps(test)
-
-# Overwrite default histogram file name for checks
-for step in [test.get_step(name) for name in ['HistCount', 'RootComp', 'ChainDump']]:
- step.input_file = 'r0000999999_athenaHLT_HLT-Histogramming.root'
+test.check_steps = TrigP1TestSteps.default_check_steps_OHMon(test, 'r0000999999_athenaHLT_HLT-Histogramming.root:run_999999/lb_-1')
import sys
sys.exit(test.run())
diff --git a/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_v1PhysP1_runStopRun_build.py b/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_v1PhysP1_runStopRun_build.py
index 3e76736bbef..d11a761fb94 100755
--- a/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_v1PhysP1_runStopRun_build.py
+++ b/Trigger/TrigValidation/TrigP1Test/test/test_trigP1_v1PhysP1_runStopRun_build.py
@@ -6,7 +6,8 @@
# Skipping art-output which has no effect for build tests.
# If you create a grid version, check art-output in existing grid tests.
-from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps
+from TrigValTools.TrigValSteering import Test, ExecStep
+from TrigP1Test import TrigP1TestSteps
ex = ExecStep.ExecStep()
ex.type = 'athenaHLT'
@@ -21,7 +22,7 @@ ex.job_options += ' < `find_data.py run-stop-run-saveHist.trans`'
test = Test.Test()
test.art_type = 'build'
test.exec_steps = [ex]
-test.check_steps = CheckSteps.default_check_steps(test)
+test.check_steps = TrigP1TestSteps.default_check_steps_OHMon(test, 'run_2.root')
# Extra merging pattern for logs produced with -ul option
logmerge = test.get_step("LogMerge")
diff --git a/Trigger/TrigValidation/TrigValTools/python/TrigValSteering/CheckSteps.py b/Trigger/TrigValidation/TrigValTools/python/TrigValSteering/CheckSteps.py
index d186e900c51..5cbcbd17794 100644
--- a/Trigger/TrigValidation/TrigValTools/python/TrigValSteering/CheckSteps.py
+++ b/Trigger/TrigValidation/TrigValTools/python/TrigValSteering/CheckSteps.py
@@ -12,6 +12,7 @@ import re
import subprocess
import json
import six
+import glob
from TrigValTools.TrigValSteering.Step import Step
from TrigValTools.TrigValSteering.Common import art_input_eos, art_input_cvmfs
@@ -183,6 +184,12 @@ class RootMergeStep(Step):
old_name = os.path.splitext(self.merged_file)
new_name = old_name[0] + self.rename_suffix + old_name[1]
self.executable = 'mv {} {}; {}'.format(self.merged_file, new_name, self.executable)
+ file_list = self.input_file.split()
+ for file_name in file_list:
+ if len(glob.glob(file_name)) < 1:
+ self.log.warning('%s: file %s requested to be merged but does not exist', self.name, file_name)
+ self.result = 1
+ return self.result, '# (internal) {} in={} out={} -> failed'.format(self.name, self.input_file, self.merged_file)
return super(RootMergeStep, self).run(dry_run)
--
GitLab