Commit fbb1c483 authored by Rafal Bielski's avatar Rafal Bielski
Browse files

Improve histogram post-processing

Define common handling of the case when OH monitoring is used and adapt the affected tests.

Make the HistMerge step print a warning and exit if input files don't exist.
parent 21de010e
......@@ -7,7 +7,7 @@ Definitions of additional validation steps in Trigger ART tests relevant only fo
The main common check steps are defined in the TrigValSteering.CheckSteps module.
'''
from TrigValTools.TrigValSteering import Step
from TrigValTools.TrigValSteering import Step, CheckSteps
import os
import inspect
......@@ -83,3 +83,41 @@ class TrigBSDumpGrepStep(Step.Step):
if self.auto_report_result:
self.report_result()
return self.result, cmd
class ExtractExpertMonitoring(CheckSteps.InputDependentStep):
'''
Step which extracts the EXPERT directory from an online monitoring file
produced by OH server into an offline-like expert-monitoring.root
'''
def __init__(self, name='ExtractExpertMonitoring'):
super(ExtractExpertMonitoring, self).__init__(name)
self.input_file = None
self.path_prefix = None
self.executable = 'rootcp'
self.args = '--recreate -r'
self.output_stream = Step.Step.OutputStream.STDOUT_ONLY
def configure(self, test):
self.args += ' {:s}:{:s}/HLT-Histogramming/*/EXPERT/* expert-monitoring.root'.format(self.input_file, self.path_prefix or '')
super(ExtractExpertMonitoring, self).configure(test)
def default_check_steps_OHMon(test, hist_path):
steps = []
# Extract expert-monitoring.root file from OH server output
extract_hist = ExtractExpertMonitoring()
hist_path_split = hist_path.split(':')
if len(hist_path_split) > 1:
extract_hist.input_file = hist_path_split[0]
extract_hist.path_prefix = hist_path_split[1]
else:
extract_hist.input_file = hist_path
steps.append(extract_hist)
# Default check steps
steps.extend(CheckSteps.default_check_steps(test))
# Remove histogram merging step
matches = [step for step in steps if step.name == 'HistMerge']
for hm_step in matches:
steps.remove(hm_step)
return steps
......@@ -4,7 +4,8 @@
# art-type: build
# art-include: master/Athena
from TrigValTools.TrigValSteering import Input, Test, Step, ExecStep, CheckSteps
from TrigValTools.TrigValSteering import Input, Test, Step, ExecStep
from TrigP1Test import TrigP1TestSteps
# Input file
f = Input.get_input('data').paths[0].encode('ascii', 'ignore')
......@@ -45,11 +46,7 @@ ex.perfmon = False # Cannot use PerfMon with -M
test = Test.Test()
test.art_type = 'build'
test.exec_steps = [ex_rm, ex_bs, ex]
test.check_steps = CheckSteps.default_check_steps(test)
# Overwrite default histogram file name for checks
for step in [test.get_step(name) for name in ['HistCount', 'RootComp', 'ChainDump']]:
step.input_file = 'r0000999999_athenaHLT_HLT-Histogramming.root'
test.check_steps = TrigP1TestSteps.default_check_steps_OHMon(test, 'r0000999999_athenaHLT_HLT-Histogramming.root:run_999999/lb_-1')
import sys
sys.exit(test.run())
......@@ -6,7 +6,8 @@
# Skipping art-output which has no effect for build tests.
# If you create a grid version, check art-output in existing grid tests.
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps
from TrigValTools.TrigValSteering import Test, ExecStep
from TrigP1Test import TrigP1TestSteps
ex = ExecStep.ExecStep()
ex.type = 'athenaHLT'
......@@ -18,11 +19,7 @@ ex.perfmon = False # perfmon currently not fully supported with athenaHLT -M
test = Test.Test()
test.art_type = 'build'
test.exec_steps = [ex]
test.check_steps = CheckSteps.default_check_steps(test)
# Overwrite default histogram file name for checks
for step in [test.get_step(name) for name in ['HistCount', 'RootComp', 'ChainDump']]:
step.input_file = 'r0000327265_athenaHLT_HLT-Histogramming.root'
test.check_steps = TrigP1TestSteps.default_check_steps_OHMon(test, 'r0000360026_athenaHLT_HLT-Histogramming.root:run_360026/lb_-1')
import sys
sys.exit(test.run())
......@@ -6,7 +6,8 @@
# Skipping art-output which has no effect for build tests.
# If you create a grid version, check art-output in existing grid tests.
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps
from TrigValTools.TrigValSteering import Test, ExecStep
from TrigP1Test import TrigP1TestSteps
ex = ExecStep.ExecStep()
ex.type = 'athenaHLT'
......@@ -21,7 +22,7 @@ ex.job_options += ' < `find_data.py run-stop-run-saveHist.trans`'
test = Test.Test()
test.art_type = 'build'
test.exec_steps = [ex]
test.check_steps = CheckSteps.default_check_steps(test)
test.check_steps = TrigP1TestSteps.default_check_steps_OHMon(test, 'run_2.root')
# Extra merging pattern for logs produced with -ul option
logmerge = test.get_step("LogMerge")
......
......@@ -4,7 +4,8 @@
# art-type: build
# art-include: master/Athena
from TrigValTools.TrigValSteering import Input, Test, Step, ExecStep, CheckSteps
from TrigValTools.TrigValSteering import Input, Test, Step, ExecStep
from TrigP1Test import TrigP1TestSteps
# Input file
f = Input.get_input('data').paths[0].encode('ascii', 'ignore')
......@@ -46,11 +47,7 @@ ex.perfmon = False # Cannot use PerfMon with -M
test = Test.Test()
test.art_type = 'build'
test.exec_steps = [ex_rm, ex_bs, ex]
test.check_steps = CheckSteps.default_check_steps(test)
# Overwrite default histogram file name for checks
for step in [test.get_step(name) for name in ['HistCount', 'RootComp', 'ChainDump']]:
step.input_file = 'r0000999999_athenaHLT_HLT-Histogramming.root'
test.check_steps = TrigP1TestSteps.default_check_steps_OHMon(test, 'r0000999999_athenaHLT_HLT-Histogramming.root:run_999999/lb_-1')
import sys
sys.exit(test.run())
......@@ -6,7 +6,8 @@
# Skipping art-output which has no effect for build tests.
# If you create a grid version, check art-output in existing grid tests.
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps
from TrigValTools.TrigValSteering import Test, ExecStep
from TrigP1Test import TrigP1TestSteps
ex = ExecStep.ExecStep()
ex.type = 'athenaHLT'
......@@ -21,7 +22,7 @@ ex.job_options += ' < `find_data.py run-stop-run-saveHist.trans`'
test = Test.Test()
test.art_type = 'build'
test.exec_steps = [ex]
test.check_steps = CheckSteps.default_check_steps(test)
test.check_steps = TrigP1TestSteps.default_check_steps_OHMon(test, 'run_2.root')
# Extra merging pattern for logs produced with -ul option
logmerge = test.get_step("LogMerge")
......
......@@ -12,6 +12,7 @@ import re
import subprocess
import json
import six
import glob
from TrigValTools.TrigValSteering.Step import Step
from TrigValTools.TrigValSteering.Common import art_input_eos, art_input_cvmfs
......@@ -183,6 +184,12 @@ class RootMergeStep(Step):
old_name = os.path.splitext(self.merged_file)
new_name = old_name[0] + self.rename_suffix + old_name[1]
self.executable = 'mv {} {}; {}'.format(self.merged_file, new_name, self.executable)
file_list = self.input_file.split()
for file_name in file_list:
if len(glob.glob(file_name)) < 1:
self.log.warning('%s: file %s requested to be merged but does not exist', self.name, file_name)
self.result = 1
return self.result, '# (internal) {} in={} out={} -> failed'.format(self.name, self.input_file, self.merged_file)
return super(RootMergeStep, self).run(dry_run)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment