Skip to content
Snippets Groups Projects
Commit d3c4be3e authored by Charles Leggett's avatar Charles Leggett
Browse files

test wrapper improvements 

modified the test wrapper script/classes to allow access to stdout and stderr
reference files names from the custom validator code in .qmt files.

This allow to implement easily validators addressing the problem reported
in LHCBPS-1693.

See merge request !281
parents 731c578a c7690f7b
No related branches found
No related tags found
1 merge request!281test wrapper improvements
Pipeline #
Hide whitespace changes
Inline Side-by-side
Showing
with 37 additions and 180 deletions
GaudiExamples/tests/qmtest/newFormat/timing_histos_test.py deleted 100644 → 0
+ 0
− 36
View file @ 731c578a
# -*- coding: utf-8 -*-
import BaseTest
from BaseTest import *
class Test(BaseTest):
def __init__(self):
BaseTest.__init__(self)
self.name = os.path.basename(__file__)[:-5]
self.program="gaudirun.py"
self.args=["$GAUDIEXAMPLESROOT/options/TimingHistograms.py"]
self.reference="refs/TimingHistograms.ref"
def validator(self,stdout,stderr, result, causes, reference, error_reference):
# Default validation.
self.validateWithReference()
# Check the content of the ROOT file.
import os
from subprocess import Popen, PIPE
testscript = os.path.join(os.environ['GAUDIEXAMPLESROOT'], 'tests', 'scripts', 'test_timing_histo_file.py')
test = Popen(['python', testscript], stdout=PIPE, stderr=PIPE)
out, err = test.communicate()
result['root_file_check.returncode'] = str(test.returncode)
if test.returncode:
causes.append('root file content')
if out:
result['root_file_check.stdout'] = result.Quote(out)
if err:
causes.append('standard error')
result['root_file_check.stderr'] = result.Quote(err)
GaudiExamples/tests/qmtest/newFormat/timing_test.py deleted 100644 → 0
+ 0
− 22
View file @ 731c578a
# -*- coding: utf-8 -*-
import BaseTest
from BaseTest import *
class Test(BaseTest):
def __init__(self):
BaseTest.__init__(self)
self.name = os.path.basename(__file__)[:-5]
self.program="gaudirun.py"
self.args=["Timing.py"]
def validator(self,stdout,stderr, result, causes, reference, error_reference):
import re
expected = ( r"Timing\s+SUCCESS\s+The timing is \(in us\)\s*\n" +
r"\|\s*\|\s*#\s*\|\s*Total\s*\|\s*Mean\+-RMS\s*\|\s*Min/Max\s*\|\s*\n" +
r"\|\s*\(1U\)\s*\|\s*\d*\s*\|(?:[-+.\deE /]+\|){3}\s*\n" +
r"\|\s*\(2U\)\s*\|\s*\d*\s*\|(?:[-+.\deE /]+\|){3}\s*\n" +
r"\|\s*\(3U\)\s*\|\s*\d*\s*\|(?:[-+.\deE /]+\|){3}\s*\n" )
if not re.search(expected,stdout):
causes.append("missing regexp match")
result["GaudiTest.regexp"] = result.Quote(expected.replace("\\n","\n"))
\ No newline at end of file
GaudiExamples/tests/qmtest/newFormat/tupleex1_test.py deleted 100644 → 0
+ 0
− 11
View file @ 731c578a
# -*- coding: utf-8 -*-
import BaseTest
from BaseTest import *
class Test(BaseTest):
def __init__(self):
BaseTest.__init__(self)
self.name = os.path.basename(__file__)[:-5]
self.program="../../scripts/TupleEx1.py"
self.reference="refs/TupleEx1.pyref"
\ No newline at end of file
GaudiExamples/tests/qmtest/newFormat/tupleex2_test.py deleted 100644 → 0
+ 0
− 11
View file @ 731c578a
# -*- coding: utf-8 -*-
import BaseTest
from BaseTest import *
class Test(BaseTest):
def __init__(self):
BaseTest.__init__(self)
self.name = os.path.basename(__file__)[:-5]
self.program="../../scripts/TupleEx2.py"
self.reference="refs/TupleEx2.pyref"
\ No newline at end of file
GaudiExamples/tests/qmtest/newFormat/tupleex3_test.py deleted 100644 → 0
+ 0
− 11
View file @ 731c578a
# -*- coding: utf-8 -*-
import BaseTest
from BaseTest import *
class Test(BaseTest):
def __init__(self):
BaseTest.__init__(self)
self.name = os.path.basename(__file__)[:-5]
self.program="../../scripts/TupleEx3.py"
self.reference="refs/TupleEx3.pyref"
\ No newline at end of file
GaudiExamples/tests/qmtest/newFormat/tupleex4_test.py deleted 100644 → 0
+ 0
− 11
View file @ 731c578a
# -*- coding: utf-8 -*-
import BaseTest
from BaseTest import *
class Test(BaseTest):
def __init__(self):
BaseTest.__init__(self)
self.name = os.path.basename(__file__)[:-5]
self.program="../../scripts/TupleEx4.py"
self.reference="refs/TupleEx4.pyref"
\ No newline at end of file
GaudiExamples/tests/qmtest/newFormat/tupleex_py_test.py deleted 100644 → 0
+ 0
− 11
View file @ 731c578a
# -*- coding: utf-8 -*-
import BaseTest
from BaseTest import *
class Test(BaseTest):
def __init__(self):
BaseTest.__init__(self)
self.name = os.path.basename(__file__)[:-5]
self.program="../../scripts/TupleEx.py"
self.reference="refs/TupleEx.pyref"
\ No newline at end of file
GaudiExamples/tests/qmtest/newFormat/tupleex_pyopts_test.py deleted 100644 → 0
+ 0
− 12
View file @ 731c578a
# -*- coding: utf-8 -*-
import BaseTest
from BaseTest import *
class Test(BaseTest):
def __init__(self):
BaseTest.__init__(self)
self.name = os.path.basename(__file__)[:-5]
self.program="gaudirun.py"
self.args=["$GAUDIEXAMPLESROOT/options/TupleEx.py"]
self.reference="refs/TupleExPyOpts.ref"
\ No newline at end of file
GaudiExamples/tests/qmtest/newFormat/tupleex_test.py deleted 100644 → 0
+ 0
− 11
View file @ 731c578a
# -*- coding: utf-8 -*-
import BaseTest
from BaseTest import *
class Test(BaseTest):
def __init__(self):
BaseTest.__init__(self)
self.name = os.path.basename(__file__)[:-5]
self.args=["$GAUDIEXAMPLESROOT/options/TupleEx.opts"]
self.reference="refs/TupleEx.ref"
\ No newline at end of file
GaudiPolicy/python/GaudiTesting/BaseTest.py
+ 5
− 17
View file @ d3c4be3e
......@@ -85,9 +85,6 @@ class BaseTest(object):
self.stack_trace = None
self.basedir = os.getcwd()
def validator(self, stdout='',stderr=''):
pass
def run(self):
logging.debug('running test %s', self.name)
......@@ -253,20 +250,11 @@ class BaseTest(object):
#-------------------------------------------------#
def ValidateOutput(self, stdout, stderr, result):
# checking if default validation or not
if self.validator is not BaseTest.validator:
self.validator(stdout, stderr, result, self.causes,
self.reference, self.error_reference)
else:
if self.stderr == '':
self.validateWithReference(stdout, stderr, result, causes)
elif stderr.strip() != self.stderr.strip():
self.causes.append('standard error')
return result, causes
if not self.stderr:
self.validateWithReference(stdout, stderr, result, self.causes)
elif stderr.strip() != self.stderr.strip():
self.causes.append('standard error')
return result, self.causes
def findReferenceBlock(self,reference=None, stdout=None, result=None, causes=None, signature_offset=0, signature=None, id = None):
"""
......
GaudiPolicy/python/GaudiTesting/QMTTest.py
+ 32
− 27
View file @ d3c4be3e
......@@ -73,39 +73,44 @@ class QMTTest(BaseTest):
return apply(self.callable, args, kwargs)
# local names to be exposed in the script
exported_symbols = {"self":self,
"stdout":stdout,
"stderr":stderr,
"result":result,
"causes":self.causes,
stdout_ref = self._expandReferenceFileName(self.reference)
stderr_ref = self._expandReferenceFileName(self.error_reference)
exported_symbols = {"self": self,
"stdout": stdout,
"stderr": stderr,
"result": result,
"causes": self.causes,
"reference": stdout_ref,
"error_reference": stderr_ref,
"findReferenceBlock":
CallWrapper(self.findReferenceBlock, {"stdout":stdout,
"result":result,
"causes":self.causes}),
CallWrapper(self.findReferenceBlock,
{"stdout": stdout,
"result": result,
"causes": self.causes}),
"validateWithReference":
CallWrapper(self.validateWithReference, {"stdout":stdout,
"stderr":stderr,
"result":result,
"causes":self.causes}),
CallWrapper(self.validateWithReference,
{"stdout": stdout,
"stderr": stderr,
"result": result,
"causes": self.causes}),
"countErrorLines":
CallWrapper(self.countErrorLines, {"stdout":stdout,
"result":result,
"causes":self.causes}),
CallWrapper(self.countErrorLines,
{"stdout": stdout,
"result": result,
"causes": self.causes}),
"checkTTreesSummaries":
CallWrapper(self.CheckTTreesSummaries, {"stdout":stdout,
"result":result,
"causes":self.causes}),
CallWrapper(self.CheckTTreesSummaries,
{"stdout": stdout,
"result": result,
"causes": self.causes}),
"checkHistosSummaries":
CallWrapper(self.CheckHistosSummaries, {"stdout":stdout,
"result":result,
"causes":self.causes})
CallWrapper(self.CheckHistosSummaries,
{"stdout": stdout,
"result": result,
"causes": self.causes})
}
#print self.validator
exec self.validator in globals(), exported_symbols
return result, self.causes
else:
if self.stderr == '':
self.validateWithReference(stdout, stderr, result, self.causes)
elif stderr.strip() != self.stderr.strip():
self.causes.append('standard error')
return result, self.causes
return super(QMTTest, self).ValidateOutput(stdout, stderr, result)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment