test wrapper improvements

modified the test wrapper script/classes to allow access to stdout and stderr
reference files names from the custom validator code in .qmt files.

This allow to implement easily validators addressing the problem reported
in LHCBPS-1693.

See merge request !281

GaudiExamples/tests/qmtest/newFormat/abort_event_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-3]
-        self.program = "gaudirun.py"
-        self.options="""from Gaudi.Configuration import *
-from Configurables import HelloWorld, AbortEventAlg
-
-importOptions("Common.opts")
-
-h1 = HelloWorld("HelloWorld1")
-h2 = HelloWorld("HelloWorld2")
-
-alg = AbortEventAlg(AbortedEventNumber = 3)
-
-app = ApplicationMgr(TopAlg = [h1,alg,h2],
-                     EvtSel = "NONE", EvtMax = 5)
-
-MessageSvc().setDebug.append("EventLoopMgr") """
-
-    def validator(self,stdout,stderr, result, causes, reference, error_reference):
-        self.findReferenceBlock("""
-HelloWorld1          INFO beginning new run....
-HelloWorld2          INFO beginning new run....
-HelloWorld1          INFO executing....
-HelloWorld2          INFO executing....
-HelloWorld1          INFO executing....
-HelloWorld2          INFO executing....
-HelloWorld1          INFO executing....
-EventLoopMgr        DEBUG AbortEvent incident fired by AbortEventAlg
-HelloWorld1          INFO executing....
-HelloWorld2          INFO executing....
-HelloWorld1          INFO executing....
-HelloWorld2          INFO executing....
-HelloWorld1          INFO ending new run....
-HelloWorld2          INFO ending new run....
-""", signature_offset = 7)

GaudiExamples/tests/qmtest/newFormat/aida2root_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-3]
-        self.args=["$GAUDIEXAMPLESROOT/options/Aida2Root.opts"]
-        self.reference = "refs/Aida2Root.ref"
-        self.options="""from Gaudi.Configuration import *
-from Configurables import HelloWorld, AbortEventAlg
-
-importOptions("Common.opts")
-
-h1 = HelloWorld("HelloWorld1")
-h2 = HelloWorld("HelloWorld2")
-
-alg = AbortEventAlg(AbortedEventNumber = 3)
-
-app = ApplicationMgr(TopAlg = [h1,alg,h2],
-                     EvtSel = "NONE", EvtMax = 5)
-
-MessageSvc().setDebug.append("EventLoopMgr") """
-
-    def validator(self,stdout,stderr, result, causes, reference, error_reference):
-        preprocessor = normalizeExamples + RegexpReplacer(when = "^Aida2Root",orig = r"(INFO.*'(skewness|kurtosis)(Err)?'.*)\|([0-9.e+\- ]*\|){2}",repl = r"\1| ### | ### |")
-        self.validateWithReference(preproc = preprocessor)

GaudiExamples/tests/qmtest/newFormat/aida2rootex_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="../../scripts/Aida2RootEx.py"
-        self.args=["-b"]
-        self.reference = "refs/Aida2RootEx.ref"
-        self.error_reference="refs/Aida2RootEx.err.ref"
-
-    def validator(self,stdout,stderr, result, causes, reference, error_reference):
-        preprocessor = (normalizeExamples + LineSkipper(regexps = [r'Info in <TCanvas::Print>: png file .* has been created']))
-        self.validateWithReference(preproc = preprocessor)
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/algerraud_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.args=["$GAUDIEXAMPLESROOT/options/AlgErrAud.opts"]
-        self.reference = "refs/AlgErrAud.ref"
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/algorithn_destructor_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="gaudirun.py"
-        self.args=["-v"]
-        self.options="""from Gaudi.Configuration import *
-from Configurables import GaudiTesting__DestructorCheckAlg as dca;
-
-ApplicationMgr(TopAlg = [ dca("TopAlg"),
-                          Sequencer(Members = [dca("SequencerAlg"), dca("SharedAlg")]),
-                          GaudiSequencer(Members = [dca("GaudiSequencerAlg"), dca("SharedAlg"), dca("SharedAlg2")]),
-                          Sequencer("Sequencer2", Members = [dca("SharedAlg2")]),
-                           ],
-               EvtSel = "NONE",
-               EvtMax = 2)
-"""
-
-    def validator(self,stdout,stderr, result, causes, reference, error_reference):
-        self.findReferenceBlock("Destructor of TopAlg", id = "TopAlgDestructor")
-        self.findReferenceBlock("Destructor of SequencerAlg", id = "SequencerAlgDestructor")
-        self.findReferenceBlock("Destructor of GaudiSequencerAlg", id = "GaudiSequencerAlgDestructor")
-        self.findReferenceBlock("Destructor of SharedAlg", id = "SharedAlgDestructor")
-        self.findReferenceBlock("Destructor of SharedAlg2", id = "SharedAlg2Destructor")
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/algsequencer_pyopts_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="gaudirun.py"
-        self.args=["$GAUDIEXAMPLESROOT/options/AlgSequencer.py"]
-        self.reference = "refs/AlgSequencer_pyopts.ref"
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/algsequencer_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.args=["$GAUDIEXAMPLESROOT/options/AlgSequencer.opts"]
-        self.reference = "refs/AlgSequencer.ref"

GaudiExamples/tests/qmtest/newFormat/algtools2_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.args=["$GAUDIEXAMPLESROOT/options/AlgTools2.opts"]
-        self.reference = "refs/AlgTools2.ref"
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/algtools_pyopts_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="gaudirun.py"
-        self.args=["$GAUDIEXAMPLESROOT/options/AlgTools.py"]
-        self.reference = "refs/AlgTools_pyopts.ref"

GaudiExamples/tests/qmtest/newFormat/algtools_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.args=["$GAUDIEXAMPLESROOT/options/AlgTools.opts"]
-        self.reference = "refs/AlgTools.ref"
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/algtypealiases_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="gaudirun.py"
-        self.reference="refs/AlgTypeAliases.ref"
-        self.options="""from Gaudi.Configuration import *
-
-from Configurables import ApplicationMgr, HelloWorld, SubAlg
-
-app = ApplicationMgr(EvtSel='NONE', EvtMax=4)
-
-app.TopAlg = ['HelloWorld/HW1', 'unalias:HelloWorld/HW2' ]
-
-app.AlgTypeAliases['HelloWorld'] = 'SubAlg'"""

GaudiExamples/tests/qmtest/newFormat/array_properties_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="../../scripts/ArrayProperties.py"
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/auditors_user_timing_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="gaudirun.py"
-        self.args=["-v"]
-        self.options="""
-from Gaudi.Configuration import *
-
-from Configurables import AuditorTestAlg, GaudiExamples__LoggingAuditor as LoggingAuditor
-
-importOptions("Common.opts")
-
-AuditorSvc().Auditors += [ TimingAuditor("TIMER"),
-                           LoggingAuditor("LoggingAuditor") ]
-
-app = ApplicationMgr(TopAlg = [AuditorTestAlg()],
-                     EvtSel = "NONE", EvtMax = 5)
-
-app.ExtSvc += [ ToolSvc(), AuditorSvc() ]
-app.AuditAlgorithms = True
-
-MessageSvc().setDebug.append("EventLoopMgr") """
-
-    def validator(self,stdout,stderr, result, causes, reference, error_reference):
-        import re
-        if not re.search("TIMER.TIMER *INFO AuditorTestAlg:loop",stdout):
-            causes.append("missing timing reoprt")
-            result["GaudiTest.expected"] = result.Quote("timing report for timer 'AuditorTestAlg:loop'")
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/boost_array_properties_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="../../scripts/BoostArrayProperties.py"
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/bug_30209_namespace_in_sequencer_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="gaudirun.py"
-        self.options="""
-from Gaudi.Configuration import *
-from Configurables import GaudiExamples__CounterSvcAlg
-
-counter2 = GaudiExamples__CounterSvcAlg("Counter2",CounterBaseName = "Test2")
-
-seq = Sequencer(Members = [ GaudiExamples__CounterSvcAlg.getType(),
-                            counter2,
-                            "GaudiExamples::StatSvcAlg:invert" ])
-
-ApplicationMgr(TopAlg = [seq], EvtSel = "NONE", EvtMax = 1)"""
-
-    def validator(self,stdout,stderr, result, causes, reference, error_reference):
-        expected = "Sequencer            INFO Member list: GaudiExamples::CounterSvcAlg, GaudiExamples::CounterSvcAlg/Counter2, GaudiExamples::StatSvcAlg:invert"
-        if not expected in stdout:
-            causes.append("missing signature")
-            result["GaudiTest.expected_line"] = result.Quote(expected)
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/bug_34121_tool_properties_in_python_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="../../scripts/bug_34121.py"
-
-    def validator(self,stdout,stderr, result, causes, reference, error_reference):
-        self.findReferenceBlock('''
-ApplicationMgr       INFO Application Manager Initialized successfully
-ApplicationMgr       INFO Application Manager Started successfully
-Alg                  INFO executing....
-Alg.Tool             INFO Double = 0
-Alg                  INFO executing....
-Alg.Tool             INFO Double = 1
-Alg                  INFO executing....
-Alg.Tool             INFO Double = 2
-Alg                  INFO executing....
-Alg.Tool             INFO Double = 3
-Alg                  INFO executing....
-Alg.Tool             INFO Double = 4
-ApplicationMgr       INFO Application Manager Stopped successfully
-Alg                  INFO finalizing....
-''')
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/bug_35913_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="gaudirun.py"
-        self.args=["-v", "--dry-run"]
-        self.options="""
-from Gaudi.Configuration import *
-from Configurables import Gaudi__Examples__HistoProps
-h = Gaudi__Examples__HistoProps()
-h.Histo1 = ('MyHistogram',-1,1,100)
-h.Histo1 = ('MyHistogram',-5,5,500)"""
-
-    def validator(self,stdout,stderr, result, causes, reference, error_reference):
-        conf_dict = eval('\n'.join(stdout.splitlines()))
-        expected = {'Gaudi::Examples::HistoProps':
-                        {'Histo1': ('MyHistogram', -5, 5, 500)}}
-
-        if conf_dict != expected:
-            causes.append("configuration result")
-            from pprint import PrettyPrinter
-            pp = PrettyPrinter()
-            result["GaudiTest.configuration.expected"] = result.Quote(pp.pformat(expected))
-            result["GaudiTest.configuration.found"] = result.Quote(pp.pformat(conf_dict))
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/bug_38882_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="../../scripts/bug_38882.py"
-
-    def validator(self,stdout,stderr, result, causes, reference, error_reference):
-        expected = "=== bug_38882_test_alg Execute ==="
-        if not expected in stdout:
-            causes.append("missing signature")
-            result["GaudiTest.expected_line"] = result.Quote(expected)
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/bug_41136_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="gaudirun.py"
-        self.args=["-v"]
-        self.exit_code=1
-        self.options="""
-from Gaudi.Configuration import *
-from Configurables import TestToolFailing, TestToolAlgFailure
-
-tta1 = TestToolAlgFailure("TestAlg1", Tools = [ TestToolFailing("Tool") ], IgnoreFailure = True)
-tta2 = TestToolAlgFailure("TestAlg2", Tools = [ TestToolFailing("Tool") ], IgnoreFailure = False)
-
-ApplicationMgr(TopAlg = [ tta1, tta2 ],
-               EvtSel = "NONE",
-               EvtMax = 1)
-"""
-
-    def validator(self,stdout,stderr, result, causes, reference, error_reference):
-        self.countErrorLines({"FATAL":2,
-                         "ERROR":7,
-                         })
\ No newline at end of file

GaudiExamples/tests/qmtest/newFormat/cmdline_conf_test.py

-# -*- coding: utf-8 -*-
-import BaseTest
-from BaseTest import *
-
-class Test(BaseTest):
-
-    def __init__(self):
-        BaseTest.__init__(self)
-        self.name = os.path.basename(__file__)[:-5]
-        self.program="gaudirun.py"
-        self.args=["-v","-n","--option","MessageSvc().setFatal.append('AnotherAlg')"]
-        self.options="""
-from Gaudi.Configuration import *
-MessageSvc().setDebug.append("AnAlgorithm")"""
-
-    def validator(self,stdout,stderr, result, causes, reference, error_reference):
-        conf_dict = eval('\n'.join(stdout.splitlines()))
-        expected = {'MessageSvc': {'setDebug': ['AnAlgorithm'], 'setFatal': ['AnotherAlg']}}
-
-        if conf_dict != expected:
-            causes.append("configuration result")
-            from pprint import PrettyPrinter
-            pp = PrettyPrinter()
-            result["GaudiTest.configuration.expected"] = result.Quote(pp.pformat(expected))
-            result["GaudiTest.configuration.found"] = result.Quote(pp.pformat(conf_dict))
\ No newline at end of file