BaseTest.py 45.5 KB
Newer Older
1
# -*- coding: utf-8 -*-
2
3

import os
Marco Clemencic's avatar
Marco Clemencic committed
4
5
import sys
import time
6
7
8
9
10
11
import signal
import threading
import platform
import tempfile
import inspect
import re
12
import logging
13

14
from subprocess import Popen, PIPE, STDOUT
Marco Clemencic's avatar
Marco Clemencic committed
15

Marco Clemencic's avatar
Marco Clemencic committed
16

Marco Clemencic's avatar
Marco Clemencic committed
17
18
19
20
21
22
23
24
def sanitize_for_xml(data):
    '''
    Take a string with invalid ASCII/UTF characters and quote them so that the
    string can be used in an XML text.

    >>> sanitize_for_xml('this is \x1b')
    'this is [NON-XML-CHAR-0x1B]'
    '''
Marco Clemencic's avatar
Marco Clemencic committed
25
26
27
    bad_chars = re.compile(
        u'[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]')

Marco Clemencic's avatar
Marco Clemencic committed
28
29
30
    def quote(match):
        'helper function'
        return ''.join('[NON-XML-CHAR-0x%2X]' % ord(c) for c in match.group())
Gitlab CI's avatar
Gitlab CI committed
31

Marco Clemencic's avatar
Marco Clemencic committed
32
33
    return bad_chars.sub(quote, data)

Marco Clemencic's avatar
Marco Clemencic committed
34

35
36
37
38
39
40
41
42
def dumpProcs(name):
    '''helper to debug GAUDI-1084, dump the list of processes'''
    from getpass import getuser
    if 'WORKSPACE' in os.environ:
        p = Popen(['ps', '-fH', '-U', getuser()], stdout=PIPE)
        with open(os.path.join(os.environ['WORKSPACE'], name), 'w') as f:
            f.write(p.communicate()[0])

Marco Clemencic's avatar
Marco Clemencic committed
43

44
45
46
47
48
49
50
51
52
53
54
55
56
57
def kill_tree(ppid, sig):
    '''
    Send a signal to a process and all its child processes (starting from the
    leaves).
    '''
    log = logging.getLogger('kill_tree')
    ps_cmd = ['ps', '--no-headers', '-o', 'pid', '--ppid', str(ppid)]
    get_children = Popen(ps_cmd, stdout=PIPE, stderr=PIPE)
    children = map(int, get_children.communicate()[0].split())
    for child in children:
        kill_tree(child, sig)
    try:
        log.debug('killing process %d', ppid)
        os.kill(ppid, sig)
58
    except OSError as err:
Marco Clemencic's avatar
Marco Clemencic committed
59
        if err.errno != 3:  # No such process
60
61
            raise
        log.debug('no such process %d', ppid)
62

Gitlab CI's avatar
Gitlab CI committed
63

64
# -------------------------------------------------------------------------#
Marco Clemencic's avatar
Marco Clemencic committed
65
66


67
68
69
70
class BaseTest(object):

    _common_tmpdir = None

71
    def __init__(self):
72
73
74
75
76
77
        self.program = ''
        self.args = []
        self.reference = ''
        self.error_reference = ''
        self.options = ''
        self.stderr = ''
78
        self.timeout = 600
79
80
81
82
83
84
        self.exit_code = None
        self.environment = None
        self.unsupported_platforms = []
        self.signal = None
        self.workdir = os.curdir
        self.use_temp_dir = False
Marco Clemencic's avatar
Marco Clemencic committed
85
        # Variables not for users
86
87
        self.status = None
        self.name = ''
88
89
        self.causes = []
        self.result = Result(self)
90
        self.returnedCode = 0
91
92
        self.out = ''
        self.err = ''
93
        self.proc = None
94
        self.stack_trace = None
95
        self.basedir = os.getcwd()
96

97
    def run(self):
98
99
        logging.debug('running test %s', self.name)

100
        if self.options:
Gitlab CI's avatar
Gitlab CI committed
101
102
103
            if re.search(
                    r'from\s+Gaudi.Configuration\s+import\s+\*|'
                    'from\s+Configurables\s+import', self.options):
104
                optionFile = tempfile.NamedTemporaryFile(suffix='.py')
105
            else:
106
107
108
109
110
                optionFile = tempfile.NamedTemporaryFile(suffix='.opts')
            optionFile.file.write(self.options)
            optionFile.seek(0)
            self.args.append(RationalizePath(optionFile.name))

Marco Clemencic's avatar
Marco Clemencic committed
111
112
113
114
        # If not specified, setting the environment
        if self.environment is None:
            self.environment = os.environ
        else:
Gitlab CI's avatar
Gitlab CI committed
115
116
            self.environment = dict(self.environment.items() +
                                    os.environ.items())
117

Gitlab CI's avatar
Gitlab CI committed
118
119
        platform_id = (os.environ.get('BINARY_TAG')
                       or os.environ.get('CMTCONFIG') or platform.platform())
120
        # If at least one regex matches we skip the test.
Gitlab CI's avatar
Gitlab CI committed
121
122
123
124
        skip_test = bool([
            None for prex in self.unsupported_platforms
            if re.search(prex, platform_id)
        ])
125

126
        if not skip_test:
127
128
129
130
131
132
133
134
135
            # handle working/temporary directory options
            workdir = self.workdir
            if self.use_temp_dir:
                if self._common_tmpdir:
                    workdir = self._common_tmpdir
                else:
                    workdir = tempfile.mkdtemp()

            # prepare the command to execute
Marco Clemencic's avatar
Marco Clemencic committed
136
            prog = ''
137
138
            if self.program != '':
                prog = self.program
Marco Clemencic's avatar
Marco Clemencic committed
139
            elif "GAUDIEXE" in os.environ:
140
                prog = os.environ["GAUDIEXE"]
Marco Clemencic's avatar
Marco Clemencic committed
141
            else:
142
143
144
                prog = "Gaudi.exe"

            dummy, prog_ext = os.path.splitext(prog)
Marco Clemencic's avatar
Marco Clemencic committed
145
            if prog_ext not in [".exe", ".py", ".bat"]:
146
147
148
149
150
                prog += ".exe"
                prog_ext = ".exe"

            prog = which(prog) or prog

151
152
            args = map(RationalizePath, self.args)

Marco Clemencic's avatar
Marco Clemencic committed
153
            if prog_ext == ".py":
154
                params = ['python', RationalizePath(prog)] + args
Marco Clemencic's avatar
Marco Clemencic committed
155
            else:
156
                params = [RationalizePath(prog)] + args
157

Gitlab CI's avatar
Gitlab CI committed
158
159
160
161
162
163
164
165
166
167
            validatorRes = Result({
                'CAUSE': None,
                'EXCEPTION': None,
                'RESOURCE': None,
                'TARGET': None,
                'TRACEBACK': None,
                'START_TIME': None,
                'END_TIME': None,
                'TIMEOUT_DETAIL': None
            })
168
169
            self.result = validatorRes

170
171
172
173
            # we need to switch directory because the validator expects to run
            # in the same dir as the program
            os.chdir(workdir)

Marco Clemencic's avatar
Marco Clemencic committed
174
175
            # launching test in a different thread to handle timeout exception
            def target():
Gitlab CI's avatar
Gitlab CI committed
176
177
178
                logging.debug('executing %r in %s', params, workdir)
                self.proc = Popen(
                    params, stdout=PIPE, stderr=PIPE, env=self.environment)
179
                logging.debug('(pid: %d)', self.proc.pid)
180
                self.out, self.err = self.proc.communicate()
181
182
183

            thread = threading.Thread(target=target)
            thread.start()
184
            # catching timeout
185
186
187
            thread.join(self.timeout)

            if thread.is_alive():
Gitlab CI's avatar
Gitlab CI committed
188
189
                logging.debug('time out in test %s (pid %d)', self.name,
                              self.proc.pid)
190
                # get the stack trace of the stuck process
Gitlab CI's avatar
Gitlab CI committed
191
192
193
194
195
                cmd = [
                    'gdb', '--pid',
                    str(self.proc.pid), '--batch',
                    '--eval-command=thread apply all backtrace'
                ]
196
                gdb = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
197
                self.stack_trace = gdb.communicate()[0]
198

199
                kill_tree(self.proc.pid, signal.SIGTERM)
200
201
                thread.join(60)
                if thread.is_alive():
202
                    kill_tree(self.proc.pid, signal.SIGKILL)
203
                self.causes.append('timeout')
204
205
206
            else:
                logging.debug('completed test %s', self.name)

Marco Clemencic's avatar
Marco Clemencic committed
207
                # Getting the error code
208
209
                logging.debug('returnedCode = %s', self.proc.returncode)
                self.returnedCode = self.proc.returncode
210

211
                logging.debug('validating test...')
Gitlab CI's avatar
Gitlab CI committed
212
213
                self.result, self.causes = self.ValidateOutput(
                    stdout=self.out, stderr=self.err, result=validatorRes)
214

215
216
217
218
219
220
221
            # remove the temporary directory if we created it
            if self.use_temp_dir and not self._common_tmpdir:
                shutil.rmtree(workdir, True)

            os.chdir(self.basedir)

            # handle application exit code
222
223
224
            if self.signal is not None:
                if int(self.returnedCode) != -int(self.signal):
                    self.causes.append('exit code')
Marco Clemencic's avatar
Marco Clemencic committed
225

226
            elif self.exit_code is not None:
Marco Clemencic's avatar
Marco Clemencic committed
227
                if int(self.returnedCode) != int(self.exit_code):
228
                    self.causes.append('exit code')
Marco Clemencic's avatar
Marco Clemencic committed
229

230
            elif self.returnedCode != 0:
231
                self.causes.append("exit code")
Marco Clemencic's avatar
Marco Clemencic committed
232
233
234
235

            if self.causes:
                self.status = "failed"
            else:
236
                self.status = "passed"
Marco Clemencic's avatar
Marco Clemencic committed
237
238

        else:
239
            self.status = "skipped"
240

241
        logging.debug('%s: %s', self.name, self.status)
Gitlab CI's avatar
Gitlab CI committed
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
        field_mapping = {
            'Exit Code': 'returnedCode',
            'stderr': 'err',
            'Arguments': 'args',
            'Environment': 'environment',
            'Status': 'status',
            'stdout': 'out',
            'Program Name': 'program',
            'Name': 'name',
            'Validator': 'validator',
            'Output Reference File': 'reference',
            'Error Reference File': 'error_reference',
            'Causes': 'causes',
            # 'Validator Result': 'result.annotations',
            'Unsupported Platforms': 'unsupported_platforms',
            'Stack Trace': 'stack_trace'
        }
259
260
261
        resultDict = [(key, getattr(self, attr))
                      for key, attr in field_mapping.iteritems()
                      if getattr(self, attr)]
Marco Clemencic's avatar
Marco Clemencic committed
262
        resultDict.append(('Working Directory',
Gitlab CI's avatar
Gitlab CI committed
263
264
                           RationalizePath(
                               os.path.join(os.getcwd(), self.workdir))))
Marco Clemencic's avatar
Marco Clemencic committed
265
        # print dict(resultDict).keys()
Marco Clemencic's avatar
Marco Clemencic committed
266
        resultDict.extend(self.result.annotations.iteritems())
Marco Clemencic's avatar
Marco Clemencic committed
267
        # print self.result.annotations.keys()
268
        return dict(resultDict)
269

270
271
272
    # -------------------------------------------------#
    # ----------------Validating tool------------------#
    # -------------------------------------------------#
273
274

    def ValidateOutput(self, stdout, stderr, result):
275
276
277
278
279
        if not self.stderr:
            self.validateWithReference(stdout, stderr, result, self.causes)
        elif stderr.strip() != self.stderr.strip():
            self.causes.append('standard error')
        return result, self.causes
280

Gitlab CI's avatar
Gitlab CI committed
281
282
283
284
285
286
287
288
    def findReferenceBlock(self,
                           reference=None,
                           stdout=None,
                           result=None,
                           causes=None,
                           signature_offset=0,
                           signature=None,
                           id=None):
289
290
291
292
        """
            Given a block of text, tries to find it in the output. The block had to be identified by a signature line. By default, the first line is used as signature, or the line pointed to by signature_offset. If signature_offset points outside the block, a signature line can be passed as signature argument. Note: if 'signature' is None (the default), a negative signature_offset is interpreted as index in a list (e.g. -1 means the last line), otherwise the it is interpreted as the number of lines before the first one of the block the signature must appear. The parameter 'id' allow to distinguish between different calls to this function in the same validation code.
            """

Marco Clemencic's avatar
Marco Clemencic committed
293
294
295
296
297
298
299
300
301
        if reference is None:
            reference = self.reference
        if stdout is None:
            stdout = self.out
        if result is None:
            result = self.result
        if causes is None:
            causes = self.causes

Gitlab CI's avatar
Gitlab CI committed
302
303
        reflines = filter(None,
                          map(lambda s: s.rstrip(), reference.splitlines()))
304
305
306
        if not reflines:
            raise RuntimeError("Empty (or null) reference")
        # the same on standard output
Marco Clemencic's avatar
Marco Clemencic committed
307
        outlines = filter(None, map(lambda s: s.rstrip(), stdout.splitlines()))
308
309
310
311
312
313
314

        res_field = "GaudiTest.RefBlock"
        if id:
            res_field += "_%s" % id

        if signature is None:
            if signature_offset < 0:
Marco Clemencic's avatar
Marco Clemencic committed
315
                signature_offset = len(reference) + signature_offset
316
317
318
319
            signature = reflines[signature_offset]
        # find the reference block in the output file
        try:
            pos = outlines.index(signature)
Gitlab CI's avatar
Gitlab CI committed
320
321
            outlines = outlines[pos - signature_offset:pos + len(reflines) -
                                signature_offset]
322
323
            if reflines != outlines:
                msg = "standard output"
324
325
                # I do not want 2 messages in causes if the function is called
                # twice
326
327
                if not msg in causes:
                    causes.append(msg)
Gitlab CI's avatar
Gitlab CI committed
328
329
                result[res_field + ".observed"] = result.Quote(
                    "\n".join(outlines))
330
331
332
333
334
335
336
        except ValueError:
            causes.append("missing signature")
        result[res_field + ".signature"] = result.Quote(signature)
        if len(reflines) > 1 or signature != reflines[0]:
            result[res_field + ".expected"] = result.Quote("\n".join(reflines))
        return causes

Gitlab CI's avatar
Gitlab CI committed
337
338
339
340
341
342
343
344
    def countErrorLines(self,
                        expected={
                            'ERROR': 0,
                            'FATAL': 0
                        },
                        stdout=None,
                        result=None,
                        causes=None):
345
346
347
348
349
350
351
        """
            Count the number of messages with required severity (by default ERROR and FATAL)
            and check if their numbers match the expected ones (0 by default).
            The dictionary "expected" can be used to tune the number of errors and fatals
            allowed, or to limit the number of expected warnings etc.
            """

Marco Clemencic's avatar
Marco Clemencic committed
352
353
354
355
356
357
        if stdout is None:
            stdout = self.out
        if result is None:
            result = self.result
        if causes is None:
            causes = self.causes
358
359
360
361
362
363
364
365

        # prepare the dictionary to record the extracted lines
        errors = {}
        for sev in expected:
            errors[sev] = []

        outlines = stdout.splitlines()
        from math import log10
Marco Clemencic's avatar
Marco Clemencic committed
366
        fmt = "%%%dd - %%s" % (int(log10(len(outlines) + 1)))
367
368
369
370
371
372

        linecount = 0
        for l in outlines:
            linecount += 1
            words = l.split()
            if len(words) >= 2 and words[1] in errors:
Marco Clemencic's avatar
Marco Clemencic committed
373
                errors[words[1]].append(fmt % (linecount, l.rstrip()))
374
375
376

        for e in errors:
            if len(errors[e]) != expected[e]:
Marco Clemencic's avatar
Marco Clemencic committed
377
                causes.append('%s(%d)' % (e, len(errors[e])))
Gitlab CI's avatar
Gitlab CI committed
378
379
380
381
                result["GaudiTest.lines.%s" % e] = result.Quote('\n'.join(
                    errors[e]))
                result["GaudiTest.lines.%s.expected#" % e] = result.Quote(
                    str(expected[e]))
382
383
384

        return causes

Gitlab CI's avatar
Gitlab CI committed
385
386
387
388
    def CheckTTreesSummaries(self,
                             stdout=None,
                             result=None,
                             causes=None,
Marco Clemencic's avatar
Marco Clemencic committed
389
390
                             trees_dict=None,
                             ignore=r"Basket|.*size|Compression"):
391
392
393
394
395
396
        """
            Compare the TTree summaries in stdout with the ones in trees_dict or in
            the reference file. By default ignore the size, compression and basket
            fields.
            The presence of TTree summaries when none is expected is not a failure.
            """
Marco Clemencic's avatar
Marco Clemencic committed
397
398
399
400
401
402
        if stdout is None:
            stdout = self.out
        if result is None:
            result = self.result
        if causes is None:
            causes = self.causes
403
        if trees_dict is None:
404
            lreference = self._expandReferenceFileName(self.reference)
405
406
407
408
409
410
411
412
413
            # call the validator if the file exists
            if lreference and os.path.isfile(lreference):
                trees_dict = findTTreeSummaries(open(lreference).read())
            else:
                trees_dict = {}

        from pprint import PrettyPrinter
        pp = PrettyPrinter()
        if trees_dict:
Marco Clemencic's avatar
Marco Clemencic committed
414
415
            result["GaudiTest.TTrees.expected"] = result.Quote(
                pp.pformat(trees_dict))
416
417
418
419
420
421
422
            if ignore:
                result["GaudiTest.TTrees.ignore"] = result.Quote(ignore)

        trees = findTTreeSummaries(stdout)
        failed = cmpTreesDicts(trees_dict, trees, ignore)
        if failed:
            causes.append("trees summaries")
Gitlab CI's avatar
Gitlab CI committed
423
424
            msg = "%s: %s != %s" % getCmpFailingValues(trees_dict, trees,
                                                       failed)
425
426
427
428
429
            result["GaudiTest.TTrees.failure_on"] = result.Quote(msg)
            result["GaudiTest.TTrees.found"] = result.Quote(pp.pformat(trees))

        return causes

Gitlab CI's avatar
Gitlab CI committed
430
431
432
433
    def CheckHistosSummaries(self,
                             stdout=None,
                             result=None,
                             causes=None,
Marco Clemencic's avatar
Marco Clemencic committed
434
435
                             dict=None,
                             ignore=None):
436
437
438
439
440
441
        """
            Compare the TTree summaries in stdout with the ones in trees_dict or in
            the reference file. By default ignore the size, compression and basket
            fields.
            The presence of TTree summaries when none is expected is not a failure.
            """
Marco Clemencic's avatar
Marco Clemencic committed
442
443
444
445
446
447
        if stdout is None:
            stdout = self.out
        if result is None:
            result = self.result
        if causes is None:
            causes = self.causes
448
449

        if dict is None:
450
            lreference = self._expandReferenceFileName(self.reference)
451
452
453
454
455
456
457
458
459
            # call the validator if the file exists
            if lreference and os.path.isfile(lreference):
                dict = findHistosSummaries(open(lreference).read())
            else:
                dict = {}

        from pprint import PrettyPrinter
        pp = PrettyPrinter()
        if dict:
Marco Clemencic's avatar
Marco Clemencic committed
460
461
            result["GaudiTest.Histos.expected"] = result.Quote(
                pp.pformat(dict))
462
463
464
465
466
467
468
469
470
471
472
473
474
            if ignore:
                result["GaudiTest.Histos.ignore"] = result.Quote(ignore)

        histos = findHistosSummaries(stdout)
        failed = cmpTreesDicts(dict, histos, ignore)
        if failed:
            causes.append("histos summaries")
            msg = "%s: %s != %s" % getCmpFailingValues(dict, histos, failed)
            result["GaudiTest.Histos.failure_on"] = result.Quote(msg)
            result["GaudiTest.Histos.found"] = result.Quote(pp.pformat(histos))

        return causes

Gitlab CI's avatar
Gitlab CI committed
475
476
477
478
479
480
    def validateWithReference(self,
                              stdout=None,
                              stderr=None,
                              result=None,
                              causes=None,
                              preproc=None):
Marco Clemencic's avatar
Marco Clemencic committed
481
482
483
484
        '''
        Default validation acti*on: compare standard output and error to the
        reference files.
        '''
485

Marco Clemencic's avatar
Marco Clemencic committed
486
487
488
489
490
491
492
493
        if stdout is None:
            stdout = self.out
        if stderr is None:
            stderr = self.err
        if result is None:
            result = self.result
        if causes is None:
            causes = self.causes
494
495
496
497
498

        # set the default output preprocessor
        if preproc is None:
            preproc = normalizeExamples
        # check standard output
499
        lreference = self._expandReferenceFileName(self.reference)
500
501
        # call the validator if the file exists
        if lreference and os.path.isfile(lreference):
Gitlab CI's avatar
Gitlab CI committed
502
503
504
            causes += ReferenceFileValidator(
                lreference, "standard output", "Output Diff",
                preproc=preproc)(stdout, result)
505
506
        elif lreference:
            causes += ["missing reference file"]
507
508
509
        # Compare TTree summaries
        causes = self.CheckTTreesSummaries(stdout, result, causes)
        causes = self.CheckHistosSummaries(stdout, result, causes)
510
        if causes and lreference:  # Write a new reference file for stdout
511
            try:
512
513
514
515
516
517
                cnt = 0
                newrefname = '.'.join([lreference, 'new'])
                while os.path.exists(newrefname):
                    cnt += 1
                    newrefname = '.'.join([lreference, '~%d~' % cnt, 'new'])
                newref = open(newrefname, "w")
518
519
520
                # sanitize newlines
                for l in stdout.splitlines():
                    newref.write(l.rstrip() + '\n')
Marco Clemencic's avatar
Marco Clemencic committed
521
                del newref  # flush and close
522
523
                result['New Output Reference File'] = os.path.relpath(
                    newrefname, self.basedir)
524
525
526
527
528
529
            except IOError:
                # Ignore IO errors when trying to update reference files
                # because we may be in a read-only filesystem
                pass

        # check standard error
530
        lreference = self._expandReferenceFileName(self.error_reference)
531
        # call the validator if we have a file to use
532
533
        if lreference:
            if os.path.isfile(lreference):
Gitlab CI's avatar
Gitlab CI committed
534
535
536
537
538
                newcauses = ReferenceFileValidator(
                    lreference,
                    "standard error",
                    "Error Diff",
                    preproc=preproc)(stderr, result)
539
540
            else:
                newcauses += ["missing error reference file"]
541
            causes += newcauses
542
543
544
545
546
547
548
            if newcauses and lreference:  # Write a new reference file for stdedd
                cnt = 0
                newrefname = '.'.join([lreference, 'new'])
                while os.path.exists(newrefname):
                    cnt += 1
                    newrefname = '.'.join([lreference, '~%d~' % cnt, 'new'])
                newref = open(newrefname, "w")
549
550
551
                # sanitize newlines
                for l in stderr.splitlines():
                    newref.write(l.rstrip() + '\n')
Marco Clemencic's avatar
Marco Clemencic committed
552
                del newref  # flush and close
553
554
                result['New Error Reference File'] = os.path.relpath(
                    newrefname, self.basedir)
555
        else:
Marco Clemencic's avatar
Marco Clemencic committed
556
            causes += BasicOutputValidator(lreference, "standard error",
Gitlab CI's avatar
Gitlab CI committed
557
558
                                           "ExecTest.expected_stderr")(stderr,
                                                                       result)
559
560
        return causes

561
562
563
564
565
566
    def _expandReferenceFileName(self, reffile):
        # if no file is passed, do nothing
        if not reffile:
            return ""

        # function to split an extension in constituents parts
567
568
569
570
        def platformSplit(p):
            import re
            delim = re.compile('-' in p and r"[-+]" or r"_")
            return set(delim.split(p))
571

Gitlab CI's avatar
Gitlab CI committed
572
573
        reference = os.path.normpath(
            os.path.join(self.basedir, os.path.expandvars(reffile)))
574
575
576
577
578

        # old-style platform-specific reference name
        spec_ref = reference[:-3] + GetPlatform(self)[0:3] + reference[-3:]
        if os.path.isfile(spec_ref):
            reference = spec_ref
Marco Clemencic's avatar
Marco Clemencic committed
579
        else:  # look for new-style platform specific reference files:
580
581
            # get all the files whose name start with the reference filename
            dirname, basename = os.path.split(reference)
Marco Clemencic's avatar
Marco Clemencic committed
582
583
            if not dirname:
                dirname = '.'
584
585
586
            head = basename + "."
            head_len = len(head)
            platform = platformSplit(GetPlatform(self))
587
588
            if 'do0' in platform:
                platform.add('dbg')
589
590
591
592
593
            candidates = []
            for f in os.listdir(dirname):
                if f.startswith(head):
                    req_plat = platformSplit(f[head_len:])
                    if platform.issuperset(req_plat):
Marco Clemencic's avatar
Marco Clemencic committed
594
595
                        candidates.append((len(req_plat), f))
            if candidates:  # take the one with highest matching
596
597
598
599
600
601
                # FIXME: it is not possible to say if x86_64-slc5-gcc43-dbg
                #        has to use ref.x86_64-gcc43 or ref.slc5-dbg
                candidates.sort()
                reference = os.path.join(dirname, candidates[-1][1])
        return reference

602

Gitlab CI's avatar
Gitlab CI committed
603
# ======= GAUDI TOOLS =======
Marco Clemencic's avatar
Marco Clemencic committed
604

605
606
607
608
609
610
611
612
import shutil
import string
import difflib
import calendar

try:
    from GaudiKernel import ROOT6WorkAroundEnabled
except ImportError:
Gitlab CI's avatar
Gitlab CI committed
613

614
615
616
617
    def ROOT6WorkAroundEnabled(id=None):
        # dummy implementation
        return False

Gitlab CI's avatar
Gitlab CI committed
618

619
# --------------------------------- TOOLS ---------------------------------#
620

Marco Clemencic's avatar
Marco Clemencic committed
621
622

def RationalizePath(p):
623
624
625
626
    """
    Function used to normalize the used path
    """
    newPath = os.path.normpath(os.path.expandvars(p))
Marco Clemencic's avatar
Marco Clemencic committed
627
    if os.path.exists(newPath):
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
        p = os.path.realpath(newPath)
    return p


def which(executable):
    """
    Locates an executable in the executables path ($PATH) and returns the full
    path to it.  An application is looked for with or without the '.exe' suffix.
    If the executable cannot be found, None is returned
    """
    if os.path.isabs(executable):
        if not os.path.exists(executable):
            if executable.endswith('.exe'):
                if os.path.exists(executable[:-4]):
                    return executable[:-4]
Marco Clemencic's avatar
Marco Clemencic committed
643
644
645
            else:
                head, executable = os.path.split(executable)
        else:
646
647
648
649
650
651
652
653
654
655
            return executable
    for d in os.environ.get("PATH").split(os.pathsep):
        fullpath = os.path.join(d, executable)
        if os.path.exists(fullpath):
            return fullpath
    if executable.endswith('.exe'):
        return which(executable[:-4])
    return None


656
657
658
# -------------------------------------------------------------------------#
# ----------------------------- Result Classe -----------------------------#
# -------------------------------------------------------------------------#
659
660
import types

Marco Clemencic's avatar
Marco Clemencic committed
661

662
663
class Result:

Marco Clemencic's avatar
Marco Clemencic committed
664
665
666
667
    PASS = 'PASS'
    FAIL = 'FAIL'
    ERROR = 'ERROR'
    UNTESTED = 'UNTESTED'
668
669
670
671
672
673
674
675
676

    EXCEPTION = ""
    RESOURCE = ""
    TARGET = ""
    TRACEBACK = ""
    START_TIME = ""
    END_TIME = ""
    TIMEOUT_DETAIL = ""

Marco Clemencic's avatar
Marco Clemencic committed
677
    def __init__(self, kind=None, id=None, outcome=PASS, annotations={}):
678
679
        self.annotations = annotations.copy()

Marco Clemencic's avatar
Marco Clemencic committed
680
    def __getitem__(self, key):
681
682
683
        assert type(key) in types.StringTypes
        return self.annotations[key]

Marco Clemencic's avatar
Marco Clemencic committed
684
    def __setitem__(self, key, value):
685
686
        assert type(key) in types.StringTypes
        assert type(value) in types.StringTypes
Marco Clemencic's avatar
Marco Clemencic committed
687
        self.annotations[key] = value
688

Marco Clemencic's avatar
Marco Clemencic committed
689
    def Quote(self, string):
690
691
692
        return string


693
694
695
# -------------------------------------------------------------------------#
# --------------------------- Validator Classes ---------------------------#
# -------------------------------------------------------------------------#
696

697
698
699
# Basic implementation of an option validator for Gaudi test. This
# implementation is based on the standard (LCG) validation functions used
# in QMTest.
700
701
702


class BasicOutputValidator:
Marco Clemencic's avatar
Marco Clemencic committed
703
704
705
706
    def __init__(self, ref, cause, result_key):
        self.ref = ref
        self.cause = cause
        self.result_key = result_key
707

Marco Clemencic's avatar
Marco Clemencic committed
708
    def __call__(self, out, result):
709
710
711
712
713
714
715
716
717
        """Validate the output of the program.
            'stdout' -- A string containing the data written to the standard output
            stream.
            'stderr' -- A string containing the data written to the standard error
            stream.
            'result' -- A 'Result' object. It may be used to annotate
            the outcome according to the content of stderr.
            returns -- A list of strings giving causes of failure."""

Marco Clemencic's avatar
Marco Clemencic committed
718
719
720
        causes = []
        # Check the output
        if not self.__CompareText(out, self.ref):
721
            causes.append(self.cause)
Marco Clemencic's avatar
Marco Clemencic committed
722
            result[self.result_key] = result.Quote(self.ref)
723
724
725
726
727
728
729
730
731
732

        return causes

    def __CompareText(self, s1, s2):
        """Compare 's1' and 's2', ignoring line endings.
            's1' -- A string.
            's2' -- A string.
            returns -- True if 's1' and 's2' are the same, ignoring
            differences in line endings."""
        if ROOT6WorkAroundEnabled('ReadRootmapCheck'):
733
734
            # FIXME: (MCl) Hide warnings from new rootmap sanity check until we
            # can fix them
Marco Clemencic's avatar
Marco Clemencic committed
735
            to_ignore = re.compile(
Gitlab CI's avatar
Gitlab CI committed
736
737
                r'Warning in <TInterpreter::ReadRootmapFile>: .* is already in .*'
            )
Marco Clemencic's avatar
Marco Clemencic committed
738

Gitlab CI's avatar
Gitlab CI committed
739
740
741
742
743
            def keep_line(l):
                return not to_ignore.match(l)

            return filter(keep_line, s1.splitlines()) == filter(
                keep_line, s2.splitlines())
744
745
746
747
        else:
            return s1.splitlines() == s2.splitlines()


748
# ------------------------ Preprocessor elements ------------------------#
749
750
751
class FilePreprocessor:
    """ Base class for a callable that takes a file and returns a modified
        version of it."""
Marco Clemencic's avatar
Marco Clemencic committed
752

753
754
    def __processLine__(self, line):
        return line
Marco Clemencic's avatar
Marco Clemencic committed
755

756
757
758
759
    def __processFile__(self, lines):
        output = []
        for l in lines:
            l = self.__processLine__(l)
Marco Clemencic's avatar
Marco Clemencic committed
760
761
            if l:
                output.append(l)
762
        return output
Marco Clemencic's avatar
Marco Clemencic committed
763

764
    def __call__(self, input):
Marco Clemencic's avatar
Marco Clemencic committed
765
        if hasattr(input, "__iter__"):
766
767
768
769
770
            lines = input
            mergeback = False
        else:
            lines = input.splitlines()
            mergeback = True
771
        output = self.__processFile__(lines)
Marco Clemencic's avatar
Marco Clemencic committed
772
773
        if mergeback:
            output = '\n'.join(output)
774
        return output
Marco Clemencic's avatar
Marco Clemencic committed
775

776
    def __add__(self, rhs):
Marco Clemencic's avatar
Marco Clemencic committed
777
778
        return FilePreprocessorSequence([self, rhs])

779
780

class FilePreprocessorSequence(FilePreprocessor):
Marco Clemencic's avatar
Marco Clemencic committed
781
    def __init__(self, members=[]):
782
        self.members = members
Marco Clemencic's avatar
Marco Clemencic committed
783

784
785
    def __add__(self, rhs):
        return FilePreprocessorSequence(self.members + [rhs])
Marco Clemencic's avatar
Marco Clemencic committed
786

787
788
789
790
791
792
    def __call__(self, input):
        output = input
        for pp in self.members:
            output = pp(output)
        return output

Marco Clemencic's avatar
Marco Clemencic committed
793

794
class LineSkipper(FilePreprocessor):
Marco Clemencic's avatar
Marco Clemencic committed
795
    def __init__(self, strings=[], regexps=[]):
796
797
        import re
        self.strings = strings
Marco Clemencic's avatar
Marco Clemencic committed
798
        self.regexps = map(re.compile, regexps)
799
800
801

    def __processLine__(self, line):
        for s in self.strings:
Marco Clemencic's avatar
Marco Clemencic committed
802
803
            if line.find(s) >= 0:
                return None
804
        for r in self.regexps:
Marco Clemencic's avatar
Marco Clemencic committed
805
806
            if r.search(line):
                return None
807
808
        return line

Marco Clemencic's avatar
Marco Clemencic committed
809

810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
class BlockSkipper(FilePreprocessor):
    def __init__(self, start, end):
        self.start = start
        self.end = end
        self._skipping = False

    def __processLine__(self, line):
        if self.start in line:
            self._skipping = True
            return None
        elif self.end in line:
            self._skipping = False
        elif self._skipping:
            return None
        return line

Marco Clemencic's avatar
Marco Clemencic committed
826

827
class RegexpReplacer(FilePreprocessor):
Marco Clemencic's avatar
Marco Clemencic committed
828
    def __init__(self, orig, repl="", when=None):
829
830
        if when:
            when = re.compile(when)
Marco Clemencic's avatar
Marco Clemencic committed
831
832
833
        self._operations = [(when, re.compile(orig), repl)]

    def __add__(self, rhs):
834
        if isinstance(rhs, RegexpReplacer):
Marco Clemencic's avatar
Marco Clemencic committed
835
            res = RegexpReplacer("", "", None)
836
837
838
839
            res._operations = self._operations + rhs._operations
        else:
            res = FilePreprocessor.__add__(self, rhs)
        return res
Marco Clemencic's avatar
Marco Clemencic committed
840

841
    def __processLine__(self, line):
Marco Clemencic's avatar
Marco Clemencic committed
842
        for w, o, r in self._operations:
843
844
845
846
            if w is None or w.search(line):
                line = o.sub(r, line)
        return line

Marco Clemencic's avatar
Marco Clemencic committed
847

848
# Common preprocessors
Marco Clemencic's avatar
Marco Clemencic committed
849
maskPointers = RegexpReplacer("0x[0-9a-fA-F]{4,16}", "0x########")
850
851
852
normalizeDate = RegexpReplacer(
    "[0-2]?[0-9]:[0-5][0-9]:[0-5][0-9] [0-9]{4}[-/][01][0-9][-/][0-3][0-9][ A-Z]*",
    "00:00:00 1970-01-01")
853
854
855
856
857
858
859
normalizeEOL = FilePreprocessor()
normalizeEOL.__processLine__ = lambda line: str(line).rstrip() + '\n'

skipEmptyLines = FilePreprocessor()
# FIXME: that's ugly
skipEmptyLines.__processLine__ = lambda line: (line.strip() and line) or None

Marco Clemencic's avatar
Marco Clemencic committed
860
# Special preprocessor sorting the list of strings (whitespace separated)
861
#  that follow a signature on a single line
Marco Clemencic's avatar
Marco Clemencic committed
862
863


864
865
866
867
class LineSorter(FilePreprocessor):
    def __init__(self, signature):
        self.signature = signature
        self.siglen = len(signature)
Marco Clemencic's avatar
Marco Clemencic committed
868

869
870
    def __processLine__(self, line):
        pos = line.find(self.signature)
Marco Clemencic's avatar
Marco Clemencic committed
871
872
873
        if pos >= 0:
            line = line[:(pos + self.siglen)]
            lst = line[(pos + self.siglen):].split()
874
875
876
877
            lst.sort()
            line += " ".join(lst)
        return line

Marco Clemencic's avatar
Marco Clemencic committed
878

879
880
881
882
class SortGroupOfLines(FilePreprocessor):
    '''
    Sort group of lines matching a regular expression
    '''
Marco Clemencic's avatar
Marco Clemencic committed
883

884
885
    def __init__(self, exp):
        self.exp = exp if hasattr(exp, 'match') else re.compile(exp)
Marco Clemencic's avatar
Marco Clemencic committed
886

887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
    def __processFile__(self, lines):
        match = self.exp.match
        output = []
        group = []
        for l in lines:
            if match(l):
                group.append(l)
            else:
                if group:
                    group.sort()
                    output.extend(group)
                    group = []
                output.append(l)
        return output

Marco Clemencic's avatar
Marco Clemencic committed
902

903
904
# Preprocessors for GaudiExamples
normalizeExamples = maskPointers + normalizeDate
Marco Clemencic's avatar
Marco Clemencic committed
905
for w, o, r in [
Gitlab CI's avatar
Gitlab CI committed
906
        # ("TIMER.TIMER",r"[0-9]", "0"), # Normalize time output
Marco Clemencic's avatar
Marco Clemencic committed
907
908
909
910
    ("TIMER.TIMER", r"\s+[+-]?[0-9]+[0-9.]*", " 0"),  # Normalize time output
    ("release all pending", r"^.*/([^/]*:.*)", r"\1"),
    ("^#.*file", r"file '.*[/\\]([^/\\]*)$", r"file '\1"),
    ("^JobOptionsSvc.*options successfully read in from",
Gitlab CI's avatar
Gitlab CI committed
911
912
913
914
915
     r"read in from .*[/\\]([^/\\]*)$",
     r"file \1"),  # normalize path to options
        # Normalize UUID, except those ending with all 0s (i.e. the class IDs)
    (None,
     r"[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}(?!-0{12})-[0-9A-Fa-f]{12}",
Marco Clemencic's avatar
Marco Clemencic committed
916
     "00000000-0000-0000-0000-000000000000"),
Gitlab CI's avatar
Gitlab CI committed
917
        # Absorb a change in ServiceLocatorHelper
Marco Clemencic's avatar
Marco Clemencic committed
918
919
    ("ServiceLocatorHelper::", "ServiceLocatorHelper::(create|locate)Service",
     "ServiceLocatorHelper::service"),
Gitlab CI's avatar
Gitlab CI committed
920
        # Remove the leading 0 in Windows' exponential format
Marco Clemencic's avatar
Marco Clemencic committed
921
    (None, r"e([-+])0([0-9][0-9])", r"e\1\2"),
Gitlab CI's avatar
Gitlab CI committed
922
        # Output line changed in Gaudi v24
Marco Clemencic's avatar
Marco Clemencic committed
923
924
    (None, r'Service reference count check:',
     r'Looping over all active services...'),
Gitlab CI's avatar
Gitlab CI committed
925
926
927
        # Ignore count of declared properties (anyway they are all printed)
    (None,
     r"^(.*(DEBUG|SUCCESS) List of ALL properties of .*#properties = )\d+",
928
     r"\1NN"),
Marco Clemencic's avatar
Marco Clemencic committed
929
    ('ApplicationMgr', r'(declareMultiSvcType|addMultiSvc): ', ''),
Marco Clemencic's avatar
Marco Clemencic committed
930
931
]:  # [ ("TIMER.TIMER","[0-9]+[0-9.]*", "") ]
    normalizeExamples += RegexpReplacer(o, r, w)
932

Gitlab CI's avatar
Gitlab CI committed
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
lineSkipper = LineSkipper(
    [
        "//GP:",
        "JobOptionsSvc        INFO # ",
        "JobOptionsSvc     WARNING # ",
        "Time User",
        "Welcome to",
        "This machine has a speed",
        "TIME:",
        "running on",
        "ToolSvc.Sequenc...   INFO",
        "DataListenerSvc      INFO XML written to file:",
        "[INFO]",
        "[WARNING]",
        "DEBUG No writable file catalog found which contains FID:",
        "DEBUG Service base class initialized successfully",
        # changed between v20 and v21
        "DEBUG Incident  timing:",
        # introduced with patch #3487
        # changed the level of the message from INFO to
        # DEBUG
        "INFO  'CnvServices':[",
        # message removed because could be printed in constructor
        "DEBUG  'CnvServices':[",
        # The signal handler complains about SIGXCPU not
        # defined on some platforms
        'SIGXCPU',
    ],
    regexps=[
        r"^JobOptionsSvc        INFO *$",
        r"^# ",  # Ignore python comments
        # skip the message reporting the version of the root file
        r"(Always|SUCCESS)\s*(Root f|[^ ]* F)ile version:",
        r"File '.*.xml' does not exist",
        r"INFO Refer to dataset .* by its file ID:",
        r"INFO Referring to dataset .* by its file ID:",
        r"INFO Disconnect from dataset",
        r"INFO Disconnected from dataset",
        r"INFO Disconnected data IO:",
        r"IncidentSvc\s*(DEBUG (Adding|Removing)|VERBOSE Calling)",
        # I want to ignore the header of the unchecked StatusCode report
        r"^StatusCodeSvc.*listing all unchecked return codes:",
        r"^StatusCodeSvc\s*INFO\s*$",
        r"Num\s*\|\s*Function\s*\|\s*Source Library",
        r"^[-+]*\s*$",
        # Hide the fake error message coming from POOL/ROOT (ROOT 5.21)
        r"ERROR Failed to modify file: .* Errno=2 No such file or directory",
        # Hide unchecked StatusCodes from dictionaries
        r"^ +[0-9]+ \|.*ROOT",
        r"^ +[0-9]+ \|.*\|.*Dict",
        # Hide success StatusCodeSvc message
        r"StatusCodeSvc.*all StatusCode instances where checked",
        # Hide EventLoopMgr total timing report
        r"EventLoopMgr.*---> Loop Finished",
        r"HiveSlimEventLo.*---> Loop Finished",
        # Remove ROOT TTree summary table, which changes from one version to the
        # other
        r"^\*.*\*$",
        # Remove Histos Summaries
        r"SUCCESS\s*Booked \d+ Histogram\(s\)",
        r"^ \|",
        r"^ ID=",
        # Ignore added/removed properties
        r"Property(.*)'Audit(Algorithm|Tool|Service)s':",
        # these were missing in tools
        r"Property(.*)'AuditRe(start|initialize)':",
        r"Property(.*)'IsIOBound':",
        # removed with gaudi/Gaudi!273
        r"Property(.*)'ErrorCount(er)?':",
        # added with gaudi/Gaudi!306
        r"Property(.*)'Sequential':",
        # added with gaudi/Gaudi!314
        r"Property(.*)'FilterCircularDependencies':",
        # removed with gaudi/Gaudi!316
        r"Property(.*)'IsClonable':",
        # ignore uninteresting/obsolete messages
        r"Property update for OutputLevel : new value =",
        r"EventLoopMgr\s*DEBUG Creating OutputStream",
    ])
1012
1013

if ROOT6WorkAroundEnabled('ReadRootmapCheck'):
1014
1015
    # FIXME: (MCl) Hide warnings from new rootmap sanity check until we can
    # fix them
Marco Clemencic's avatar
Marco Clemencic committed
1016
    lineSkipper += LineSkipper(regexps=[
1017
        r'Warning in <TInterpreter::ReadRootmapFile>: .* is already in .*',
Marco Clemencic's avatar
Marco Clemencic committed
1018
    ])
1019

Gitlab CI's avatar
Gitlab CI committed
1020
1021
1022
1023
normalizeExamples = (
    lineSkipper + normalizeExamples + skipEmptyLines + normalizeEOL +
    LineSorter("Services to release : ") +
    SortGroupOfLines(r'^\S+\s+(DEBUG|SUCCESS) Property \[\'Name\':'))
1024

1025
# --------------------- Validation functions/classes ---------------------#
1026

Marco Clemencic's avatar
Marco Clemencic committed
1027

1028
class ReferenceFileValidator:
Marco Clemencic's avatar
Marco Clemencic committed
1029
    def __init__(self, reffile, cause, result_key, preproc=normalizeExamples):
1030
        self.reffile = os.path.expandvars(reffile)
Marco Clemencic's avatar
Marco Clemencic committed
1031
        self.cause = cause
1032
1033
1034
        self.result_key = result_key
        self.preproc = preproc

Marco Clemencic's avatar
Marco Clemencic committed
1035
    def __call__(self, stdout, result):
1036
        causes = []
1037
        if os.path.isfile(self.reffile):
1038
            orig = open(self.reffile).xreadlines()
1039
1040
            if self.preproc:
                orig = self.preproc(orig)
Marco Clemencic's avatar
Marco Clemencic committed
1041
                result[self.result_key + '.preproc.orig'] = \
1042
                    result.Quote('\n'.join(map(str.strip, orig)))
1043
1044
1045
1046
1047
1048
        else:
            orig = []
        new = stdout.splitlines()
        if self.preproc:
            new = self.preproc(new)

Marco Clemencic's avatar
Marco Clemencic committed
1049
        diffs = difflib.ndiff(orig, new, charjunk=difflib.IS_CHARACTER_JUNK)
Gitlab CI's avatar
Gitlab CI committed
1050
1051
        filterdiffs = map(lambda x: x.strip(),
                          filter(lambda x: x[0] != " ", diffs))
1052
1053
1054
1055
1056
1057
        if filterdiffs:
            result[self.result_key] = result.Quote("\n".join(filterdiffs))
            result[self.result_key] += result.Quote("""
                Legend:
                -) reference file
                +) standard output of the test""")
Marco Clemencic's avatar
Marco Clemencic committed
1058
            result[self.result_key + '.preproc.new'] = \
1059
                result.Quote('\n'.join(map(str.strip, new)))
1060
1061
1062
            causes.append(self.cause)
        return causes

Marco Clemencic's avatar
Marco Clemencic committed
1063

1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
def findTTreeSummaries(stdout):
    """
        Scan stdout to find ROOT TTree summaries and digest them.
        """
    stars = re.compile(r"^\*+$")
    outlines = stdout.splitlines()
    nlines = len(outlines)
    trees = {}

    i = 0
Marco Clemencic's avatar
Marco Clemencic committed
1074
    while i < nlines:  # loop over the output
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
        # look for
        while i < nlines and not stars.match(outlines[i]):
            i += 1
        if i < nlines:
            tree, i = _parseTTreeSummary(outlines, i)
            if tree:
                trees[tree["Name"]] = tree

    return trees

Marco Clemencic's avatar
Marco Clemencic committed
1085
1086

def cmpTreesDicts(reference, to_check, ignore=None):
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
    """
        Check that all the keys in reference are in to_check too, with the same value.
        If the value is a dict, the function is called recursively. to_check can
        contain more keys than reference, that will not be tested.
        The function returns at the first difference found.
        """
    fail_keys = []
    # filter the keys in the reference dictionary
    if ignore:
        ignore_re = re.compile(ignore)
Marco Clemencic's avatar
Marco Clemencic committed
1097
        keys = [key for key in reference if not ignore_re.match(key)]
1098
1099
1100
1101
    else:
        keys = reference.keys()
    # loop over the keys (not ignored) in the reference dictionary
    for k in keys:
Marco Clemencic's avatar
Marco Clemencic committed
1102
        if k in to_check:  # the key must be in the dictionary to_check
1103
            if (type(reference[k]) is dict) and (type(to_check[k]) is dict):
1104
1105
                # if both reference and to_check values are dictionaries,
                # recurse
Gitlab CI's avatar
Gitlab CI committed
1106
1107
                failed = fail_keys = cmpTreesDicts(reference[k], to_check[k],
                                                   ignore)