GaudiTest.py 89.9 KB
Newer Older
marcocle's avatar
marcocle committed
1
2
3
4
########################################################################
# File:   GaudiTest.py
# Author: Marco Clemencic CERN/PH-LBC
########################################################################
Marco Clemencic's avatar
Marco Clemencic committed
5
__author__ = 'Marco Clemencic CERN/PH-LBC'
marcocle's avatar
marcocle committed
6
7
8
9
10
11
12
13
14
15
########################################################################
# Imports
########################################################################
import os
import sys
import re
import tempfile
import shutil
import string
import difflib
16
17
import time
import calendar
18
19
import codecs

marcocle's avatar
marcocle committed
20
21
from subprocess import Popen, PIPE, STDOUT

22
23
24
try:
    from GaudiKernel import ROOT6WorkAroundEnabled
except ImportError:
Gitlab CI's avatar
Gitlab CI committed
25

26
27
28
29
    def ROOT6WorkAroundEnabled(id=None):
        # dummy implementation
        return False

Gitlab CI's avatar
Gitlab CI committed
30

31
32
33
# ensure the preferred locale
os.environ['LC_ALL'] = 'C'

34
35
36
37
38
39
40
# Needed for the XML wrapper
try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

# redefinition of timedelta.total_seconds() because it is not present in the 2.6 version
Marco Clemencic's avatar
Marco Clemencic committed
41
42
43
44


def total_seconds_replacement(timedelta):
    return timedelta.days * 86400 + timedelta.seconds + timedelta.microseconds / 1000000
45
46


marcocle's avatar
marcocle committed
47
48
import qm
from qm.test.classes.command import ExecTestBase
marcocle's avatar
marcocle committed
49
from qm.test.result_stream import ResultStream
marcocle's avatar
marcocle committed
50

Marco Clemencic's avatar
Marco Clemencic committed
51
# Needed by the re-implementation of TimeoutExecutable
marcocle's avatar
marcocle committed
52
import qm.executable
53
import signal
marcocle's avatar
marcocle committed
54
55
56
57
58
# The classes in this module are implemented differently depending on
# the operating system in use.
if sys.platform == "win32":
    import msvcrt
    import pywintypes
Marco Clemencic's avatar
Marco Clemencic committed
59
    from threading import *
marcocle's avatar
marcocle committed
60
61
62
63
64
65
66
    import win32api
    import win32con
    import win32event
    import win32file
    import win32pipe
    import win32process
else:
67
    from six.moves import cPickle
marcocle's avatar
marcocle committed
68
69
70
71
72
73
74
    import fcntl
    import select
    import qm.sigmask

########################################################################
# Utility Classes
########################################################################
Marco Clemencic's avatar
Marco Clemencic committed
75
76


marcocle's avatar
marcocle committed
77
78
79
80
class TemporaryEnvironment:
    """
    Class to changes the environment temporarily.
    """
Marco Clemencic's avatar
Marco Clemencic committed
81
82

    def __init__(self, orig=os.environ, keep_same=False):
marcocle's avatar
marcocle committed
83
84
85
86
        """
        Create a temporary environment on top of the one specified
        (it can be another TemporaryEnvironment instance).
        """
Marco Clemencic's avatar
Marco Clemencic committed
87
        # print "New environment"
marcocle's avatar
marcocle committed
88
89
90
        self.old_values = {}
        self.env = orig
        self._keep_same = keep_same
marcocle's avatar
marcocle committed
91

Marco Clemencic's avatar
Marco Clemencic committed
92
    def __setitem__(self, key, value):
marcocle's avatar
marcocle committed
93
94
95
        """
        Set an environment variable recording the previous value.
        """
Marco Clemencic's avatar
Marco Clemencic committed
96
97
        if key not in self.old_values:
            if key in self.env:
marcocle's avatar
marcocle committed
98
99
100
101
102
                if not self._keep_same or self.env[key] != value:
                    self.old_values[key] = self.env[key]
            else:
                self.old_values[key] = None
        self.env[key] = value
marcocle's avatar
marcocle committed
103

Marco Clemencic's avatar
Marco Clemencic committed
104
    def __getitem__(self, key):
marcocle's avatar
marcocle committed
105
106
107
108
109
        """
        Get an environment variable.
        Needed to provide the same interface as os.environ.
        """
        return self.env[key]
marcocle's avatar
marcocle committed
110

Marco Clemencic's avatar
Marco Clemencic committed
111
    def __delitem__(self, key):
marcocle's avatar
marcocle committed
112
113
114
115
        """
        Unset an environment variable.
        Needed to provide the same interface as os.environ.
        """
Marco Clemencic's avatar
Marco Clemencic committed
116
        if key not in self.env:
marcocle's avatar
marcocle committed
117
118
119
120
121
122
123
124
125
126
            raise KeyError(key)
        self.old_values[key] = self.env[key]
        del self.env[key]

    def keys(self):
        """
        Return the list of defined environment variables.
        Needed to provide the same interface as os.environ.
        """
        return self.env.keys()
marcocle's avatar
marcocle committed
127

marcocle's avatar
marcocle committed
128
129
130
131
132
133
    def items(self):
        """
        Return the list of (name,value) pairs for the defined environment variables.
        Needed to provide the same interface as os.environ.
        """
        return self.env.items()
marcocle's avatar
marcocle committed
134

Marco Clemencic's avatar
Marco Clemencic committed
135
    def __contains__(self, key):
marcocle's avatar
marcocle committed
136
137
138
139
140
        """
        Operator 'in'.
        Needed to provide the same interface as os.environ.
        """
        return key in self.env
marcocle's avatar
marcocle committed
141

marcocle's avatar
marcocle committed
142
143
    def restore(self):
        """
144
        Revert all the changes done to the original environment.
marcocle's avatar
marcocle committed
145
        """
Marco Clemencic's avatar
Marco Clemencic committed
146
        for key, value in self.old_values.items():
marcocle's avatar
marcocle committed
147
148
149
150
151
            if value is None:
                del self.env[key]
            else:
                self.env[key] = value
        self.old_values = {}
marcocle's avatar
marcocle committed
152

marcocle's avatar
marcocle committed
153
154
155
156
    def __del__(self):
        """
        Revert the changes on destruction.
        """
Marco Clemencic's avatar
Marco Clemencic committed
157
        # print "Restoring the environment"
marcocle's avatar
marcocle committed
158
159
        self.restore()

Marco Clemencic's avatar
Marco Clemencic committed
160
    def gen_script(self, shell_type):
marcocle's avatar
marcocle committed
161
162
163
        """
        Generate a shell script to reproduce the changes in the environment.
        """
Marco Clemencic's avatar
Marco Clemencic committed
164
        shells = ['csh', 'sh', 'bat']
marcocle's avatar
marcocle committed
165
        if shell_type not in shells:
Gitlab CI's avatar
Gitlab CI committed
166
167
            raise RuntimeError("Shell type '%s' unknown. Available: %s" %
                               (shell_type, shells))
marcocle's avatar
marcocle committed
168
        out = ""
Marco Clemencic's avatar
Marco Clemencic committed
169
        for key, value in self.old_values.items():
marcocle's avatar
marcocle committed
170
171
172
            if key not in self.env:
                # unset variable
                if shell_type == 'csh':
Marco Clemencic's avatar
Marco Clemencic committed
173
                    out += 'unsetenv %s\n' % key
marcocle's avatar
marcocle committed
174
                elif shell_type == 'sh':
Marco Clemencic's avatar
Marco Clemencic committed
175
                    out += 'unset %s\n' % key
marcocle's avatar
marcocle committed
176
                elif shell_type == 'bat':
Marco Clemencic's avatar
Marco Clemencic committed
177
                    out += 'set %s=\n' % key
marcocle's avatar
marcocle committed
178
179
180
            else:
                # set variable
                if shell_type == 'csh':
Marco Clemencic's avatar
Marco Clemencic committed
181
                    out += 'setenv %s "%s"\n' % (key, self.env[key])
marcocle's avatar
marcocle committed
182
                elif shell_type == 'sh':
Marco Clemencic's avatar
Marco Clemencic committed
183
                    out += 'export %s="%s"\n' % (key, self.env[key])
marcocle's avatar
marcocle committed
184
                elif shell_type == 'bat':
Marco Clemencic's avatar
Marco Clemencic committed
185
                    out += 'set %s=%s\n' % (key, self.env[key])
marcocle's avatar
marcocle committed
186
187
        return out

Marco Clemencic's avatar
Marco Clemencic committed
188

marcocle's avatar
marcocle committed
189
190
191
192
193
194
195
class TempDir:
    """Small class for temporary directories.
    When instantiated, it creates a temporary directory and the instance
    behaves as the string containing the directory name.
    When the instance goes out of scope, it removes all the content of
    the temporary directory (automatic clean-up).
    """
Marco Clemencic's avatar
Marco Clemencic committed
196
197

    def __init__(self, keep=False, chdir=False):
marcocle's avatar
marcocle committed
198
199
200
201
202
203
        self.name = tempfile.mkdtemp()
        self._keep = keep
        self._origdir = None
        if chdir:
            self._origdir = os.getcwd()
            os.chdir(self.name)
marcocle's avatar
marcocle committed
204

marcocle's avatar
marcocle committed
205
206
    def __str__(self):
        return self.name
marcocle's avatar
marcocle committed
207

marcocle's avatar
marcocle committed
208
209
210
211
212
    def __del__(self):
        if self._origdir:
            os.chdir(self._origdir)
        if self.name and not self._keep:
            shutil.rmtree(self.name)
marcocle's avatar
marcocle committed
213

Marco Clemencic's avatar
Marco Clemencic committed
214
215
216
    def __getattr__(self, attr):
        return getattr(self.name, attr)

marcocle's avatar
marcocle committed
217
218
219
220
221
222
223
224

class TempFile:
    """Small class for temporary files.
    When instantiated, it creates a temporary directory and the instance
    behaves as the string containing the directory name.
    When the instance goes out of scope, it removes all the content of
    the temporary directory (automatic clean-up).
    """
Marco Clemencic's avatar
Marco Clemencic committed
225

Gitlab CI's avatar
Gitlab CI committed
226
227
228
229
230
231
    def __init__(self,
                 suffix='',
                 prefix='tmp',
                 dir=None,
                 text=False,
                 keep=False):
marcocle's avatar
marcocle committed
232
233
234
        self.file = None
        self.name = None
        self._keep = keep
marcocle's avatar
marcocle committed
235

Marco Clemencic's avatar
Marco Clemencic committed
236
237
        self._fd, self.name = tempfile.mkstemp(suffix, prefix, dir, text)
        self.file = os.fdopen(self._fd, "r+")
marcocle's avatar
marcocle committed
238

marcocle's avatar
marcocle committed
239
240
    def __str__(self):
        return self.name
marcocle's avatar
marcocle committed
241

marcocle's avatar
marcocle committed
242
243
244
245
246
    def __del__(self):
        if self.file:
            self.file.close()
        if self.name and not self._keep:
            os.remove(self.name)
marcocle's avatar
marcocle committed
247

Marco Clemencic's avatar
Marco Clemencic committed
248
249
250
    def __getattr__(self, attr):
        return getattr(self.file, attr)

marcocle's avatar
marcocle committed
251
252
253
254

class CMT:
    """Small wrapper to call CMT.
    """
Marco Clemencic's avatar
Marco Clemencic committed
255
256

    def __init__(self, path=None):
marcocle's avatar
marcocle committed
257
258
259
        if path is None:
            path = os.getcwd()
        self.path = path
marcocle's avatar
marcocle committed
260

Marco Clemencic's avatar
Marco Clemencic committed
261
    def _run_cmt(self, command, args):
marcocle's avatar
marcocle committed
262
263
264
        # prepare command line
        if type(args) is str:
            args = [args]
Marco Clemencic's avatar
Marco Clemencic committed
265
        cmd = "cmt %s" % command
marcocle's avatar
marcocle committed
266
        for arg in args:
Marco Clemencic's avatar
Marco Clemencic committed
267
            cmd += ' "%s"' % arg
marcocle's avatar
marcocle committed
268

marcocle's avatar
marcocle committed
269
270
271
272
273
274
275
276
        # go to the execution directory
        olddir = os.getcwd()
        os.chdir(self.path)
        # run cmt
        result = os.popen4(cmd)[1].read()
        # return to the old directory
        os.chdir(olddir)
        return result
marcocle's avatar
marcocle committed
277

Marco Clemencic's avatar
Marco Clemencic committed
278
    def __getattr__(self, attr):
marcocle's avatar
marcocle committed
279
        return lambda args=[]: self._run_cmt(attr, args)
marcocle's avatar
marcocle committed
280

Marco Clemencic's avatar
Marco Clemencic committed
281
    def runtime_env(self, env=None):
marcocle's avatar
marcocle committed
282
283
284
285
286
287
288
289
        """Returns a dictionary containing the runtime environment produced by CMT.
        If a dictionary is passed a modified instance of it is returned.
        """
        if env is None:
            env = {}
        for l in self.setup("-csh").splitlines():
            l = l.strip()
            if l.startswith("setenv"):
Marco Clemencic's avatar
Marco Clemencic committed
290
                dummy, name, value = l.split(None, 3)
marcocle's avatar
marcocle committed
291
292
                env[name] = value.strip('"')
            elif l.startswith("unsetenv"):
Marco Clemencic's avatar
Marco Clemencic committed
293
                dummy, name = l.split(None, 2)
marcocle's avatar
marcocle committed
294
295
296
                if name in env:
                    del env[name]
        return env
Marco Clemencic's avatar
Marco Clemencic committed
297
298
299

    def show_macro(self, k):
        r = self.show(["macro", k])
marcocle's avatar
marcocle committed
300
301
302
        if r.find("CMT> Error: symbol not found") >= 0:
            return None
        else:
Marco Clemencic's avatar
Marco Clemencic committed
303
            return self.show(["macro_value", k]).strip()
marcocle's avatar
marcocle committed
304

305

Marco Clemencic's avatar
Marco Clemencic committed
306
# Locates an executable in the executables path ($PATH) and returns the full
marcocle's avatar
marcocle committed
307
#  path to it.
marcocle's avatar
marcocle committed
308
#  If the executable cannot be found, None is returned
marcocle's avatar
marcocle committed
309
def which(executable):
310
311
312
313
314
    """
    Locates an executable in the executables path ($PATH) and returns the full
    path to it.  An application is looked for with or without the '.exe' suffix.
    If the executable cannot be found, None is returned
    """
marcocle's avatar
marcocle committed
315
    if os.path.isabs(executable):
316
317
318
319
        if not os.path.exists(executable):
            if executable.endswith('.exe'):
                if os.path.exists(executable[:-4]):
                    return executable[:-4]
marcocle's avatar
marcocle committed
320
        return executable
marcocle's avatar
marcocle committed
321
    for d in os.environ.get("PATH").split(os.pathsep):
322
        fullpath = os.path.join(d, executable)
marcocle's avatar
marcocle committed
323
324
        if os.path.exists(fullpath):
            return fullpath
325
326
    if executable.endswith('.exe'):
        return which(executable[:-4])
marcocle's avatar
marcocle committed
327
328
    return None

Marco Clemencic's avatar
Marco Clemencic committed
329

marcocle's avatar
marcocle committed
330
def rationalizepath(p):
331
332
333
    np = os.path.normpath(os.path.expandvars(p))
    if os.path.exists(np):
        p = os.path.realpath(np)
marcocle's avatar
marcocle committed
334
335
    return p

Marco Clemencic's avatar
Marco Clemencic committed
336

337
338
339
340
341
342
343
344
345
346
347
348
349
350
# XML Escaping character
import re

# xml 1.0 valid characters:
#    Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
# so to invert that, not in Char ::
#       x0 - x8 | xB | xC | xE - x1F
#       (most control characters, though TAB, CR, LF allowed)
#       | #xD800 - #xDFFF
#       (unicode surrogate characters)
#       | #xFFFE | #xFFFF |
#       (unicode end-of-plane non-characters)
#       >= 110000
#       that would be beyond unicode!!!
Marco Clemencic's avatar
Marco Clemencic committed
351
352
353
_illegal_xml_chars_RE = re.compile(
    u'[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]')

354

Marco Clemencic's avatar
Marco Clemencic committed
355
def hexreplace(match):
356
    "Return the hex string "
Marco Clemencic's avatar
Marco Clemencic committed
357
358
    return "".join(map(hexConvert, match.group()))

359
360
361

def hexConvert(char):
    return hex(ord(char))
Marco Clemencic's avatar
Marco Clemencic committed
362
363


364
365
366
def convert_xml_illegal_chars(val):
    return _illegal_xml_chars_RE.sub(hexreplace, val)

Marco Clemencic's avatar
Marco Clemencic committed
367

368
369
370
371
372
373
374
375
def escape_xml_illegal_chars(val, replacement='?'):
    """Filter out characters that are illegal in XML.
    Looks for any character in val that is not allowed in XML
    and replaces it with replacement ('?' by default).

    """
    return _illegal_xml_chars_RE.sub(replacement, val)

Gitlab CI's avatar
Gitlab CI committed
376

marcocle's avatar
marcocle committed
377
378
379
########################################################################
# Output Validation Classes
########################################################################
Marco Clemencic's avatar
Marco Clemencic committed
380
381


marcocle's avatar
marcocle committed
382
383
384
385
386
class BasicOutputValidator:
    """Basic implementation of an option validator for Gaudi tests.
    This implementation is based on the standard (LCG) validation functions
    used in QMTest.
    """
Marco Clemencic's avatar
Marco Clemencic committed
387
388

    def __init__(self, ref, cause, result_key):
marcocle's avatar
marcocle committed
389
390
391
        self.reference = ref
        self.cause = cause
        self.result_key = result_key
marcocle's avatar
marcocle committed
392

marcocle's avatar
marcocle committed
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
    def __call__(self, out, result):
        """Validate the output of the program.

        'stdout' -- A string containing the data written to the standard output
        stream.

        'stderr' -- A string containing the data written to the standard error
        stream.

        'result' -- A 'Result' object. It may be used to annotate
        the outcome according to the content of stderr.

        returns -- A list of strings giving causes of failure."""

        causes = []
        # Check to see if theoutput matches.
        if not self.__CompareText(out, self.reference):
marcocle's avatar
marcocle committed
410
            causes.append(self.cause)
marcocle's avatar
marcocle committed
411
412
413
            result[self.result_key] = result.Quote(self.reference)

        return causes
marcocle's avatar
marcocle committed
414

marcocle's avatar
marcocle committed
415
416
417
418
419
420
421
422
423
424
425
426
    def __CompareText(self, s1, s2):
        """Compare 's1' and 's2', ignoring line endings.

        's1' -- A string.

        's2' -- A string.

        returns -- True if 's1' and 's2' are the same, ignoring
        differences in line endings."""

        # The "splitlines" method works independently of the line ending
        # convention in use.
427
428
        if ROOT6WorkAroundEnabled('ReadRootmapCheck'):
            # FIXME: (MCl) Hide warnings from new rootmap sanity check until we can fix them
Marco Clemencic's avatar
Marco Clemencic committed
429
            to_ignore = re.compile(
Gitlab CI's avatar
Gitlab CI committed
430
431
                r'Warning in <TInterpreter::ReadRootmapFile>: .* is already in .*'
            )
Marco Clemencic's avatar
Marco Clemencic committed
432

Gitlab CI's avatar
Gitlab CI committed
433
434
435
436
437
            def keep_line(l):
                return not to_ignore.match(l)

            return filter(keep_line, s1.splitlines()) == filter(
                keep_line, s2.splitlines())
438
439
        else:
            return s1.splitlines() == s2.splitlines()
marcocle's avatar
marcocle committed
440

Marco Clemencic's avatar
Marco Clemencic committed
441

marcocle's avatar
marcocle committed
442
443
444
class FilePreprocessor:
    """ Base class for a callable that takes a file and returns a modified
    version of it."""
Marco Clemencic's avatar
Marco Clemencic committed
445

marcocle's avatar
marcocle committed
446
447
    def __processLine__(self, line):
        return line
Marco Clemencic's avatar
Marco Clemencic committed
448

marcocle's avatar
marcocle committed
449
    def __call__(self, input):
Marco Clemencic's avatar
Marco Clemencic committed
450
        if hasattr(input, "__iter__"):
marcocle's avatar
marcocle committed
451
452
453
454
455
456
457
458
            lines = input
            mergeback = False
        else:
            lines = input.splitlines()
            mergeback = True
        output = []
        for l in lines:
            l = self.__processLine__(l)
Marco Clemencic's avatar
Marco Clemencic committed
459
460
461
462
            if l:
                output.append(l)
        if mergeback:
            output = '\n'.join(output)
marcocle's avatar
marcocle committed
463
        return output
Marco Clemencic's avatar
Marco Clemencic committed
464

marcocle's avatar
marcocle committed
465
    def __add__(self, rhs):
Marco Clemencic's avatar
Marco Clemencic committed
466
467
        return FilePreprocessorSequence([self, rhs])

marcocle's avatar
marcocle committed
468
469

class FilePreprocessorSequence(FilePreprocessor):
Marco Clemencic's avatar
Marco Clemencic committed
470
    def __init__(self, members=[]):
marcocle's avatar
marcocle committed
471
        self.members = members
Marco Clemencic's avatar
Marco Clemencic committed
472

marcocle's avatar
marcocle committed
473
474
    def __add__(self, rhs):
        return FilePreprocessorSequence(self.members + [rhs])
Marco Clemencic's avatar
Marco Clemencic committed
475

marcocle's avatar
marcocle committed
476
477
478
479
480
481
    def __call__(self, input):
        output = input
        for pp in self.members:
            output = pp(output)
        return output

Marco Clemencic's avatar
Marco Clemencic committed
482

marcocle's avatar
marcocle committed
483
class LineSkipper(FilePreprocessor):
Marco Clemencic's avatar
Marco Clemencic committed
484
    def __init__(self, strings=[], regexps=[]):
marcocle's avatar
marcocle committed
485
486
        import re
        self.strings = strings
Marco Clemencic's avatar
Marco Clemencic committed
487
        self.regexps = map(re.compile, regexps)
marcocle's avatar
marcocle committed
488

marcocle's avatar
marcocle committed
489
490
    def __processLine__(self, line):
        for s in self.strings:
Marco Clemencic's avatar
Marco Clemencic committed
491
492
            if line.find(s) >= 0:
                return None
marcocle's avatar
marcocle committed
493
        for r in self.regexps:
Marco Clemencic's avatar
Marco Clemencic committed
494
495
            if r.search(line):
                return None
marcocle's avatar
marcocle committed
496
497
        return line

Marco Clemencic's avatar
Marco Clemencic committed
498

marcocle's avatar
marcocle committed
499
500
501
502
503
class BlockSkipper(FilePreprocessor):
    def __init__(self, start, end):
        self.start = start
        self.end = end
        self._skipping = False
marcocle's avatar
marcocle committed
504

marcocle's avatar
marcocle committed
505
506
507
508
509
510
511
512
513
514
    def __processLine__(self, line):
        if self.start in line:
            self._skipping = True
            return None
        elif self.end in line:
            self._skipping = False
        elif self._skipping:
            return None
        return line

Marco Clemencic's avatar
Marco Clemencic committed
515

marcocle's avatar
marcocle committed
516
class RegexpReplacer(FilePreprocessor):
Marco Clemencic's avatar
Marco Clemencic committed
517
    def __init__(self, orig, repl="", when=None):
marcocle's avatar
marcocle committed
518
519
        if when:
            when = re.compile(when)
Marco Clemencic's avatar
Marco Clemencic committed
520
521
522
        self._operations = [(when, re.compile(orig), repl)]

    def __add__(self, rhs):
marcocle's avatar
marcocle committed
523
        if isinstance(rhs, RegexpReplacer):
Marco Clemencic's avatar
Marco Clemencic committed
524
            res = RegexpReplacer("", "", None)
marcocle's avatar
marcocle committed
525
526
527
528
            res._operations = self._operations + rhs._operations
        else:
            res = FilePreprocessor.__add__(self, rhs)
        return res
Marco Clemencic's avatar
Marco Clemencic committed
529

marcocle's avatar
marcocle committed
530
    def __processLine__(self, line):
Marco Clemencic's avatar
Marco Clemencic committed
531
        for w, o, r in self._operations:
marcocle's avatar
marcocle committed
532
533
534
535
            if w is None or w.search(line):
                line = o.sub(r, line)
        return line

Marco Clemencic's avatar
Marco Clemencic committed
536

marcocle's avatar
marcocle committed
537
# Common preprocessors
Marco Clemencic's avatar
Marco Clemencic committed
538
maskPointers = RegexpReplacer("0x[0-9a-fA-F]{4,16}", "0x########")
Gitlab CI's avatar
Gitlab CI committed
539
540
541
normalizeDate = RegexpReplacer(
    "[0-2]?[0-9]:[0-5][0-9]:[0-5][0-9] [0-9]{4}[-/][01][0-9][-/][0-3][0-9] *(CES?T)?",
    "00:00:00 1970-01-01")
marcocle's avatar
marcocle committed
542
543
544
545
546
547
548
normalizeEOL = FilePreprocessor()
normalizeEOL.__processLine__ = lambda line: str(line).rstrip() + '\n'

skipEmptyLines = FilePreprocessor()
# FIXME: that's ugly
skipEmptyLines.__processLine__ = lambda line: (line.strip() and line) or None

Marco Clemencic's avatar
Marco Clemencic committed
549
# Special preprocessor sorting the list of strings (whitespace separated)
marcocle's avatar
marcocle committed
550
#  that follow a signature on a single line
Marco Clemencic's avatar
Marco Clemencic committed
551
552


marcocle's avatar
marcocle committed
553
554
555
556
class LineSorter(FilePreprocessor):
    def __init__(self, signature):
        self.signature = signature
        self.siglen = len(signature)
Marco Clemencic's avatar
Marco Clemencic committed
557

marcocle's avatar
marcocle committed
558
559
    def __processLine__(self, line):
        pos = line.find(self.signature)
Marco Clemencic's avatar
Marco Clemencic committed
560
561
562
        if pos >= 0:
            line = line[:(pos + self.siglen)]
            lst = line[(pos + self.siglen):].split()
marcocle's avatar
marcocle committed
563
564
565
566
            lst.sort()
            line += " ".join(lst)
        return line

Marco Clemencic's avatar
Marco Clemencic committed
567

marcocle's avatar
marcocle committed
568
# Preprocessors for GaudiExamples
marcocle's avatar
marcocle committed
569
normalizeExamples = maskPointers + normalizeDate
Marco Clemencic's avatar
Marco Clemencic committed
570
for w, o, r in [
Gitlab CI's avatar
Gitlab CI committed
571
        # ("TIMER.TIMER",r"[0-9]", "0"), # Normalize time output
Marco Clemencic's avatar
Marco Clemencic committed
572
573
574
575
576
    ("TIMER.TIMER", r"\s+[+-]?[0-9]+[0-9.]*", " 0"),  # Normalize time output
    ("release all pending", r"^.*/([^/]*:.*)", r"\1"),
    ("0x########", r"\[.*/([^/]*.*)\]", r"[\1]"),
    ("^#.*file", r"file '.*[/\\]([^/\\]*)$", r"file '\1"),
    ("^JobOptionsSvc.*options successfully read in from",
Gitlab CI's avatar
Gitlab CI committed
577
578
579
580
581
     r"read in from .*[/\\]([^/\\]*)$",
     r"file \1"),  # normalize path to options
        # Normalize UUID, except those ending with all 0s (i.e. the class IDs)
    (None,
     r"[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}(?!-0{12})-[0-9A-Fa-f]{12}",
Marco Clemencic's avatar
Marco Clemencic committed
582
     "00000000-0000-0000-0000-000000000000"),
Gitlab CI's avatar
Gitlab CI committed
583
        # Absorb a change in ServiceLocatorHelper
Marco Clemencic's avatar
Marco Clemencic committed
584
585
    ("ServiceLocatorHelper::", "ServiceLocatorHelper::(create|locate)Service",
     "ServiceLocatorHelper::service"),
Gitlab CI's avatar
Gitlab CI committed
586
        # Remove the leading 0 in Windows' exponential format
Marco Clemencic's avatar
Marco Clemencic committed
587
    (None, r"e([-+])0([0-9][0-9])", r"e\1\2"),
Gitlab CI's avatar
Gitlab CI committed
588
        # Output line changed in Gaudi v24
Marco Clemencic's avatar
Marco Clemencic committed
589
590
    (None, r'Service reference count check:',
     r'Looping over all active services...'),
Gitlab CI's avatar
Gitlab CI committed
591
        # Change of property name in Algorithm (GAUDI-1030)
Marco Clemencic's avatar
Marco Clemencic committed
592
593
594
    (None, r"Property(.*)'ErrorCount':", r"Property\1'ErrorCounter':"),
]:  # [ ("TIMER.TIMER","[0-9]+[0-9.]*", "") ]
    normalizeExamples += RegexpReplacer(o, r, w)
595

Gitlab CI's avatar
Gitlab CI committed
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
lineSkipper = LineSkipper(
    [
        "//GP:",
        "JobOptionsSvc        INFO # ",
        "JobOptionsSvc     WARNING # ",
        "Time User",
        "Welcome to",
        "This machine has a speed",
        "TIME:",
        "running on",
        "ToolSvc.Sequenc...   INFO",
        "DataListenerSvc      INFO XML written to file:",
        "[INFO]",
        "[WARNING]",
        "DEBUG No writable file catalog found which contains FID:",
        "0 local",  # hack for ErrorLogExample
        "DEBUG Service base class initialized successfully",  # changed between v20 and v21
        "DEBUG Incident  timing:",  # introduced with patch #3487
        # changed the level of the message from INFO to DEBUG
        "INFO  'CnvServices':[",
        # The signal handler complains about SIGXCPU not defined on some platforms
        'SIGXCPU',
        # FIXME: special lines printed in GaudiHive
        'EventLoopMgr      SUCCESS Event Number = ',
        'EventLoopMgr      SUCCESS ---> Loop Finished',
    ],
    regexps=[
        r"^JobOptionsSvc        INFO *$",
        r"^#",  # Ignore python comments
        # skip the message reporting the version of the root file
        r"(Always|SUCCESS)\s*(Root f|[^ ]* F)ile version:",
        # hack for ErrorLogExample
        r"0x[0-9a-fA-F#]+ *Algorithm::sysInitialize\(\) *\[",
        # hack for ErrorLogExample
        r"0x[0-9a-fA-F#]* *__gxx_personality_v0 *\[",
        r"File '.*.xml' does not exist",
        r"INFO Refer to dataset .* by its file ID:",
        r"INFO Referring to dataset .* by its file ID:",
        r"INFO Disconnect from dataset",
        r"INFO Disconnected from dataset",
        r"INFO Disconnected data IO:",
        r"IncidentSvc\s*(DEBUG (Adding|Removing)|VERBOSE Calling)",
        # I want to ignore the header of the unchecked StatusCode report
        r"^StatusCodeSvc.*listing all unchecked return codes:",
        r"^StatusCodeSvc\s*INFO\s*$",
        r"Num\s*\|\s*Function\s*\|\s*Source Library",
        r"^[-+]*\s*$",
        # Hide the fake error message coming from POOL/ROOT (ROOT 5.21)
        r"ERROR Failed to modify file: .* Errno=2 No such file or directory",
        # Hide unchecked StatusCodes from dictionaries
        r"^ +[0-9]+ \|.*ROOT",
        r"^ +[0-9]+ \|.*\|.*Dict",
        # Hide success StatusCodeSvc message
        r"StatusCodeSvc.*all StatusCode instances where checked",
        # Hide EventLoopMgr total timing report
        r"EventLoopMgr.*---> Loop Finished",
        # Remove ROOT TTree summary table, which changes from one version to the other
        r"^\*.*\*$",
        # Remove Histos Summaries
        r"SUCCESS\s*Booked \d+ Histogram\(s\)",
        r"^ \|",
        r"^ ID=",
    ])
659
660
if ROOT6WorkAroundEnabled('ReadRootmapCheck'):
    # FIXME: (MCl) Hide warnings from new rootmap sanity check until we can fix them
Marco Clemencic's avatar
Marco Clemencic committed
661
    lineSkipper += LineSkipper(regexps=[
662
        r'Warning in <TInterpreter::ReadRootmapFile>: .* is already in .*',
Marco Clemencic's avatar
Marco Clemencic committed
663
    ])
664
665
666

normalizeExamples = (lineSkipper + normalizeExamples + skipEmptyLines +
                     normalizeEOL + LineSorter("Services to release : "))
marcocle's avatar
marcocle committed
667

Marco Clemencic's avatar
Marco Clemencic committed
668

marcocle's avatar
marcocle committed
669
class ReferenceFileValidator:
Marco Clemencic's avatar
Marco Clemencic committed
670
    def __init__(self, reffile, cause, result_key, preproc=normalizeExamples):
marcocle's avatar
marcocle committed
671
672
673
674
        self.reffile = os.path.expandvars(reffile)
        self.cause = cause
        self.result_key = result_key
        self.preproc = preproc
Marco Clemencic's avatar
Marco Clemencic committed
675

marcocle's avatar
marcocle committed
676
677
678
679
680
681
682
683
    def __call__(self, stdout, result):
        causes = []
        if os.path.isfile(self.reffile):
            orig = open(self.reffile).xreadlines()
            if self.preproc:
                orig = self.preproc(orig)
        else:
            orig = []
marcocle's avatar
marcocle committed
684

marcocle's avatar
marcocle committed
685
686
687
688
        new = stdout.splitlines()
        if self.preproc:
            new = self.preproc(new)
        #open(self.reffile + ".test","w").writelines(new)
Marco Clemencic's avatar
Marco Clemencic committed
689
        diffs = difflib.ndiff(orig, new, charjunk=difflib.IS_CHARACTER_JUNK)
Gitlab CI's avatar
Gitlab CI committed
690
691
        filterdiffs = map(lambda x: x.strip(),
                          filter(lambda x: x[0] != " ", diffs))
marcocle's avatar
marcocle committed
692
693
694
695
696
697
698
699
        #filterdiffs = [x.strip() for x in diffs]
        if filterdiffs:
            result[self.result_key] = result.Quote("\n".join(filterdiffs))
            result[self.result_key] += result.Quote("""
Legend:
        -) reference file
        +) standard output of the test""")
            causes.append(self.cause)
marcocle's avatar
marcocle committed
700

marcocle's avatar
marcocle committed
701
702
        return causes

Gitlab CI's avatar
Gitlab CI committed
703

marcocle's avatar
marcocle committed
704
705
706
########################################################################
# Useful validation functions
########################################################################
Marco Clemencic's avatar
Marco Clemencic committed
707
708


Gitlab CI's avatar
Gitlab CI committed
709
710
711
712
713
714
def findReferenceBlock(reference,
                       stdout,
                       result,
                       causes,
                       signature_offset=0,
                       signature=None,
Marco Clemencic's avatar
Marco Clemencic committed
715
                       id=None):
marcocle's avatar
marcocle committed
716
717
718
719
720
721
722
723
724
725
726
727
728
    """
    Given a block of text, tries to find it in the output.
    The block had to be identified by a signature line. By default, the first
    line is used as signature, or the line pointed to by signature_offset. If
    signature_offset points outside the block, a signature line can be passed as
    signature argument. Note: if 'signature' is None (the default), a negative
    signature_offset is interpreted as index in a list (e.g. -1 means the last
    line), otherwise the it is interpreted as the number of lines before the
    first one of the block the signature must appear.
    The parameter 'id' allow to distinguish between different calls to this
    function in the same validation code.
    """
    # split reference file, sanitize EOLs and remove empty lines
Marco Clemencic's avatar
Marco Clemencic committed
729
    reflines = filter(None, map(lambda s: s.rstrip(), reference.splitlines()))
marcocle's avatar
marcocle committed
730
731
732
    if not reflines:
        raise RuntimeError("Empty (or null) reference")
    # the same on standard output
Marco Clemencic's avatar
Marco Clemencic committed
733
    outlines = filter(None, map(lambda s: s.rstrip(), stdout.splitlines()))
marcocle's avatar
marcocle committed
734

marcocle's avatar
marcocle committed
735
736
    res_field = "GaudiTest.RefBlock"
    if id:
marcocle's avatar
marcocle committed
737
738
        res_field += "_%s" % id

marcocle's avatar
marcocle committed
739
740
    if signature is None:
        if signature_offset < 0:
Marco Clemencic's avatar
Marco Clemencic committed
741
            signature_offset = len(reference) + signature_offset
marcocle's avatar
marcocle committed
742
743
744
745
        signature = reflines[signature_offset]
    # find the reference block in the output file
    try:
        pos = outlines.index(signature)
Gitlab CI's avatar
Gitlab CI committed
746
747
        outlines = outlines[pos - signature_offset:pos + len(reflines) -
                            signature_offset]
marcocle's avatar
marcocle committed
748
749
750
        if reflines != outlines:
            msg = "standard output"
            # I do not want 2 messages in causes if teh function is called twice
marcocle's avatar
marcocle committed
751
            if not msg in causes:
marcocle's avatar
marcocle committed
752
753
754
755
756
757
758
                causes.append(msg)
            result[res_field + ".observed"] = result.Quote("\n".join(outlines))
    except ValueError:
        causes.append("missing signature")
    result[res_field + ".signature"] = result.Quote(signature)
    if len(reflines) > 1 or signature != reflines[0]:
        result[res_field + ".expected"] = result.Quote("\n".join(reflines))
marcocle's avatar
marcocle committed
759

marcocle's avatar
marcocle committed
760
761
    return causes

Marco Clemencic's avatar
Marco Clemencic committed
762
763

def countErrorLines(expected={'ERROR': 0, 'FATAL': 0}, **kwargs):
marcocle's avatar
marcocle committed
764
765
766
767
768
769
770
771
772
    """
    Count the number of messages with required severity (by default ERROR and FATAL)
    and check if their numbers match the expected ones (0 by default).
    The dictionary "expected" can be used to tune the number of errors and fatals
    allowed, or to limit the number of expected warnings etc.
    """
    stdout = kwargs["stdout"]
    result = kwargs["result"]
    causes = kwargs["causes"]
marcocle's avatar
marcocle committed
773

marcocle's avatar
marcocle committed
774
775
776
777
    # prepare the dictionary to record the extracted lines
    errors = {}
    for sev in expected:
        errors[sev] = []
marcocle's avatar
marcocle committed
778

marcocle's avatar
marcocle committed
779
780
    outlines = stdout.splitlines()
    from math import log10
Marco Clemencic's avatar
Marco Clemencic committed
781
    fmt = "%%%dd - %%s" % (int(log10(len(outlines)) + 1))
marcocle's avatar
marcocle committed
782

marcocle's avatar
marcocle committed
783
784
785
786
787
    linecount = 0
    for l in outlines:
        linecount += 1
        words = l.split()
        if len(words) >= 2 and words[1] in errors:
Marco Clemencic's avatar
Marco Clemencic committed
788
            errors[words[1]].append(fmt % (linecount, l.rstrip()))
marcocle's avatar
marcocle committed
789

marcocle's avatar
marcocle committed
790
791
    for e in errors:
        if len(errors[e]) != expected[e]:
Marco Clemencic's avatar
Marco Clemencic committed
792
            causes.append('%s(%d)' % (e, len(errors[e])))
Gitlab CI's avatar
Gitlab CI committed
793
794
795
796
            result["GaudiTest.lines.%s" % e] = result.Quote('\n'.join(
                errors[e]))
            result["GaudiTest.lines.%s.expected#" % e] = result.Quote(
                str(expected[e]))
marcocle's avatar
marcocle committed
797

marcocle's avatar
marcocle committed
798
799
800
801
802
803
804
805
806
807
    return causes


def _parseTTreeSummary(lines, pos):
    """
    Parse the TTree summary table in lines, starting from pos.
    Returns a tuple with the dictionary with the digested informations and the
    position of the first line after the summary.
    """
    result = {}
Marco Clemencic's avatar
Marco Clemencic committed
808
    i = pos + 1  # first line is a sequence of '*'
marcocle's avatar
marcocle committed
809
    count = len(lines)
marcocle's avatar
marcocle committed
810

Gitlab CI's avatar
Gitlab CI committed
811
812
    def splitcols(l):
        return [f.strip() for f in l.strip("*\n").split(':', 2)]
Marco Clemencic's avatar
Marco Clemencic committed
813

marcocle's avatar
marcocle committed
814
815
816
817
    def parseblock(ll):
        r = {}
        cols = splitcols(ll[0])
        r["Name"], r["Title"] = cols[1:]
marcocle's avatar
marcocle committed
818

marcocle's avatar
marcocle committed
819
820
        cols = splitcols(ll[1])
        r["Entries"] = int(cols[1])
marcocle's avatar
marcocle committed
821

marcocle's avatar
marcocle committed
822
823
824
825
826
827
        sizes = cols[2].split()
        r["Total size"] = int(sizes[2])
        if sizes[-1] == "memory":
            r["File size"] = 0
        else:
            r["File size"] = int(sizes[-1])
marcocle's avatar
marcocle committed
828

marcocle's avatar
marcocle committed
829
830
831
832
833
834
        cols = splitcols(ll[2])
        sizes = cols[2].split()
        if cols[0] == "Baskets":
            r["Baskets"] = int(cols[1])
            r["Basket size"] = int(sizes[2])
        r["Compression"] = float(sizes[-1])
marcocle's avatar
marcocle committed
835
836
        return r

marcocle's avatar
marcocle committed
837
    if i < (count - 3) and lines[i].startswith("*Tree"):
Marco Clemencic's avatar
Marco Clemencic committed
838
        result = parseblock(lines[i:i + 3])
marcocle's avatar
marcocle committed
839
840
841
        result["Branches"] = {}
        i += 4
        while i < (count - 3) and lines[i].startswith("*Br"):
842
843
844
845
            if i < (count - 2) and lines[i].startswith("*Branch "):
                # skip branch header
                i += 3
                continue
Marco Clemencic's avatar
Marco Clemencic committed
846
            branch = parseblock(lines[i:i + 3])
marcocle's avatar
marcocle committed
847
848
            result["Branches"][branch["Name"]] = branch
            i += 4
marcocle's avatar
marcocle committed
849

marcocle's avatar
marcocle committed
850
851
    return (result, i)

Marco Clemencic's avatar
Marco Clemencic committed
852

marcocle's avatar
marcocle committed
853
854
855
856
857
858
859
860
def findTTreeSummaries(stdout):
    """
    Scan stdout to find ROOT TTree summaries and digest them.
    """
    stars = re.compile(r"^\*+$")
    outlines = stdout.splitlines()
    nlines = len(outlines)
    trees = {}
marcocle's avatar
marcocle committed
861

marcocle's avatar
marcocle committed
862
    i = 0
Marco Clemencic's avatar
Marco Clemencic committed
863
    while i < nlines:  # loop over the output
marcocle's avatar
marcocle committed
864
865
866
867
868
869
870
        # look for
        while i < nlines and not stars.match(outlines[i]):
            i += 1
        if i < nlines:
            tree, i = _parseTTreeSummary(outlines, i)
            if tree:
                trees[tree["Name"]] = tree
marcocle's avatar
marcocle committed
871

marcocle's avatar
marcocle committed
872
873
    return trees

Marco Clemencic's avatar
Marco Clemencic committed
874
875

def cmpTreesDicts(reference, to_check, ignore=None):
marcocle's avatar
marcocle committed
876
877
878
879
    """
    Check that all the keys in reference are in to_check too, with the same value.
    If the value is a dict, the function is called recursively. to_check can
    contain more keys than reference, that will not be tested.
marcocle's avatar
marcocle committed
880
    The function returns at the first difference found.
marcocle's avatar
marcocle committed
881
882
883
884
885
    """
    fail_keys = []
    # filter the keys in the reference dictionary
    if ignore:
        ignore_re = re.compile(ignore)
Marco Clemencic's avatar
Marco Clemencic committed
886
        keys = [key for key in reference if not ignore_re.match(key)]
marcocle's avatar
marcocle committed
887
888
889
890
    else:
        keys = reference.keys()
    # loop over the keys (not ignored) in the reference dictionary
    for k in keys:
Marco Clemencic's avatar
Marco Clemencic committed
891
        if k in to_check:  # the key must be in the dictionary to_check
marcocle's avatar
marcocle committed
892
893
            if (type(reference[k]) is dict) and (type(to_check[k]) is dict):
                # if both reference and to_check values are dictionaries, recurse
Gitlab CI's avatar
Gitlab CI committed
894
895
                failed = fail_keys = cmpTreesDicts(reference[k], to_check[k],
                                                   ignore)
marcocle's avatar
marcocle committed
896
897
898
            else:
                # compare the two values
                failed = to_check[k] != reference[k]
Marco Clemencic's avatar
Marco Clemencic committed
899
        else:  # handle missing keys in the dictionary to check (i.e. failure)
marcocle's avatar
marcocle committed
900
901
902
903
            to_check[k] = None
            failed = True
        if failed:
            fail_keys.insert(0, k)
Marco Clemencic's avatar
Marco Clemencic committed
904
905
906
            break  # exit from the loop at the first failure
    return fail_keys  # return the list of keys bringing to the different values

marcocle's avatar
marcocle committed
907
908
909
910
911

def getCmpFailingValues(reference, to_check, fail_path):
    c = to_check
    r = reference
    for k in fail_path:
Marco Clemencic's avatar
Marco Clemencic committed
912
913
        c = c.get(k, None)
        r = r.get(k, None)
marcocle's avatar
marcocle committed
914
        if c is None or r is None:
Marco Clemencic's avatar
Marco Clemencic committed
915
            break  # one of the dictionaries is not deep enough
marcocle's avatar
marcocle committed
916
917
    return (fail_path, r, c)

Marco Clemencic's avatar
Marco Clemencic committed
918

marcocle's avatar
marcocle committed
919
920
921
# signature of the print-out of the histograms
h_count_re = re.compile(r"^(.*)SUCCESS\s+Booked (\d+) Histogram\(s\) :\s+(.*)")

Marco Clemencic's avatar
Marco Clemencic committed
922

marcocle's avatar
marcocle committed
923
924
925
926
927
928
def parseHistosSummary(lines, pos):
    """
    Extract the histograms infos from the lines starting at pos.
    Returns the position of the first line after the summary block.
    """
    global h_count_re
Marco Clemencic's avatar
Marco Clemencic committed
929
    h_table_head = re.compile(
Gitlab CI's avatar
Gitlab CI committed
930
931
        r'SUCCESS\s+(1D|2D|3D|1D profile|2D profile) histograms in directory\s+"(\w*)"'
    )
marcocle's avatar
marcocle committed
932
    h_short_summ = re.compile(r"ID=([^\"]+)\s+\"([^\"]+)\"\s+(.*)")
marcocle's avatar
marcocle committed
933

marcocle's avatar
marcocle committed
934
    nlines = len(lines)
marcocle's avatar
marcocle committed
935

marcocle's avatar
marcocle committed
936
937
938
939
940
    # decode header
    m = h_count_re.search(lines[pos])
    name = m.group(1).strip()
    total = int(m.group(2))
    header = {}
Marco Clemencic's avatar
Marco Clemencic committed
941
    for k, v in [x.split("=") for x in m.group(3).split()]:
marcocle's avatar
marcocle committed
942
943
944
        header[k] = int(v)
    pos += 1
    header["Total"] = total
marcocle's avatar
marcocle committed
945

marcocle's avatar
marcocle committed
946
947
948
949
    summ = {}
    while pos < nlines:
        m = h_table_head.search(lines[pos])
        if m:
Marco Clemencic's avatar
Marco Clemencic committed
950
            t, d = m.groups(1)  # type and directory
marcocle's avatar
marcocle committed
951
952
953
954
955
956
957
958
959
            t = t.replace(" profile", "Prof")
            pos += 1
            if pos < nlines:
                l = lines[pos]
            else:
                l = ""
            cont = {}
            if l.startswith(" | ID"):
                # table format
Marco Clemencic's avatar
Marco Clemencic committed
960
                titles = [x.strip() for x in l.split("|")][1:]
marcocle's avatar
marcocle committed
961
962
963
                pos += 1
                while pos < nlines and lines[pos].startswith(" |"):
                    l = lines[pos]
Marco Clemencic's avatar
Marco Clemencic committed
964
                    values = [x.strip() for x in l.split("|")][1:]
marcocle's avatar
marcocle committed
965
966
967
968
969
970
971
                    hcont = {}
                    for i in range(len(titles)):
                        hcont[titles[i]] = values[i]
                    cont[hcont["ID"]] = hcont
                    pos += 1
            elif l.startswith(" ID="):
                while pos < nlines and lines[pos].startswith(" ID="):
Gitlab CI's avatar
Gitlab CI committed
972
973
974
975
                    values = [
                        x.strip()
                        for x in h_short_summ.search(lines[pos]).groups()
                    ]
marcocle's avatar
marcocle committed
976
977
                    cont[values[0]] = values
                    pos += 1
Marco Clemencic's avatar
Marco Clemencic committed
978
979
980
            else:  # not interpreted
                raise RuntimeError(
                    "Cannot understand line %d: '%s'" % (pos, l))
marcocle's avatar
marcocle committed
981
982
983
984
985
986
987
988
989
990
991
            if not d in summ:
                summ[d] = {}
            summ[d][t] = cont
            summ[d]["header"] = header
        else:
            break
    if not summ:
        # If the full table is not present, we use only the header
        summ[name] = {"header": header}
    return summ, pos

Marco Clemencic's avatar
Marco Clemencic committed
992

marcocle's avatar
marcocle committed
993
994
995
996
997
998
999
1000
def findHistosSummaries(stdout):
    """
    Scan stdout to find ROOT TTree summaries and digest them.
    """
    outlines = stdout.splitlines()
    nlines = len(outlines) - 1
    summaries = {}
    global h_count_re
marcocle's avatar
marcocle committed
1001

marcocle's avatar
marcocle committed
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
    pos = 0
    while pos < nlines:
        summ = {}
        # find first line of block:
        match = h_count_re.search(outlines[pos])
        while pos < nlines and not match:
            pos += 1
            match = h_count_re.search(outlines[pos])
        if match:
            summ, pos = parseHistosSummary(outlines, pos)
        summaries.update(summ)
    return summaries

Marco Clemencic's avatar
Marco Clemencic committed
1015

marcocle's avatar
marcocle committed
1016
class GaudiFilterExecutable(qm.executable.Filter):
Marco Clemencic's avatar
Marco Clemencic committed
1017
    def __init__(self, input, timeout=-1):
marcocle's avatar
marcocle committed
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
        """Create a new 'Filter'.

        'input' -- The string containing the input to provide to the
        child process.

        'timeout' -- As for 'TimeoutExecutable.__init__'."""

        super(GaudiFilterExecutable, self).__init__(input, timeout)
        self.__input = input
        self.__timeout = timeout
        self.stack_trace_file = None
        # Temporary file to pass the stack trace from one process to the other
        # The file must be closed and reopened when needed to avoid conflicts
        # between the processes
        tmpf = tempfile.mkstemp()
        os.close(tmpf[0])
Marco Clemencic's avatar
Marco Clemencic committed
1034
        self.stack_trace_file = tmpf[1]  # remember only the name
marcocle's avatar
marcocle committed
1035

marcocle's avatar
marcocle committed
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
    def __UseSeparateProcessGroupForChild(self):
        """Copied from TimeoutExecutable to allow the re-implementation of
           _HandleChild.
        """
        if sys.platform == "win32":
            # In Windows 2000 (or later), we should use "jobs" by
            # analogy with UNIX process groups.  However, that
            # functionality is not (yet) provided by the Python Win32
            # extensions.
            return 0
marcocle's avatar
marcocle committed
1046

marcocle's avatar
marcocle committed
1047
        return self.__timeout >= 0 or self.__timeout == -2
Gitlab CI's avatar
Gitlab CI committed
1048

marcocle's avatar
marcocle committed
1049
1050
    ##
    # Needs to replace the ones from RedirectedExecutable and TimeoutExecutable
Marco Clemencic's avatar
Marco Clemencic committed
1051

marcocle's avatar
marcocle committed
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
    def _HandleChild(self):
        """Code copied from both FilterExecutable and TimeoutExecutable.
        """
        # Close the pipe ends that we do not need.
        if self._stdin_pipe:
            self._ClosePipeEnd(self._stdin_pipe[0])
        if self._stdout_pipe:
            self._ClosePipeEnd(self._stdout_pipe[1])
        if self._stderr_pipe:
            self._ClosePipeEnd(self._stderr_pipe[1])

        # The pipes created by 'RedirectedExecutable' must be closed
        # before the monitor process (created by 'TimeoutExecutable')
        # is created.  Otherwise, if the child process dies, 'select'
        # in the parent will not return if the monitor process may
        # still have one of the file descriptors open.
marcocle's avatar
marcocle committed
1068

marcocle's avatar
marcocle committed
1069
        super(qm.executable.TimeoutExecutable, self)._HandleChild()
marcocle's avatar
marcocle committed
1070

marcocle's avatar
marcocle committed
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
        if self.__UseSeparateProcessGroupForChild():
            # Put the child into its own process group.  This step is
            # performed in both the parent and the child; therefore both
            # processes can safely assume that the creation of the process
            # group has taken place.
            child_pid = self._GetChildPID()
            try:
                os.setpgid(child_pid, child_pid)
            except:
                # The call to setpgid may fail if the child has exited,
                # or has already called 'exec'.  In that case, we are
                # guaranteed that the child has already put itself in the
                # desired process group.
                pass
            # Create the monitoring process.
            #
            # If the monitoring process is in parent's process group and
            # kills the child after waitpid has returned in the parent, we
            # may end up trying to kill a process group other than the one
            # that we intend to kill.  Therefore, we put the monitoring
            # process in the same process group as the child; that ensures
            # that the process group will persist until the monitoring
            # process kills it.
            self.__monitor_pid = os.fork()
            if self.__monitor_pid != 0:
                # Make sure that the monitoring process is placed into the
                # child's process group before the parent process calls
                # 'waitpid'.  In this way, we are guaranteed that the process
marcocle's avatar
marcocle committed
1099
                # group as the child
marcocle's avatar
marcocle committed
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
                os.setpgid(self.__monitor_pid, child_pid)
            else:
                # Put the monitoring process into the child's process
                # group.  We know the process group still exists at
                # this point because either (a) we are in the process
                # group, or (b) the parent has not yet called waitpid.
                os.setpgid(0, child_pid)

                # Close all open file descriptors.  They are not needed
                # in the monitor process.  Furthermore, when the parent
                # closes the write end of the stdin pipe to the child,
                # we do not want the pipe to remain open; leaving the
                # pipe open in the monitor process might cause the child
                # to block waiting for additional input.
                try:
                    max_fds = os.sysconf("SC_OPEN_MAX")
                except:
                    max_fds = 256
                for fd in xrange(max_fds):
                    try:
                        os.close(fd)
                    except:
                        pass
                try:
                    if self.__timeout >= 0:
                        # Give the child time to run.
Marco Clemencic's avatar
Marco Clemencic committed
1126
                        time.sleep(self.__timeout)
marcocle's avatar
marcocle committed
1127
                        #######################################################
Marco Clemencic's avatar
Marco Clemencic committed
1128
1129
                        # This is the interesting part: dump the stack trace to a file
                        if sys.platform == "linux2":  # we should be have /proc and gdb
Gitlab CI's avatar
Gitlab CI committed
1130
1131
1132
1133
1134
1135
1136
1137
                            cmd = [
                                "gdb",
                                os.path.join("/proc", str(child_pid), "exe"),
                                str(child_pid), "-batch", "-n", "-x",
                                "'%s'" % os.path.join(
                                    os.path.dirname(__file__),
                                    "stack-trace.gdb")
                            ]
marcocle's avatar
marcocle committed
1138
1139
1140
                            # FIXME: I wanted to use subprocess.Popen, but it doesn't want to work
                            #        in this context.
                            o = os.popen(" ".join(cmd)).read()
Marco Clemencic's avatar
Marco Clemencic committed
1141
                            open(self.stack_trace_file, "w").write(o)
marcocle's avatar
marcocle committed
1142
                        #######################################################
marcocle's avatar
marcocle committed
1143

marcocle's avatar
marcocle committed
1144
1145
1146
1147
                        # Kill all processes in the child process group.
                        os.kill(0, signal.SIGKILL)
                    else:
                        # This call to select will never terminate.
Marco Clemencic's avatar
Marco Clemencic committed
1148
                        select.select([], [], [])
marcocle's avatar
marcocle committed
1149
1150
1151
1152
1153
1154
                finally:
                    # Exit.  This code is in a finally clause so that
                    # we are guaranteed to get here no matter what.
                    os._exit(0)
        elif self.__timeout >= 0 and sys.platform == "win32":
            # Create a monitoring thread.
Marco Clemencic's avatar
Marco Clemencic committed
1155
            self.__monitor_thread = Thread(target=self.__Monitor)
marcocle's avatar
marcocle committed
1156
1157
1158
1159
1160
1161
1162
1163
1164
            self.__monitor_thread.start()

    if sys.platform == "win32":

        def __Monitor(self):
            """Code copied from FilterExecutable.
            Kill the child if the timeout expires.

            This function is run in the monitoring thread."""
marcocle's avatar
marcocle committed
1165

marcocle's avatar
marcocle committed
1166
1167
1168
1169
1170