From 419970d5d36185e7837bd39a546c61d5b2a99222 Mon Sep 17 00:00:00 2001 From: Silia TAIDER <silia.taider@cern.ch> Date: Thu, 12 Sep 2024 15:47:00 +0200 Subject: [PATCH] Move functionalities from the makereport hook to logreport --- .../python/GaudiTesting/FixtureResult.py | 13 ++- .../python/GaudiTesting/SubprocessBaseTest.py | 14 +-- .../pytest/ctest_measurements_reporter.py | 99 +++++++------------ .../python/GaudiTesting/pytest/fixtures.py | 25 +---- 4 files changed, 56 insertions(+), 95 deletions(-) diff --git a/GaudiPolicy/python/GaudiTesting/FixtureResult.py b/GaudiPolicy/python/GaudiTesting/FixtureResult.py index 67fac425b2..f2bc0c1cd5 100644 --- a/GaudiPolicy/python/GaudiTesting/FixtureResult.py +++ b/GaudiPolicy/python/GaudiTesting/FixtureResult.py @@ -35,7 +35,7 @@ class FixtureResult: completed_process: subprocess.CompletedProcess, start_time: datetime, end_time: datetime, - failure: Union[ProcessTimeoutError, ExceededStreamError, None], + run_exception: Union[ProcessTimeoutError, ExceededStreamError, None], command: List[str], expanded_command: List[str], env: dict, @@ -44,7 +44,7 @@ class FixtureResult: self.completed_process = completed_process self.start_time = start_time self.end_time = end_time - self.failure = failure + self.run_exception = run_exception self.command = command self.expanded_command = expanded_command self.runtime_environment = env @@ -84,13 +84,12 @@ class FixtureResult: "return_code": self.completed_process.returncode if self.completed_process.returncode else None, - "run_exception": str(self.failure) if self.failure else None, "elapsed_time": self.elapsed_time(), - "stack_trace": self.failure.stack_trace - if hasattr(self.failure, "stack_trace") + "stack_trace": self.run_exception.stack_trace + if hasattr(self.run_exception, "stack_trace") else None, - "stream_exceeded": self.failure.exceeded_stream - if hasattr(self.failure, "exceeded_stream") + "stream_exceeded": self.run_exception.exceeded_stream + if hasattr(self.run_exception, "exceeded_stream") else None, } ) diff --git a/GaudiPolicy/python/GaudiTesting/SubprocessBaseTest.py b/GaudiPolicy/python/GaudiTesting/SubprocessBaseTest.py index 5e9e0f9722..6022e1f167 100644 --- a/GaudiPolicy/python/GaudiTesting/SubprocessBaseTest.py +++ b/GaudiPolicy/python/GaudiTesting/SubprocessBaseTest.py @@ -166,7 +166,7 @@ class SubprocessBaseTest: stdout_chunks, stderr_chunks = [], [] stdout = stderr = "" - exceeded_stream = stack_trace = failure = None + exceeded_stream = stack_trace = run_exception = None streams = { proc.stdout.fileno(): (stdout_chunks, "stdout"), proc.stderr.fileno(): (stderr_chunks, "stderr"), @@ -193,9 +193,11 @@ class SubprocessBaseTest: if thread.is_alive(): stack_trace = cls._handle_timeout(proc) - failure = ProcessTimeoutError("Process timed out", stack_trace) + run_exception = ProcessTimeoutError("Process timed out", stack_trace) elif exceeded_stream: - failure = ExceededStreamError("Stream exceeded size limit", exceeded_stream) + run_exception = ExceededStreamError( + "Stream exceeded size limit", exceeded_stream + ) end_time = datetime.now() @@ -210,7 +212,7 @@ class SubprocessBaseTest: completed_process=completed_process, start_time=start_time, end_time=end_time, - failure=failure, + run_exception=run_exception, command=cls.command, expanded_command=command, env=env, @@ -233,8 +235,8 @@ class SubprocessBaseTest: if reference_path: record_property("reference_file", str(reference_path)) - if fixture_result.failure: - pytest.fail(f"{fixture_result.failure}") + if fixture_result.run_exception: + pytest.fail(f"{fixture_result.run_exception}") @pytest.mark.do_not_collect_source def test_returncode(self, returncode: int) -> None: diff --git a/GaudiPolicy/python/GaudiTesting/pytest/ctest_measurements_reporter.py b/GaudiPolicy/python/GaudiTesting/pytest/ctest_measurements_reporter.py index 8636613f26..b640c7eee5 100644 --- a/GaudiPolicy/python/GaudiTesting/pytest/ctest_measurements_reporter.py +++ b/GaudiPolicy/python/GaudiTesting/pytest/ctest_measurements_reporter.py @@ -11,18 +11,19 @@ import os import re import xml.sax.saxutils as XSS +from collections import defaultdict # This plugin integrates with pytest to capture and report test results # in a format compatible with CTest using DartMeasurement tags - # Key functions and hooks include: -# - sanitize_for_xml: Sanitizes a string by escaping non-XML characters. -# - pytest_configure: Initializes a dictionary to store test suite properties. -# - pytest_runtest_makereport: Collects information about failing tests and -# exceptions during test execution. -# - pytest_sessionfinish: Outputs the collected test information in a format +# - sanitize_for_xml: Sanitize a string by escaping non-XML characters. +# - pytest_report_header: Add strings to the pytest header. +# - pytest_runtest_logreport: Collect test results and durations. +# - pytest_sessionfinish: Output the collected test information in a format # suitable for CTest. +results = {} + def sanitize_for_xml(data): bad_chars = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\ud800-\udfff\ufffe\uffff]") @@ -34,44 +35,31 @@ def sanitize_for_xml(data): return bad_chars.sub(quote, data) -def pytest_sessionstart(session): - # list passed, failed and skipped tests - session.results = {"pass": set(), "fail": set(), "skip": set()} +def pytest_report_header(config, start_path, startdir): # make sure CTest does not drop output lines on successful tests - print("CTEST_FULL_OUTPUT") - - -def _skipped_item(item): - if any(item.iter_markers(name="skip")): - return True - if any(mark.args[0] for mark in item.iter_markers(name="skipif")): - return True - return False - - -def pytest_runtest_makereport(item, call): - name = ( - f"{item.cls.__name__}.{item.name}" - if hasattr(item, "cls") and item.cls is not None - else item.name + return "CTEST_FULL_OUTPUT" + + +def pytest_runtest_logreport(report): + # collect user properties + head_line = report.head_line + results.update( + { + f"{head_line}.{k}": v + for k, v in report.user_properties + if f"{head_line}.{k}" not in results + } ) - result = None - if call.when == "setup": - if _skipped_item(item): - result = "skip" - elif call.when == "call": - if call.excinfo is not None: - if call.excinfo.typename == "Skipped": - result = "skip" - else: - result = "fail" - item.user_properties.append(("exception_info", str(call.excinfo.value))) - else: - result = "pass" + # collect test outcome + if not report.passed: + results[f"{report.head_line}.outcome"] = report.outcome + else: + results.setdefault(f"{report.head_line}.outcome", "passed") - if result is not None: - item.session.results[result].add(name) + # collect test duration + if report.when == "call": + results[f"{report.head_line}.duration"] = round(report.duration, 2) def pytest_sessionfinish(session, exitstatus): @@ -82,27 +70,13 @@ def pytest_sessionfinish(session, exitstatus): # user requested to disable CTest measurements printouts return - results = list((name, sorted(value)) for name, value in session.results.items()) - - prefix = "" - for item in session.items: - prefix = ( - f"{item.cls.__name__}.{item.name}" - if hasattr(item, "cls") and item.cls is not None - else item.name - ) - for name, value in item.user_properties: - results.append((f"{prefix}.{name}", value)) - - if hasattr(session, "sources"): - results.extend( - (f"{name}.source_code", value) for name, value in session.sources.items() - ) - - if hasattr(session, "docstrings"): - results.extend( - (f"{name}.doc", value) for name, value in session.docstrings.items() - ) + outcomes = defaultdict(list) + for key in results: + if key.endswith(".outcome"): + outcomes[results[key]].append(key[:-8]) + results.update( + (f"outcome.{outcome}", sorted(tests)) for outcome, tests in outcomes.items() + ) ignore_keys = {"test_fixture_setup.completed_process"} template = ( @@ -111,9 +85,10 @@ def pytest_sessionfinish(session, exitstatus): to_print = [ (key, value) - for key, value in results + for key, value in results.items() if not any(key.endswith(ignore_key) for ignore_key in ignore_keys) and value ] + to_print.sort() for key, value in to_print: sanitized_value = XSS.escape(sanitize_for_xml(str(value))) # workaround for a limitation of CTestXML2HTML diff --git a/GaudiPolicy/python/GaudiTesting/pytest/fixtures.py b/GaudiPolicy/python/GaudiTesting/pytest/fixtures.py index 969914a105..c885ed380e 100644 --- a/GaudiPolicy/python/GaudiTesting/pytest/fixtures.py +++ b/GaudiPolicy/python/GaudiTesting/pytest/fixtures.py @@ -11,7 +11,6 @@ import inspect import os import subprocess -import time from collections import defaultdict from pathlib import Path from typing import Callable, Generator, Optional @@ -19,6 +18,7 @@ from typing import Callable, Generator, Optional import pytest import yaml from GaudiTesting.FixtureResult import FixtureResult +from GaudiTesting.pytest.ctest_measurements_reporter import results from GaudiTesting.SubprocessBaseTest import SubprocessBaseTest from GaudiTesting.utils import ( CodeWrapper, @@ -51,7 +51,6 @@ yaml.representer.SafeRepresenter.add_representer( # - returncode: Captures the return code of the subprocess. # - cwd: Captures the directory in which the program was executed. # - check_for_exceptions: Skips the test if exceptions are found in the result. -# - capture_validation_time: Captures the validation time for each test function. # - capture_class_docstring: Captures the docstring of the test class. # - reference: Creates a .new file if the output data is different from the reference. @@ -67,11 +66,6 @@ def pytest_configure(config): ) -def pytest_sessionstart(session): - session.sources = {} - session.docstrings = {} - - def pytest_collection_modifyitems(config, items): """ Record source code of tests. @@ -86,7 +80,7 @@ def pytest_collection_modifyitems(config, items): else item.originalname ) source_code = CodeWrapper(inspect.getsource(item.function), "python") - item.session.sources[name] = source_code + results[f"{name}.source_code"] = source_code def _get_shared_cwd_id(cls: type) -> Optional[str]: @@ -174,19 +168,10 @@ def check_for_exceptions( ) -> None: if ( fixture_result - and fixture_result.failure is not None + and fixture_result.run_exception is not None and "test_fixture_setup" not in request.keywords ): - pytest.skip(f"{fixture_result.failure}") - - -@pytest.fixture(scope="function", autouse=True) -def capture_validation_time( - record_property: Callable[[str, str], None], -) -> Generator[None, None, None]: - val_start_time = time.perf_counter() - yield - record_property("validate_time", round(time.perf_counter() - val_start_time, 2)) + pytest.skip(f"{fixture_result.run_exception}") @pytest.fixture(scope="class", autouse=True) @@ -195,7 +180,7 @@ def capture_class_docstring( ) -> None: cls = request.cls if cls and cls.__doc__: - request.session.docstrings[cls.__name__] = inspect.getdoc(cls) + results[f"{cls.__name__}.doc"] = inspect.getdoc(cls) @pytest.fixture(scope="class") -- GitLab