Initial CI artifacts copying prototype

AtlasTest/CITest/CMakeLists.txt

@@ -26,3 +26,6 @@ foreach( project Athena AnalysisBase AthGeneration AthSimulation )
       include( ${project}.cmake )
    endif()
 endforeach()
+
+# Install scripts
+atlas_install_runtime( scripts/*.sh )

AtlasTest/CITest/scripts/CopyCIArtifact.sh

+#!/usr/bin/bash
+#
+# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
+#
+# A script used to copy CI artifacts in a common area
+#
+# This script works only
+# - in ctest-based tests (detected by ATLAS_CTEST_TESTNAME)
+# - in the CI (detected by gitlabMergeRequestIid).
+# Multiple files can be passed to the command
+#   CopyCIArtifact.sh FILE...
+# Files are copied to
+#   /eos/atlas/atlascerngroupdisk/proj-sit/gitlabci/MR<number>_<hash>/<test_name>
+#
+
+# exit on failure
+set -e
+
+# check if we are running in the CI
+if [[ -z ${gitlabMergeRequestIid+x} ]]; then
+  echo "Not running in the CI. No files will be copied"
+  exit 0
+fi
+
+# check if we are running inside a test
+if [[ -z ${ATLAS_CTEST_TESTNAME+x} ]]; then
+  echo "Not running as part of a test. No files will be copied"
+  exit 0
+fi
+
+# check if a file was passed
+if [[ -z ${1+x} ]]; then
+  echo "Error: No file passed to the script" 1>&2
+  echo "Usage: CopyCIArtifact.sh FILE..." 1>&2
+  exit 1
+fi
+
+# check if we have EOS access
+base_folder="/eos/atlas/atlascerngroupdisk/proj-sit/gitlabci"
+if [[ ! -d "${base_folder}" ]]; then
+  echo "Error: CI artifacts area is not accessible" 1>&2
+  exit 2
+fi
+
+# generate output folder name
+output_folder="${base_folder}/MR${gitlabMergeRequestIid}"
+if [[ -n ${gitlabMergeRequestLastCommit+x} ]]; then
+  output_folder="${output_folder}_${gitlabMergeRequestLastCommit}"
+fi
+output_folder="${output_folder}/${ATLAS_CTEST_TESTNAME}"
+
+# make the output folder
+mkdir -p "${output_folder}"
+
+# copy the file
+cp "${@}" "${output_folder}"
+
+echo "Copied '${*}' to '${output_folder}'"

Tools/WorkflowTestRunner/python/Checks.py

@@ -188,6 +188,19 @@ class FrozenTier0PolicyCheck(WorkflowCheck):
 
             self.logger.error(f"Your change breaks the frozen tier0 policy in test {test.ID}.")
             self.logger.error("Please make sure this has been discussed in the correct meeting (RIG or Simulation) meeting and approved by the relevant experts.")
+
+            # copy the artifacts
+            if self.setup.disable_release_setup:
+                comparison_command = f"CopyCIArtifact.sh {validation_file}"
+                output, error = subprocess.Popen(["/bin/bash", "-c", comparison_command], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+                output, error = output.decode("utf-8"), error.decode("utf-8")
+
+                if error or not output:
+                    self.logger.error(f"Tried copying '{validation_file}' to the CI artifacts area but it failed.")
+                    self.logger.error(f"  {error.strip()}")
+                else:
+                    self.logger.error(output)
+
             with log_file.open() as file:
                 for line in file:
                     self.logger.info(f"  {line.strip()}")

Tools/WorkflowTestRunner/python/ScriptUtils.py

@@ -163,12 +163,7 @@ def get_test_setup(name: str, options: Namespace, log: logging.Logger) -> TestSe
 
     # Is an ATLAS release setup?
     if "AtlasPatchVersion" not in environ and "AtlasArea" not in environ and "AtlasBaseDir" not in environ and "AtlasVersion" not in environ:
-        log.error("Exit. Please setup the an ATLAS release")
-        exit(3)
-
-
-    if "AtlasPatchVersion" not in environ and "AtlasArea" not in environ and "AtlasBaseDir" in environ and "AtlasVersion" not in environ:
-        log.warning("Please be aware that you are running a release which seems to not be a Tier0 release, where in general q-tests are not guaranteed to work.")
+        log.warning("Not running in a standard ATLAS release setup.\n")
 
     # setup reference path
     setup.reference_run_path /= f"reference_test_{setup.unique_ID}"

Tools/WorkflowTestRunner/python/Test.py

@@ -36,6 +36,10 @@ class TestSetup:
             self.logger.info(f"WARNING: You have specified a dedicated release as reference {reference} and as validation {validation} release.")
             self.logger.info("Your local setup area will not be considered!!!")
             self.logger.info("this option is mainly designed for comparing release versions!!")
+        elif reference:
+            self.release_reference = reference
+            self.release_validation = ''
+            self.logger.info(f"You have specified a dedicated release as reference {reference}.")
         else:
             self.release_reference = get_release_setup(self.logger, self.disable_release_setup)
             self.release_validation = self.release_reference
@@ -130,7 +134,7 @@ class WorkflowTest:
         self.validation_path.mkdir(parents=True, exist_ok=True)
 
         cmd = f"cd {self.setup.pwd};"
-        if self.setup.disable_release_setup:
+        if self.setup.disable_release_setup or not self.setup.release_validation:
             pass
         elif "WorkDir_DIR" in environ:
             cmake_build_dir = environ["WorkDir_DIR"]