.gitlab-ci.yml

@@ -29,7 +29,7 @@ basic_tests:
   <<: *test_definition
   script:
     - pip install .
-    - pytest -v
+    - pytest
 
 lint:
   <<: *test_definition

environment.yaml

@@ -31,6 +31,7 @@ dependencies:
   - python2-secrets
   - pyyaml
   - lbapcommon >=0.0.2
+  - gitlab-runner-api
   # Avoid https://github.com/pypa/setuptools_scm/issues/440
   - setuptools_scm !=4.0.0
   - six
@@ -40,7 +41,6 @@ dependencies:
   - requests
   - docker-py
   - pip:
-    - git+https://gitlab.cern.ch/lhcb-gitlab-runners/runner-api.git
     - git+https://gitlab.cern.ch/cburr/dirac_prod.git
   # Testing dependencies
   - pytest

setup.cfg

@@ -2,4 +2,4 @@
 max_line_length = 120
 
 [tool:pytest]
-addopts = --cov=LbAnalysisProductions --cov-report=term-missing
+addopts = "--cov=LbAnalysisProductions --cov-report=term-missing -rx -v --color=yes --tb=long"

src/LbAnalysisProductions/config.py

@@ -18,6 +18,7 @@ import os
 import boto3
 from botocore.client import ClientError as BotoClientError
 import git
+import LbAPCommon
 
 
 def get_activation_cmd():
@@ -92,25 +93,4 @@ if cern_user_to_dirac_nick_fn:
     with open(cern_user_to_dirac_nick_fn, "rt") as fp:
         CERN_USER_TO_DIRAC_NICK = json.load(fp)
 
-known_working_groups = [
-    "B2CC",
-    "B2OC",
-    "BandQ",
-    "BnoC",
-    "Calib",
-    "Calo",
-    "Charm",
-    "DPA",
-    "HLT",
-    "IFT",
-    "Luminosity",
-    "PID",
-    "QCD",
-    "QEE",
-    "RD",
-    "RTA",
-    "Simulation",
-    "SL",
-    "Tagging",
-    "Tracking",
-]
+known_working_groups = LbAPCommon.config.known_working_groups

src/LbAnalysisProductions/gitlab_runner.py

@@ -517,14 +517,23 @@ def update_test_status(test):
                         lfn, repr(test.job.parent.tests[-1].output_files)
                     )
 
-        (
-            test.events_processed,
-            test.events_requested,
-            test.run_time,
-            test.cpu_norm,
-            input_files,
-            output_files,
-        ) = validators.parse_bookkeeping_xml(obj["Body"], expected_input_files)
+        try:
+            (
+                test.events_processed,
+                test.events_requested,
+                test.run_time,
+                test.cpu_norm,
+                input_files,
+                output_files,
+            ) = validators.parse_bookkeeping_xml(obj["Body"], expected_input_files)
+        except Exception:
+            log.exception("Failed to parse bookkeeping XML")
+            log.error(
+                "Error parsing bookkeeping XML for %s %s, marking as failed",
+                test.production.name,
+                test.job.name,
+            )
+            test.status = TestStatusEnum.FAILED
         test.input_files = [
             InputFile(path=f.path, size=f.size, dataset_size=f.dataset_size)
             for f in input_files

src/LbAnalysisProductions/job_configuration.py

@@ -22,6 +22,8 @@ __all__ = [
 import random
 import re
 
+import six
+
 from LbAnalysisProductions.logging import log
 
 
@@ -70,19 +72,32 @@ def from_bk_query(bk_query, is_turbo):
         from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import (
             BookkeepingClient,
         )
+        from LHCbDIRAC.TransformationSystem.Client.TransformationClient import (
+            TransformationClient,
+        )
 
         dddb_tags = set()
         conddb_tags = set()
         for prodID in BKQuery(bk_query).getBKProductions():
-            res = BookkeepingClient().getSteps(prodID)
-            if not res["OK"]:
-                raise RuntimeError(res)
-            for _, _, _, _, dddb, conddb, _, _, _ in res["Value"]:
-                dddb_tags.add(dddb)
-                conddb_tags.add(conddb)
-
-        assert len(dddb_tags) == 1, dddb_tags
-        assert len(conddb_tags) == 1, conddb_tags
+            while isinstance(prodID, six.integer_types):
+                res = BookkeepingClient().getProductionInformation(prodID)
+                if not res["OK"]:
+                    raise RuntimeError(res)
+                for _, _, _, _, dddb, conddb, _, _, _ in res["Value"]["Steps"]:
+                    dddb_tags.add(dddb)
+                    conddb_tags.add(conddb)
+                # In some cases we have to inspect parent transformations to find tags
+                parent = (
+                    TransformationClient()
+                    .getBookkeepingQuery(prodID)
+                    .get("Value", {})
+                    .get("ProductionID", "")
+                )
+                log.debug("Found parent production ID %s for %s", parent, prodID)
+                prodID = parent
+
+        assert len({d for d in dddb_tags if d is not None}) == 1, dddb_tags
+        assert len({d for d in conddb_tags if d is not None}) == 1, conddb_tags
         assert dddb_tags != {None}, "Simulated database tags should never be None"
         assert conddb_tags != {None}, "Simulated database tags should never be None"
 

tests/conftest.py

@@ -61,6 +61,7 @@ def pytest_collection_modifyitems(config, items):
 def MC_JOB():
     return {
         "wg": "B2CC",
+        "inform": "someone@example.com",
         "application": "DaVinci/v42r7p2",
         "input": {
             "bk_query": "/MC/2017/Beam6500GeV-2017-MagDown-Nu1.6-25ns-Pythia8/Sim09g/Trig0x62661709/Reco17/11144001/LDST",
@@ -74,6 +75,7 @@ def MC_JOB():
 def DATA_JOB():
     return {
         "wg": "Charm",
+        "inform": "someone@example.com",
         "application": "Castelao/v2r1",
         "input": {
             "bk_query": "/LHCb/Collision18/Beam6500GeV-VeloClosed-MagDown/Real Data/Reco18/Stripping34r0p1/90000000/CHARM.MDST",

tests/data/repo_dir/BsToJpsiPhi/info.yaml

@@ -39,7 +39,6 @@ defaults:
         - Bs2JpsiPhi.py
         - Bd2JpsiKstar.py
         - Bu2JpsiKplus.py
-    inform:
 
 {{ year }}_{{ polarity }}_LEPTONIC:
     application: Castelao/v2r1

tests/data/repo_dir_autoconf/BsToJpsiPhi/info.yaml

@@ -40,7 +40,6 @@ defaults:
         - Bs2JpsiPhi.py
         - Bd2JpsiKstar.py
         - Bu2JpsiKplus.py
-    inform:
 
 {{ year }}_{{ polarity }}_LEPTONIC:
     application: Castelao/v2r1

tests/test_parsing.py

@@ -322,7 +322,6 @@ def test_no_options(db_session, repo_dir, MC_JOB):
 @pytest.mark.parametrize(
     "inform",
     [
-        [],
         "username",
         "my-email+something@example.com",
         ["username"],
@@ -368,7 +367,7 @@ def test_no_inform(db_session, repo_dir, MC_JOB):
     with open(join(repo_dir, "prod_name", "info.yaml"), "wt") as fp:
         yaml.safe_dump(prod_info, fp)
     p = Production.create(None, repo_dir, "prod_name")
-    assert p["job_name"].inform == []
+    assert p["job_name"].inform == ["someone@example.com"]
 
 
 def test_simple(db_session, repo_dir, MC_JOB, DATA_JOB):