Merge branch 'aiwieder/job_chain_files' into 'master'

More robust passing on of job chain outputs

See merge request !50

src/LbAPLocal/cli.py

@@ -10,7 +10,9 @@
 ###############################################################################
 import json
 import os
+import re
 import shlex
+from glob import glob
 from os.path import join
 
 import click
@@ -250,36 +252,33 @@ def test(production_name, job_name, dependent_input, validate):
 
     # Validating Gaudi options
     test_ntuple_path = out_dir / f"00012345_00006789_1.{output_file}"
-
-    errors, warnings = [], []
-    if os.path.isfile(test_ntuple_path):
-        # TODO: This hasn't been updated to work with lb-prod-run
-        click.secho("Options parsing is currently unavailable", fg="yellow")
-        # if ".root" in test_ntuple_path.lower():
-        #     errors, warnings = do_options_parsing(
-        #         # TODO
-        #         env_cmd,
-        #         out_dir,
-        #         join(out_dir, "output.pkl"),
-        #         test_ntuple_path,
-        #         job_name,
-        #         prod_data,
-        #     )
-    elif any(
-        str(test_ntuple_path).lower() == join(out_dir, name).lower()
-        for name in out_dir.iterdir()
-    ):
-        warnings = [
-            "The output file has a different capitalisation to what is expected!"
-            " If this is a stripping job it could be due to a custom_stream"
-            " name not being written in all capital letters."
-        ]
+    for file_path in glob(str(out_dir / "*")):
+        if re.match(
+            str(test_ntuple_path),
+            file_path,
+            re.IGNORECASE,
+        ):
+            test_ntuple_path = file_path
+            break
     else:
         raise click.ClickException(
             "ERROR: The expected output file does not exist!\n"
             f"Expected: {test_ntuple_path}"
         )
 
+    errors, warnings = [], []
+    # TODO: This hasn't been updated to work with lb-prod-run
+    click.secho("Options parsing is currently unavailable", fg="yellow")
+    # if ".root" in test_ntuple_path.lower():
+    #     errors, warnings = do_options_parsing(
+    #         # TODO
+    #         env_cmd,
+    #         out_dir,
+    #         join(out_dir, "output.pkl"),
+    #         test_ntuple_path,
+    #         job_name,
+    #         prod_data,
+    #     )
     warnings.extend(yaml_warnings)
     for warning in warnings:
         click.secho(f"WARNING: {warning}", fg="yellow")

src/LbAPLocal/utils.py

@@ -11,6 +11,7 @@
 import glob
 import os
 import pathlib
+import re
 import subprocess
 import tempfile
 from concurrent.futures import ThreadPoolExecutor
@@ -176,26 +177,21 @@ def recursively_create_input(production_name, job_name, filetype, prod_data):
     output_dir = result.split("\n")[-2].split(" ")[-1]
     # if this is False clearly some other error has occurred so don't bother continuing
     if os.path.exists(output_dir):
-        # Set the dependent input to lower case to handle inconsistencies between
-        # yaml definition and produced file
-        dependent_input = (
-            output_dir + f"/00012345_00006789_1.{filetype.lower()}"
-        ).lower()
-
-        # Set up a dictionary to map lower case output file names to their
-        # actual file name locations
-        output_files_dict = {
-            join(output_dir, name).lower(): join(output_dir, name)
-            for name in os.listdir(output_dir)
-        }
-
-        # Check we have the desired output
-        if dependent_input not in output_files_dict:
+        dependent_input = output_dir + f"/00012345_00006789_1.{filetype}"
+        for file_path in glob.glob(f"{output_dir}/*"):
+            if re.match(
+                dependent_input,
+                file_path,
+                re.IGNORECASE,
+            ):
+                dependent_input = file_path
+                break
+        else:
             raise OSError(
                 f"{job_name} did not produce the desired output {dependent_input}"
             )
 
-        return output_files_dict[dependent_input]
+        return dependent_input
     else:
         raise OSError(
             f"The expected output directory {output_dir} does not exist! "