Commit 00f59fa3 authored by Aidan Richard Wiederhold's avatar Aidan Richard Wiederhold
Browse files

Merge branch 'aiwieder/job_chain_files' into 'master'

More robust passing on of job chain outputs

See merge request !50
parents 179742d8 58f16fde
Pipeline #3880555 passed with stage
in 5 minutes and 58 seconds
......@@ -10,7 +10,9 @@
###############################################################################
import json
import os
import re
import shlex
from glob import glob
from os.path import join
import click
......@@ -250,36 +252,33 @@ def test(production_name, job_name, dependent_input, validate):
# Validating Gaudi options
test_ntuple_path = out_dir / f"00012345_00006789_1.{output_file}"
errors, warnings = [], []
if os.path.isfile(test_ntuple_path):
# TODO: This hasn't been updated to work with lb-prod-run
click.secho("Options parsing is currently unavailable", fg="yellow")
# if ".root" in test_ntuple_path.lower():
# errors, warnings = do_options_parsing(
# # TODO
# env_cmd,
# out_dir,
# join(out_dir, "output.pkl"),
# test_ntuple_path,
# job_name,
# prod_data,
# )
elif any(
str(test_ntuple_path).lower() == join(out_dir, name).lower()
for name in out_dir.iterdir()
):
warnings = [
"The output file has a different capitalisation to what is expected!"
" If this is a stripping job it could be due to a custom_stream"
" name not being written in all capital letters."
]
for file_path in glob(str(out_dir / "*")):
if re.match(
str(test_ntuple_path),
file_path,
re.IGNORECASE,
):
test_ntuple_path = file_path
break
else:
raise click.ClickException(
"ERROR: The expected output file does not exist!\n"
f"Expected: {test_ntuple_path}"
)
errors, warnings = [], []
# TODO: This hasn't been updated to work with lb-prod-run
click.secho("Options parsing is currently unavailable", fg="yellow")
# if ".root" in test_ntuple_path.lower():
# errors, warnings = do_options_parsing(
# # TODO
# env_cmd,
# out_dir,
# join(out_dir, "output.pkl"),
# test_ntuple_path,
# job_name,
# prod_data,
# )
warnings.extend(yaml_warnings)
for warning in warnings:
click.secho(f"WARNING: {warning}", fg="yellow")
......
......@@ -11,6 +11,7 @@
import glob
import os
import pathlib
import re
import subprocess
import tempfile
from concurrent.futures import ThreadPoolExecutor
......@@ -176,26 +177,21 @@ def recursively_create_input(production_name, job_name, filetype, prod_data):
output_dir = result.split("\n")[-2].split(" ")[-1]
# if this is False clearly some other error has occurred so don't bother continuing
if os.path.exists(output_dir):
# Set the dependent input to lower case to handle inconsistencies between
# yaml definition and produced file
dependent_input = (
output_dir + f"/00012345_00006789_1.{filetype.lower()}"
).lower()
# Set up a dictionary to map lower case output file names to their
# actual file name locations
output_files_dict = {
join(output_dir, name).lower(): join(output_dir, name)
for name in os.listdir(output_dir)
}
# Check we have the desired output
if dependent_input not in output_files_dict:
dependent_input = output_dir + f"/00012345_00006789_1.{filetype}"
for file_path in glob.glob(f"{output_dir}/*"):
if re.match(
dependent_input,
file_path,
re.IGNORECASE,
):
dependent_input = file_path
break
else:
raise OSError(
f"{job_name} did not produce the desired output {dependent_input}"
)
return output_files_dict[dependent_input]
return dependent_input
else:
raise OSError(
f"The expected output directory {output_dir} does not exist! "
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment