diff --git a/python/MooreTests/generate_metadata.py b/python/MooreTests/generate_metadata.py new file mode 100755 index 0000000000000000000000000000000000000000..2861cf30c4a1e78d8ce5ebaef2905e7adc549ab6 --- /dev/null +++ b/python/MooreTests/generate_metadata.py @@ -0,0 +1,96 @@ +############################################################################### +# (c) Copyright 2000-2023 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +''' + Generates a configuration yaml for the FULL stream output of an Hlt2 BW job. + Intended to be used as input for chained sprucing BW jobs. + ```yaml + input_files: + - 'mdf:/path/on/eos' + dddb_tag: + conddb_tag: + ... all other sample info... + + input_rate: + created_date: + ``` +''' +import argparse +import json +import yaml +import os +from PRConfig.bandwidth_helpers import FileNameHelper +from PRConfig.TestFileDB import test_file_db + +LHCBPR_WWW_FLOC = "mdf:root://eoslhcb.cern.ch//eos/lhcb/storage/lhcbpr/www/UpgradeRateTest/hlt2_bw_testing__production{fsuffix}" +HLT2CONF_WEB = "https://gitlab.cern.ch/lhcb/Moore/-/blob/master/Hlt/Hlt2Conf" + + +def parse_yaml(file_path): + with open(os.path.expandvars(file_path), 'r') as f: + return yaml.safe_load(f) + + +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument( + '-n', + '--events', + default=-1, + type=lambda x: int(round(float(x))), + help='nb of events to process', + required=True) + parser.add_argument( + '-c', + '--config', + type=str, + required=True, + help='Path to yaml config file defining the input.') + args = parser.parse_args() + fname_helper = FileNameHelper(process="hlt2") + config = parse_yaml(args.config) + + opts = [f"input_files: {LHCBPR_WWW_FLOC.format(fsuffix='__full.mdf')}"] + opts.append( + f"input_manifest_file: {LHCBPR_WWW_FLOC.format(fsuffix='.tck.json')}") + opts.append(f"input_raw_format: {config['input_raw_format']}") + opts.append(f"velo_radial_opening: {config['velo_radial_opening']}") + opts.append( + f"filtering_info_links:\n - {args.config.replace('$HLT2CONFROOT',HLT2CONF_WEB)}" + ) + opts.append(f"input_type: 'MDF'") + opts.append(f"Simulation: True") + + conds = test_file_db[config['testfiledb_key']].qualifiers + opts.append(f"dddb_tag: {conds['DDDB']}") + opts.append(f"conddb_tag: {conds['CondDB']}") + + # Access the + ifile = fname_helper.event_no_fname( + stream_config='production', stream='full') + with open(ifile, 'r') as f: + ijson = json.load(f) # {stream: [evt_numbers]} + full_evts = len(ijson['full']) + + hlt2_evts = args.events + input_rate = config['input_rate'] + opts.append(f"input_rate: {input_rate * full_evts / hlt2_evts} # in kHz") + opts.append(f"n_evts: {full_evts}") + + ofile = fname_helper.metadata_path( + stream_config="production", stream="full") + with open(ofile, 'w') as f: + for opt in opts: + f.write(opt + '\n') + + +if __name__ == "__main__": + main() diff --git a/python/MooreTests/run_bandwidth_test_jobs.py b/python/MooreTests/run_bandwidth_test_jobs.py index 407f6fc61217f01d4f0aa166a11e1ef4aff37b3e..04519fcb5f764631cd13e0739e24a7e16b37df89 100644 --- a/python/MooreTests/run_bandwidth_test_jobs.py +++ b/python/MooreTests/run_bandwidth_test_jobs.py @@ -174,11 +174,14 @@ if __name__ == '__main__': raise KeyError( f'{args.config} does not provide "input_manifest_file" but --use-manifest is in use.' ) - if "testfiledb_key" in config.keys(): + + # This logic ordering is important. + # Allows for input_file to be hard-coded but still set_conds from testfiledb. + if "input_files" in config.keys(): + inputs_fns = config["input_files"] + elif "testfiledb_key" in config.keys(): from PRConfig.TestFileDB import test_file_db inputs_fns = test_file_db[config['testfiledb_key']].filenames - elif "input_files" in config.keys(): - inputs_fns = config["input_files"] else: raise KeyError( f'{args.config} does not provide either the "testfiledb_key" or "input_files".' diff --git a/python/PRConfig/bandwidth_helpers.py b/python/PRConfig/bandwidth_helpers.py index 537d4d17114a2fffeef06d13cece6a65fbe85103..51f682cc5706281d0b1e0232120a50e1a879d4a2 100644 --- a/python/PRConfig/bandwidth_helpers.py +++ b/python/PRConfig/bandwidth_helpers.py @@ -65,6 +65,12 @@ class FileNameHelper(object): return os.path.join(self.base_dir, self.output_subdir, fname) if full_path else fname + def metadata_path(self, stream_config, stream): + return os.path.join( + self.base_dir, self.mdf_subdir, + self._join(self._file_pfx(), stream_config, stream, "metadata") + + ".yaml") + def line_descr_path(self, full_path=True): fname = self._join(self.process, "line_descriptives") + ".html" return os.path.join(self.base_dir, self.output_subdir,