From 5c01cc1abf5ce7010d79703207625d3e97df6b49 Mon Sep 17 00:00:00 2001 From: Davide Fazzini <davide.fazzini@cern.ch> Date: Thu, 20 Jan 2022 23:29:28 +0100 Subject: [PATCH 01/11] adding create_options_template command --- DaVinciSys/scripts/davinci | 46 +++++- DaVinciSys/tests/test_davinci_script.py | 64 +++++++++ .../python/DaVinci/utilities_script.py | 135 ++++++++++++++++++ 3 files changed, 240 insertions(+), 5 deletions(-) diff --git a/DaVinciSys/scripts/davinci b/DaVinciSys/scripts/davinci index 799f15f1a..7140cf228 100755 --- a/DaVinciSys/scripts/davinci +++ b/DaVinciSys/scripts/davinci @@ -27,7 +27,9 @@ A DaVinci job using simulated data can be run using the command line: """ import os, sys, click -from DaVinci.utilities_script import dump_call, get_configurable_opts, set_testfiledb +from DaVinci.utilities_script import (dump_call, get_configurable_opts, + set_testfiledb, create_inputdb_template, + create_jobopt_template, get_dummy_config) from DaVinci.ConfigurationUpgrade import run_davinci_app from DaVinci.config import options from DaVinci.optionChecker import log_click @@ -40,6 +42,7 @@ else: @click.group() @click.option( + "-o", "--export", default=None, help= @@ -53,6 +56,7 @@ else: "Explicitly include default values of properties in the final configuration.", ) @click.option( + "-n", "--dry-run", is_flag=True, default=False, @@ -125,6 +129,36 @@ def run_job(configurables, export=None, with_defaults=False, dry_run=None): exit(app.run()) +@main.command( + context_settings=dict( + ignore_unknown_options=False, + allow_extra_args=False + )) +@click.argument("optfile") +@click.option("-f", "--filename", default="", help="Name for the template file.") +@click.option("-k", "--key_db", default="", help="Key to be used in the inputfile DB.") +@click.option("-i", "--interactive", is_flag=True, default=False, + help="Option for creating the template interactively.") +def create_options_template(optfile, filename, key_db, interactive): + """ + Create a template for the options file to be passed to DaVinci when running a job.\n + E.g. ./run davinci create_template job (for a template of the jopoptfile argument).\n + E.g. ./run davinci create_template data (for a template of the inputfiledb argument). + + Note: + Click automatically converts "_" in "-", so this function can be invoked calling create-options-template as shown in the help. + """ + assert optfile in ("job", "data"), "option file value not recognised."\ + "Use 'job' for creating a template of the joboptfile or 'data' for the inputfiledb." + + if optfile == "job": + create_jobopt_template(filename, interactive) + else: + create_inputdb_template(filename, key_db, interactive) + + return get_dummy_config() + + inputfiledb_helper = "TestFileDB-like file containing job input and conditions information (.yaml). Takes the pair of values 'filedb-key', 'filedb-path'." \ "If you want to use the standard TestFileDB set 'filedb-path' = '-'.\n" \ "E.g. for a local DB:\n./run davinci [command] --inputfiledb Upgrade_Bd2KstarMuMu_ldst\n" \ @@ -134,18 +168,18 @@ joboptfile_helper = "Option file containing the job information (.yaml, .py)" simplejob_helper = "Option for running a simple DaVinci job without any specific configuration (.py)." override_data_options_helper = "Allow overriding default data options defined in DaVinci Database." - @main.command( context_settings=dict( ignore_unknown_options=True, allow_extra_args=True, )) @click.option( + "-i", "--inputfiledb", default=("", "TestFileDB"), nargs=2, help=inputfiledb_helper) -@click.option("--joboptfile", default="", help=joboptfile_helper) +@click.option("-j", "--joboptfile", default="", help=joboptfile_helper) @click.option( "--simplejob", is_flag=True, default=False, help=simplejob_helper) @click.option( @@ -163,7 +197,7 @@ def run_mc(ctx, inputfiledb, joboptfile, simplejob, override_data_options): All options passed via the command line which are not recognised by Click are stored into the ctx.args element. Ctx.args is a simple array and each extra option is stored using two values: the first one is the key and the second one is the corresponding value. - Ex.: --evt_max 100 will be stored as: ctx.args[0] = --evt_max, ctx.args[1] = 100 + E.g.: --evt_max 100 will be stored as: ctx.args[0] = --evt_max, ctx.args[1] = 100 Note: Click automatically converts "_" in "-", so this function can be invoked calling run-mc as shown in the help. @@ -171,6 +205,7 @@ def run_mc(ctx, inputfiledb, joboptfile, simplejob, override_data_options): # Run on MC sample options.simulation = True + # Test file DB key request overrides inputfiledb inputfiledb_key, inputfiledb_file = set_testfiledb(inputfiledb) @@ -195,11 +230,12 @@ def run_mc(ctx, inputfiledb, joboptfile, simplejob, override_data_options): allow_extra_args=True, )) @click.option( + "-i", "--inputfiledb", default=("", "TestFileDB"), nargs=2, help=inputfiledb_helper) -@click.option("--joboptfile", default="", help=joboptfile_helper) +@click.option("-j", "--joboptfile", default="", help=joboptfile_helper) @click.option( "--simplejob", is_flag=True, default=False, help=simplejob_helper) @click.option( diff --git a/DaVinciSys/tests/test_davinci_script.py b/DaVinciSys/tests/test_davinci_script.py index 2779cdab5..1c4a60de6 100644 --- a/DaVinciSys/tests/test_davinci_script.py +++ b/DaVinciSys/tests/test_davinci_script.py @@ -65,3 +65,67 @@ def test_override_job_option_bis(): assert any('.process = "Turbo";' in line for line in f.readlines()) os.remove("test_override_job_option_bis.opts") + + +def test_run_option_shortcut(): + """ + Verify that option values passed to davinci command are accepted also + when not using the full name (e.g. --inputfiledb ==> -i). + """ + cmd = 'davinci --dry-run -o test_run_option_shortcut.opts run-mc -i Upgrade_Bd2KstarMuMu -' + result = subprocess.run(cmd, shell=True) + + # Just be maniac - the command should work ;-) + assert result.returncode == 0 + + # Inspect the .opts file and check if the input files option is filled correctly. + with open("test_run_option_shortcut.opts") as f: + assert not any('.input_files = [];' in line for line in f.readlines()) + + os.remove("test_run_option_shortcut.opts") + + +def test_create_joboption_template(): + """ + Verify that the 'create_options_template' command produces a template for the joboptfile option files. + """ + cmd = 'davinci create-options-template job -f test_jobopt_template.yaml' + result = subprocess.run(cmd, shell=True) + + # Just be maniac - the command should work ;-) + assert result.returncode == 0 + + with open("test_jobopt_template.yaml") as f: + assert any('evt_max: -1' in line for line in f.readlines()) + + +def test_create_inputdb_template(): + """ + Verify that the 'create_options_template' command produces a template for the inputfiledb option files. + """ + cmd = 'davinci create-options-template data -f test_inputdb_template.yaml -k test_key' + result = subprocess.run(cmd, shell=True) + + # Just be maniac - the command should work ;-) + assert result.returncode == 0 + + with open("test_inputdb_template.yaml") as f: + assert any("data_type: Upgrade" in line for line in f.readlines()) + + +def test_run_using_templates(): + """ + Verify that DaVinci runs correctly when using the templates produced by the 'create_options_template' command. + """ + cmd = 'davinci -o test_run_using_templates.opts run-mc -i test_key test_inputdb_template.yaml -j test_jobopt_template.yaml --enable_unpack False' + result = subprocess.run(cmd, shell=True) + + # Just be maniac - the command should work ;-) + assert result.returncode == 0 + + with open("test_run_using_templates.opts") as f: + assert any('.data_type = "Upgrade";' in line for line in f.readlines()) + + os.remove('test_jobopt_template.yaml') + os.remove('test_inputdb_template.yaml') + os.remove('test_run_using_templates.opts') diff --git a/Phys/DaVinci/python/DaVinci/utilities_script.py b/Phys/DaVinci/python/DaVinci/utilities_script.py index 612581ffd..9961181c3 100644 --- a/Phys/DaVinci/python/DaVinci/utilities_script.py +++ b/Phys/DaVinci/python/DaVinci/utilities_script.py @@ -104,3 +104,138 @@ def set_testfiledb(inputfiledb): 1] if not inputfiledb[1] == "-" else "TestFileDB" return inputfiledb_key, inputfiledb_file + + +def get_dummy_config(): + """ + Simple function returning a dummy instance of the configurables to be passed to DaVinci. + """ + from PyConf.application import ComponentConfig + from Configurables import ApplicationMgr + + config = ComponentConfig() + config.add(ApplicationMgr(AppName="DaVinci")) + + return config + + +def create_jobopt_template(file_name, interactive): + """ + Function for creating simple template for the joboptfile dict. + + Args: + - file_name (str): name for the template file. Default = 'joboptfile_template.yaml'. + - interactive (bool): boolean for creating the template interactively. + + Output: + - write a template for the job option file with the name defined by the user. + """ + import yaml, click + # Get the DaVinci dict containing the default values. + from DaVinci.options_default import __optsDict__ as opts_dict + + dict_template = {} + if file_name == "": + file_name = "joboptfile_template.yaml" + + # These are the data options that are set via the inputfiledb, so we remove them from the joboptfile. + list_data_qualifiers = ["data_type", "input_type", "simulation", "conddb_tag", "dddb_tag", "input_files"] + for name, config in opts_dict.items(): + if not name in list_data_qualifiers: + if not interactive: + dict_template.update({name : config["value"]}) + else: + value = click.prompt(f"Set the value for the option {name}:", + default=config["value"], + type=type(config["value"])) + # Adding to the dict only if the value is not the default one. + if not value == config["value"]: + dict_template.update({name : value}) + + with open(file_name, 'w') as jobfile: + yaml.safe_dump(dict_template, jobfile, default_flow_style=False) + + +def create_inputdb_template(file_name, db_key, interactive): + """ + Function for creating simple template for the joboptfile dict. + + Args: + - file_name (str): name for the template file. Default = 'inputdb_template.yaml'. + - db_key (str): database key for identifying the input samples. + - interactive (bool): boolean for creating the template interactively. + + Output: + - write a template for the TestFileDB-like input file with the name defined by the user. + """ + import yaml, click + from PRConfig import TestFileDB + + dict_template = {} + config = {} + if file_name == "": + file_name = "inputdb_template.yaml" + if db_key == "": + db_key = "key_template" + + filedb_ref = TestFileDB.test_file_db['Upgrade_Bd2KstarMuMu'] + list_qualifiers = ["data_type", "input_type", "simulation", "conddb_tag", "dddb_tag"] + list_metadata = ["Author", "Date"] + + # Map for linking qualifiers and metadata names between TestFileDB and DaVinci + _optNameMap = { + "conddb_tag": "CondDB", + "dddb_tag": "DDDB", + "data_type": "DataType", + "input_type" : "Format", + "simulation" : "Simulation", + "Author": "Author", + "Date": "Date" + } + + list_filenames = [] + dict_qualifiers = {} + dict_metadata = {} + + if not interactive: + list_filenames = filedb_ref.filenames + + for option in list_qualifiers: + dict_qualifiers.update({option: filedb_ref.qualifiers[_optNameMap[option]]}) + for option in list_metadata: + dict_metadata.update({option: filedb_ref.qualifiers[_optNameMap[option]]}) + dict_metadata.update({"Comment": filedb_ref.comment}) + else: + nFiles = click.prompt(f"How many input files do you want to use? (set to 0 if you wnat to read them from a .txt or .py file).", + default=1, type=int) + if not nFiles: + print("TO BE IMPLEMENTED!") + else: + for n in range(0, nFiles): + value = click.prompt(f"Insert the path to the file {n+1}:", + default="", type=str) + list_filenames.append(value) + + for option in list_qualifiers: + value_ref = filedb_ref.qualifiers[_optNameMap[option]] + value = click.prompt(f"Set the value for the qualifier {option}:", + type=type(value_ref)) + dict_qualifiers.update({option : value}) + + for option in list_metadata: + value = click.prompt(f"Set the value for the metadata {option}:", + default="", type=str) + dict_metadata.update({option : value}) + + value = click.prompt(f"Write any other comment related to the input files:", + default="", type=str) + dict_metadata.update({"Comment" : value}) + + config.update({"filenames": list_filenames}) + config.update({"qualifiers": dict_qualifiers}) + config.update({"metadata": dict_metadata}) + + dict_template.update({db_key : config}) + + with open(file_name, 'w') as jobfile: + yaml.safe_dump(dict_template, jobfile, default_flow_style=False) -- GitLab From 1897519811555a8a3de6b51f5cc24b3f87c47f44 Mon Sep 17 00:00:00 2001 From: Gitlab CI <noreply@cern.ch> Date: Thu, 20 Jan 2022 22:58:09 +0000 Subject: [PATCH 02/11] Fixed formatting patch generated by https://gitlab.cern.ch/lhcb/DaVinci/-/jobs/18937066 --- DaVinciSys/scripts/davinci | 29 ++++--- DaVinciSys/tests/test_davinci_script.py | 2 +- .../python/DaVinci/utilities_script.py | 83 ++++++++++++------- 3 files changed, 70 insertions(+), 44 deletions(-) diff --git a/DaVinciSys/scripts/davinci b/DaVinciSys/scripts/davinci index 7140cf228..bdd069d72 100755 --- a/DaVinciSys/scripts/davinci +++ b/DaVinciSys/scripts/davinci @@ -27,7 +27,7 @@ A DaVinci job using simulated data can be run using the command line: """ import os, sys, click -from DaVinci.utilities_script import (dump_call, get_configurable_opts, +from DaVinci.utilities_script import (dump_call, get_configurable_opts, set_testfiledb, create_inputdb_template, create_jobopt_template, get_dummy_config) from DaVinci.ConfigurationUpgrade import run_davinci_app @@ -42,7 +42,7 @@ else: @click.group() @click.option( - "-o", + "-o", "--export", default=None, help= @@ -56,7 +56,7 @@ else: "Explicitly include default values of properties in the final configuration.", ) @click.option( - "-n", + "-n", "--dry-run", is_flag=True, default=False, @@ -131,14 +131,18 @@ def run_job(configurables, export=None, with_defaults=False, dry_run=None): @main.command( context_settings=dict( - ignore_unknown_options=False, - allow_extra_args=False - )) + ignore_unknown_options=False, allow_extra_args=False)) @click.argument("optfile") -@click.option("-f", "--filename", default="", help="Name for the template file.") -@click.option("-k", "--key_db", default="", help="Key to be used in the inputfile DB.") -@click.option("-i", "--interactive", is_flag=True, default=False, - help="Option for creating the template interactively.") +@click.option( + "-f", "--filename", default="", help="Name for the template file.") +@click.option( + "-k", "--key_db", default="", help="Key to be used in the inputfile DB.") +@click.option( + "-i", + "--interactive", + is_flag=True, + default=False, + help="Option for creating the template interactively.") def create_options_template(optfile, filename, key_db, interactive): """ Create a template for the options file to be passed to DaVinci when running a job.\n @@ -168,13 +172,14 @@ joboptfile_helper = "Option file containing the job information (.yaml, .py)" simplejob_helper = "Option for running a simple DaVinci job without any specific configuration (.py)." override_data_options_helper = "Allow overriding default data options defined in DaVinci Database." + @main.command( context_settings=dict( ignore_unknown_options=True, allow_extra_args=True, )) @click.option( - "-i", + "-i", "--inputfiledb", default=("", "TestFileDB"), nargs=2, @@ -230,7 +235,7 @@ def run_mc(ctx, inputfiledb, joboptfile, simplejob, override_data_options): allow_extra_args=True, )) @click.option( - "-i", + "-i", "--inputfiledb", default=("", "TestFileDB"), nargs=2, diff --git a/DaVinciSys/tests/test_davinci_script.py b/DaVinciSys/tests/test_davinci_script.py index 1c4a60de6..29594625e 100644 --- a/DaVinciSys/tests/test_davinci_script.py +++ b/DaVinciSys/tests/test_davinci_script.py @@ -119,7 +119,7 @@ def test_run_using_templates(): """ cmd = 'davinci -o test_run_using_templates.opts run-mc -i test_key test_inputdb_template.yaml -j test_jobopt_template.yaml --enable_unpack False' result = subprocess.run(cmd, shell=True) - + # Just be maniac - the command should work ;-) assert result.returncode == 0 diff --git a/Phys/DaVinci/python/DaVinci/utilities_script.py b/Phys/DaVinci/python/DaVinci/utilities_script.py index 9961181c3..fa67ff66a 100644 --- a/Phys/DaVinci/python/DaVinci/utilities_script.py +++ b/Phys/DaVinci/python/DaVinci/utilities_script.py @@ -139,22 +139,26 @@ def create_jobopt_template(file_name, interactive): file_name = "joboptfile_template.yaml" # These are the data options that are set via the inputfiledb, so we remove them from the joboptfile. - list_data_qualifiers = ["data_type", "input_type", "simulation", "conddb_tag", "dddb_tag", "input_files"] + list_data_qualifiers = [ + "data_type", "input_type", "simulation", "conddb_tag", "dddb_tag", + "input_files" + ] for name, config in opts_dict.items(): if not name in list_data_qualifiers: if not interactive: - dict_template.update({name : config["value"]}) + dict_template.update({name: config["value"]}) else: - value = click.prompt(f"Set the value for the option {name}:", - default=config["value"], - type=type(config["value"])) + value = click.prompt( + f"Set the value for the option {name}:", + default=config["value"], + type=type(config["value"])) # Adding to the dict only if the value is not the default one. if not value == config["value"]: - dict_template.update({name : value}) + dict_template.update({name: value}) with open(file_name, 'w') as jobfile: yaml.safe_dump(dict_template, jobfile, default_flow_style=False) - + def create_inputdb_template(file_name, db_key, interactive): """ @@ -174,12 +178,14 @@ def create_inputdb_template(file_name, db_key, interactive): dict_template = {} config = {} if file_name == "": - file_name = "inputdb_template.yaml" + file_name = "inputdb_template.yaml" if db_key == "": db_key = "key_template" filedb_ref = TestFileDB.test_file_db['Upgrade_Bd2KstarMuMu'] - list_qualifiers = ["data_type", "input_type", "simulation", "conddb_tag", "dddb_tag"] + list_qualifiers = [ + "data_type", "input_type", "simulation", "conddb_tag", "dddb_tag" + ] list_metadata = ["Author", "Date"] # Map for linking qualifiers and metadata names between TestFileDB and DaVinci @@ -187,55 +193,70 @@ def create_inputdb_template(file_name, db_key, interactive): "conddb_tag": "CondDB", "dddb_tag": "DDDB", "data_type": "DataType", - "input_type" : "Format", - "simulation" : "Simulation", + "input_type": "Format", + "simulation": "Simulation", "Author": "Author", "Date": "Date" } - + list_filenames = [] dict_qualifiers = {} dict_metadata = {} - + if not interactive: list_filenames = filedb_ref.filenames for option in list_qualifiers: - dict_qualifiers.update({option: filedb_ref.qualifiers[_optNameMap[option]]}) + dict_qualifiers.update({ + option: + filedb_ref.qualifiers[_optNameMap[option]] + }) for option in list_metadata: - dict_metadata.update({option: filedb_ref.qualifiers[_optNameMap[option]]}) + dict_metadata.update({ + option: + filedb_ref.qualifiers[_optNameMap[option]] + }) dict_metadata.update({"Comment": filedb_ref.comment}) else: - nFiles = click.prompt(f"How many input files do you want to use? (set to 0 if you wnat to read them from a .txt or .py file).", - default=1, type=int) + nFiles = click.prompt( + f"How many input files do you want to use? (set to 0 if you wnat to read them from a .txt or .py file).", + default=1, + type=int) if not nFiles: print("TO BE IMPLEMENTED!") else: for n in range(0, nFiles): - value = click.prompt(f"Insert the path to the file {n+1}:", - default="", type=str) + value = click.prompt( + f"Insert the path to the file {n+1}:", + default="", + type=str) list_filenames.append(value) - + for option in list_qualifiers: value_ref = filedb_ref.qualifiers[_optNameMap[option]] - value = click.prompt(f"Set the value for the qualifier {option}:", - type=type(value_ref)) - dict_qualifiers.update({option : value}) + value = click.prompt( + f"Set the value for the qualifier {option}:", + type=type(value_ref)) + dict_qualifiers.update({option: value}) for option in list_metadata: - value = click.prompt(f"Set the value for the metadata {option}:", - default="", type=str) - dict_metadata.update({option : value}) - - value = click.prompt(f"Write any other comment related to the input files:", - default="", type=str) - dict_metadata.update({"Comment" : value}) + value = click.prompt( + f"Set the value for the metadata {option}:", + default="", + type=str) + dict_metadata.update({option: value}) + + value = click.prompt( + f"Write any other comment related to the input files:", + default="", + type=str) + dict_metadata.update({"Comment": value}) config.update({"filenames": list_filenames}) config.update({"qualifiers": dict_qualifiers}) config.update({"metadata": dict_metadata}) - dict_template.update({db_key : config}) + dict_template.update({db_key: config}) with open(file_name, 'w') as jobfile: yaml.safe_dump(dict_template, jobfile, default_flow_style=False) -- GitLab From 3a6c0a2752c77b90a03a5094b74436033b25197c Mon Sep 17 00:00:00 2001 From: Davide Fazzini <davide.fazzini@cern.ch> Date: Fri, 21 Jan 2022 00:06:56 +0100 Subject: [PATCH 03/11] Fixed linting --- Phys/DaVinci/python/DaVinci/utilities_script.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Phys/DaVinci/python/DaVinci/utilities_script.py b/Phys/DaVinci/python/DaVinci/utilities_script.py index fa67ff66a..aced8001e 100644 --- a/Phys/DaVinci/python/DaVinci/utilities_script.py +++ b/Phys/DaVinci/python/DaVinci/utilities_script.py @@ -144,7 +144,7 @@ def create_jobopt_template(file_name, interactive): "input_files" ] for name, config in opts_dict.items(): - if not name in list_data_qualifiers: + if name not in list_data_qualifiers: if not interactive: dict_template.update({name: config["value"]}) else: @@ -219,7 +219,7 @@ def create_inputdb_template(file_name, db_key, interactive): dict_metadata.update({"Comment": filedb_ref.comment}) else: nFiles = click.prompt( - f"How many input files do you want to use? (set to 0 if you wnat to read them from a .txt or .py file).", + "How many input files do you want to use? (set to 0 if you want to read them from a .txt or .py file).", default=1, type=int) if not nFiles: @@ -227,7 +227,7 @@ def create_inputdb_template(file_name, db_key, interactive): else: for n in range(0, nFiles): value = click.prompt( - f"Insert the path to the file {n+1}:", + f"Insert the path to the {n+1} file:", default="", type=str) list_filenames.append(value) @@ -247,7 +247,7 @@ def create_inputdb_template(file_name, db_key, interactive): dict_metadata.update({option: value}) value = click.prompt( - f"Write any other comment related to the input files:", + "Write any other comment related to the input files:", default="", type=str) dict_metadata.update({"Comment": value}) -- GitLab From f267c4e43c199af6c1f85908bb2477185771048e Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <eduardo.rodrigues@cern.ch> Date: Fri, 21 Jan 2022 14:09:28 +0100 Subject: [PATCH 04/11] Apply 1 suggestion(s) to 1 file(s) --- DaVinciSys/scripts/davinci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DaVinciSys/scripts/davinci b/DaVinciSys/scripts/davinci index bdd069d72..4cbd1b93d 100755 --- a/DaVinciSys/scripts/davinci +++ b/DaVinciSys/scripts/davinci @@ -136,7 +136,7 @@ def run_job(configurables, export=None, with_defaults=False, dry_run=None): @click.option( "-f", "--filename", default="", help="Name for the template file.") @click.option( - "-k", "--key_db", default="", help="Key to be used in the inputfile DB.") + "-k", "--key_db", default="", help="Key to be used in the inputfile DB template.") @click.option( "-i", "--interactive", -- GitLab From 6b585e937c40de79c5d7b018a74348c6719419c7 Mon Sep 17 00:00:00 2001 From: Davide Fazzini <davide.fazzini@cern.ch> Date: Sat, 22 Jan 2022 20:58:18 +0100 Subject: [PATCH 05/11] applying suggestions --- DaVinciSys/scripts/davinci | 43 +++-- DaVinciSys/tests/test_davinci_script.py | 45 ++--- .../python/DaVinci/utilities_script.py | 169 ++++++------------ 3 files changed, 98 insertions(+), 159 deletions(-) diff --git a/DaVinciSys/scripts/davinci b/DaVinciSys/scripts/davinci index 4cbd1b93d..373151219 100755 --- a/DaVinciSys/scripts/davinci +++ b/DaVinciSys/scripts/davinci @@ -132,33 +132,56 @@ def run_job(configurables, export=None, with_defaults=False, dry_run=None): @main.command( context_settings=dict( ignore_unknown_options=False, allow_extra_args=False)) -@click.argument("optfile") +@click.argument("option_type") @click.option( - "-f", "--filename", default="", help="Name for the template file.") -@click.option( - "-k", "--key_db", default="", help="Key to be used in the inputfile DB template.") + "-f", "--filename", default="options_template.yaml", help="Name for the template file.") @click.option( "-i", "--interactive", is_flag=True, default=False, help="Option for creating the template interactively.") -def create_options_template(optfile, filename, key_db, interactive): +def create_options_template(option_type, filename, interactive): """ Create a template for the options file to be passed to DaVinci when running a job.\n - E.g. ./run davinci create_template job (for a template of the jopoptfile argument).\n - E.g. ./run davinci create_template data (for a template of the inputfiledb argument). + E.g. ./run davinci create-options-template job (for a template of the joboptfile argument).\n + E.g. ./run davinci create-options-template data (for a template of the inputfiledb argument). Note: Click automatically converts "_" in "-", so this function can be invoked calling create-options-template as shown in the help. """ - assert optfile in ("job", "data"), "option file value not recognised."\ + assert option_type in ("job", "data"), "option type value not recognised."\ "Use 'job' for creating a template of the joboptfile or 'data' for the inputfiledb." - if optfile == "job": + if option_type == "job": create_jobopt_template(filename, interactive) else: - create_inputdb_template(filename, key_db, interactive) + create_inputdb_template(filename) + + return get_dummy_config() + + +@main.command( + context_settings=dict( + ignore_unknown_options=False, allow_extra_args=False)) +@click.option( + "-f", "--filenames", default=("inputdb_template.yaml", "jobopt_template.yaml"), + nargs=2, help="Name for the template files: inputfiledb and joboptfile.") +def create_options_templates(filenames): + """ + Create a template for the two options files to be passed to DaVinci when running a job:\n + - the inputfiledb containing all the information related to the input data;\n + - the joboptfile containing all the information related to the job to be run.\n + + E.g. ./run davinci create-options-templates -f inputdb_template.yaml jobopt_template.yaml\n + + Note: + Click automatically converts "_" in "-", so this function can be invoked calling + create-options-template as shown in the help. + """ + + create_inputdb_template(filenames[0]) + create_jobopt_template(filenames[1]) return get_dummy_config() diff --git a/DaVinciSys/tests/test_davinci_script.py b/DaVinciSys/tests/test_davinci_script.py index 29594625e..d2bbb48fb 100644 --- a/DaVinciSys/tests/test_davinci_script.py +++ b/DaVinciSys/tests/test_davinci_script.py @@ -85,47 +85,22 @@ def test_run_option_shortcut(): os.remove("test_run_option_shortcut.opts") -def test_create_joboption_template(): +def test_create_options_templates(): """ - Verify that the 'create_options_template' command produces a template for the joboptfile option files. + Verify that the 'create_options_templates' command produces a template for the inputfiledb and joboptfileoption files. """ - cmd = 'davinci create-options-template job -f test_jobopt_template.yaml' + cmd = 'davinci create-options-templates -f test_inputdb_template.yaml test_jobopt_template.yaml' result = subprocess.run(cmd, shell=True) # Just be maniac - the command should work ;-) assert result.returncode == 0 - with open("test_jobopt_template.yaml") as f: - assert any('evt_max: -1' in line for line in f.readlines()) + import yaml + with open(os.path.expandvars("test_inputdb_template.yaml")) as inputdb_file: + assert yaml.safe_load(inputdb_file) + + with open(os.path.expandvars("test_inputdb_template.yaml")) as jobopt_file: + assert yaml.safe_load(jobopt_file) - -def test_create_inputdb_template(): - """ - Verify that the 'create_options_template' command produces a template for the inputfiledb option files. - """ - cmd = 'davinci create-options-template data -f test_inputdb_template.yaml -k test_key' - result = subprocess.run(cmd, shell=True) - - # Just be maniac - the command should work ;-) - assert result.returncode == 0 - - with open("test_inputdb_template.yaml") as f: - assert any("data_type: Upgrade" in line for line in f.readlines()) - - -def test_run_using_templates(): - """ - Verify that DaVinci runs correctly when using the templates produced by the 'create_options_template' command. - """ - cmd = 'davinci -o test_run_using_templates.opts run-mc -i test_key test_inputdb_template.yaml -j test_jobopt_template.yaml --enable_unpack False' - result = subprocess.run(cmd, shell=True) - - # Just be maniac - the command should work ;-) - assert result.returncode == 0 - - with open("test_run_using_templates.opts") as f: - assert any('.data_type = "Upgrade";' in line for line in f.readlines()) - - os.remove('test_jobopt_template.yaml') os.remove('test_inputdb_template.yaml') - os.remove('test_run_using_templates.opts') + os.remove('test_jobopt_template.yaml') diff --git a/Phys/DaVinci/python/DaVinci/utilities_script.py b/Phys/DaVinci/python/DaVinci/utilities_script.py index aced8001e..0e3e9299e 100644 --- a/Phys/DaVinci/python/DaVinci/utilities_script.py +++ b/Phys/DaVinci/python/DaVinci/utilities_script.py @@ -119,7 +119,7 @@ def get_dummy_config(): return config -def create_jobopt_template(file_name, interactive): +def create_jobopt_template(file_name = "jobopt_template.yaml", interactive = False): """ Function for creating simple template for the joboptfile dict. @@ -130,133 +130,74 @@ def create_jobopt_template(file_name, interactive): Output: - write a template for the job option file with the name defined by the user. """ - import yaml, click - # Get the DaVinci dict containing the default values. - from DaVinci.options_default import __optsDict__ as opts_dict - - dict_template = {} - if file_name == "": - file_name = "joboptfile_template.yaml" - - # These are the data options that are set via the inputfiledb, so we remove them from the joboptfile. - list_data_qualifiers = [ - "data_type", "input_type", "simulation", "conddb_tag", "dddb_tag", - "input_files" - ] - for name, config in opts_dict.items(): - if name not in list_data_qualifiers: - if not interactive: - dict_template.update({name: config["value"]}) - else: + if not interactive: + template_str = "# Template job option YAML file.\n"\ + "# Best guesses are provided below for various options. Please adapt as per your needs.\n\n"\ + "annsvc_config: '<path_to_annsvc_json>'\n"\ + "evt_max: -1\n"\ + "ntuple_file: 'ntuple_file.root'\n"\ + "histo_file: 'histos_file.root'\n"\ + "enable_unpack: True\n"\ + "process: 'Spruce'\n" + + with open(file_name, 'w') as jobopt_file_file: + jobopt_file_file.write(template_str) + else: + import yaml, click + # Get the DaVinci dict containing the default values. + from DaVinci.options_default import __optsDict__ as opts_dict + + dict_template = {} + # These are the data options that are set via the inputfiledb, so we remove them from the joboptfile. + list_data_qualifiers = [ + "data_type", "input_type", "simulation", "conddb_tag", + "dddb_tag", "input_files" + ] + + for name, config in opts_dict.items(): + if name not in list_data_qualifiers: value = click.prompt( - f"Set the value for the option {name}:", + f"Set the value for the option {name}:\n[{config['text']}]\n", default=config["value"], type=type(config["value"])) # Adding to the dict only if the value is not the default one. if not value == config["value"]: dict_template.update({name: value}) - - with open(file_name, 'w') as jobfile: - yaml.safe_dump(dict_template, jobfile, default_flow_style=False) + + if len(dict_template): + with open(file_name, 'w') as jobopt_file: + yaml.safe_dump(dict_template, jobopt_file, default_flow_style=False) + else: + click.echo("All option values match the default ones, no job option file is needed!") -def create_inputdb_template(file_name, db_key, interactive): +def create_inputdb_template(file_name = "inputdb_template.yaml"): """ - Function for creating simple template for the joboptfile dict. + Function for creating simple template for the TestFileDB-like file. Args: - file_name (str): name for the template file. Default = 'inputdb_template.yaml'. - - db_key (str): database key for identifying the input samples. - - interactive (bool): boolean for creating the template interactively. Output: - write a template for the TestFileDB-like input file with the name defined by the user. """ - import yaml, click - from PRConfig import TestFileDB - - dict_template = {} - config = {} - if file_name == "": - file_name = "inputdb_template.yaml" - if db_key == "": - db_key = "key_template" - - filedb_ref = TestFileDB.test_file_db['Upgrade_Bd2KstarMuMu'] - list_qualifiers = [ - "data_type", "input_type", "simulation", "conddb_tag", "dddb_tag" - ] - list_metadata = ["Author", "Date"] - - # Map for linking qualifiers and metadata names between TestFileDB and DaVinci - _optNameMap = { - "conddb_tag": "CondDB", - "dddb_tag": "DDDB", - "data_type": "DataType", - "input_type": "Format", - "simulation": "Simulation", - "Author": "Author", - "Date": "Date" - } - - list_filenames = [] - dict_qualifiers = {} - dict_metadata = {} - if not interactive: - list_filenames = filedb_ref.filenames - - for option in list_qualifiers: - dict_qualifiers.update({ - option: - filedb_ref.qualifiers[_optNameMap[option]] - }) - for option in list_metadata: - dict_metadata.update({ - option: - filedb_ref.qualifiers[_optNameMap[option]] - }) - dict_metadata.update({"Comment": filedb_ref.comment}) - else: - nFiles = click.prompt( - "How many input files do you want to use? (set to 0 if you want to read them from a .txt or .py file).", - default=1, - type=int) - if not nFiles: - print("TO BE IMPLEMENTED!") - else: - for n in range(0, nFiles): - value = click.prompt( - f"Insert the path to the {n+1} file:", - default="", - type=str) - list_filenames.append(value) - - for option in list_qualifiers: - value_ref = filedb_ref.qualifiers[_optNameMap[option]] - value = click.prompt( - f"Set the value for the qualifier {option}:", - type=type(value_ref)) - dict_qualifiers.update({option: value}) - - for option in list_metadata: - value = click.prompt( - f"Set the value for the metadata {option}:", - default="", - type=str) - dict_metadata.update({option: value}) - - value = click.prompt( - "Write any other comment related to the input files:", - default="", - type=str) - dict_metadata.update({"Comment": value}) - - config.update({"filenames": list_filenames}) - config.update({"qualifiers": dict_qualifiers}) - config.update({"metadata": dict_metadata}) - - dict_template.update({db_key: config}) - - with open(file_name, 'w') as jobfile: - yaml.safe_dump(dict_template, jobfile, default_flow_style=False) + template_str = "# Template input data YAML file.\n"\ + "# Best guesses are provided below for various qualifiers. Please adapt as per your needs.\n\n"\ + "MyInputData:\n"\ + " filenames:\n"\ + " - '<a_file_path>'\n"\ + " - '<another_file_path>'\n"\ + " qualifiers:\n"\ + " data_type: Upgrade\n"\ + " input_type: DST\n"\ + " simulation: true\n"\ + " conddb_tag: <e.g. sim-20171127-vc-md100>\n"\ + " dddb_tag: <e.g. dddb-20171126>\n"\ + " metadata:\n"\ + " Author: '<me :-)>'\n"\ + " Date: '<date_of_creation>'\n"\ + " Comment: '<E.g. how the files were created or for what purpose>'" + + with open(file_name, 'w') as inputdb_file: + inputdb_file.write(template_str) -- GitLab From 98dcc5c4bbd231ce3a4b242e921a57648a6124ba Mon Sep 17 00:00:00 2001 From: Gitlab CI <noreply@cern.ch> Date: Sat, 22 Jan 2022 19:59:24 +0000 Subject: [PATCH 06/11] Fixed formatting patch generated by https://gitlab.cern.ch/lhcb/DaVinci/-/jobs/18975338 --- DaVinciSys/scripts/davinci | 14 ++++++++---- DaVinciSys/tests/test_davinci_script.py | 5 +++-- .../python/DaVinci/utilities_script.py | 22 +++++++++++-------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/DaVinciSys/scripts/davinci b/DaVinciSys/scripts/davinci index 373151219..313287e56 100755 --- a/DaVinciSys/scripts/davinci +++ b/DaVinciSys/scripts/davinci @@ -134,7 +134,10 @@ def run_job(configurables, export=None, with_defaults=False, dry_run=None): ignore_unknown_options=False, allow_extra_args=False)) @click.argument("option_type") @click.option( - "-f", "--filename", default="options_template.yaml", help="Name for the template file.") + "-f", + "--filename", + default="options_template.yaml", + help="Name for the template file.") @click.option( "-i", "--interactive", @@ -165,8 +168,11 @@ def create_options_template(option_type, filename, interactive): context_settings=dict( ignore_unknown_options=False, allow_extra_args=False)) @click.option( - "-f", "--filenames", default=("inputdb_template.yaml", "jobopt_template.yaml"), - nargs=2, help="Name for the template files: inputfiledb and joboptfile.") + "-f", + "--filenames", + default=("inputdb_template.yaml", "jobopt_template.yaml"), + nargs=2, + help="Name for the template files: inputfiledb and joboptfile.") def create_options_templates(filenames): """ Create a template for the two options files to be passed to DaVinci when running a job:\n @@ -179,7 +185,7 @@ def create_options_templates(filenames): Click automatically converts "_" in "-", so this function can be invoked calling create-options-template as shown in the help. """ - + create_inputdb_template(filenames[0]) create_jobopt_template(filenames[1]) diff --git a/DaVinciSys/tests/test_davinci_script.py b/DaVinciSys/tests/test_davinci_script.py index d2bbb48fb..c21c84436 100644 --- a/DaVinciSys/tests/test_davinci_script.py +++ b/DaVinciSys/tests/test_davinci_script.py @@ -96,9 +96,10 @@ def test_create_options_templates(): assert result.returncode == 0 import yaml - with open(os.path.expandvars("test_inputdb_template.yaml")) as inputdb_file: + with open( + os.path.expandvars("test_inputdb_template.yaml")) as inputdb_file: assert yaml.safe_load(inputdb_file) - + with open(os.path.expandvars("test_inputdb_template.yaml")) as jobopt_file: assert yaml.safe_load(jobopt_file) diff --git a/Phys/DaVinci/python/DaVinci/utilities_script.py b/Phys/DaVinci/python/DaVinci/utilities_script.py index 0e3e9299e..58575acbf 100644 --- a/Phys/DaVinci/python/DaVinci/utilities_script.py +++ b/Phys/DaVinci/python/DaVinci/utilities_script.py @@ -119,7 +119,8 @@ def get_dummy_config(): return config -def create_jobopt_template(file_name = "jobopt_template.yaml", interactive = False): +def create_jobopt_template(file_name="jobopt_template.yaml", + interactive=False): """ Function for creating simple template for the joboptfile dict. @@ -147,11 +148,11 @@ def create_jobopt_template(file_name = "jobopt_template.yaml", interactive = Fal # Get the DaVinci dict containing the default values. from DaVinci.options_default import __optsDict__ as opts_dict - dict_template = {} + dict_template = {} # These are the data options that are set via the inputfiledb, so we remove them from the joboptfile. list_data_qualifiers = [ - "data_type", "input_type", "simulation", "conddb_tag", - "dddb_tag", "input_files" + "data_type", "input_type", "simulation", "conddb_tag", "dddb_tag", + "input_files" ] for name, config in opts_dict.items(): @@ -163,15 +164,18 @@ def create_jobopt_template(file_name = "jobopt_template.yaml", interactive = Fal # Adding to the dict only if the value is not the default one. if not value == config["value"]: dict_template.update({name: value}) - + if len(dict_template): with open(file_name, 'w') as jobopt_file: - yaml.safe_dump(dict_template, jobopt_file, default_flow_style=False) + yaml.safe_dump( + dict_template, jobopt_file, default_flow_style=False) else: - click.echo("All option values match the default ones, no job option file is needed!") + click.echo( + "All option values match the default ones, no job option file is needed!" + ) -def create_inputdb_template(file_name = "inputdb_template.yaml"): +def create_inputdb_template(file_name="inputdb_template.yaml"): """ Function for creating simple template for the TestFileDB-like file. @@ -198,6 +202,6 @@ def create_inputdb_template(file_name = "inputdb_template.yaml"): " Author: '<me :-)>'\n"\ " Date: '<date_of_creation>'\n"\ " Comment: '<E.g. how the files were created or for what purpose>'" - + with open(file_name, 'w') as inputdb_file: inputdb_file.write(template_str) -- GitLab From e2d026f54d8c5d30484d0dc9b73eded894bf0ec0 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <eduardo.rodrigues@cern.ch> Date: Mon, 24 Jan 2022 11:09:50 +0100 Subject: [PATCH 07/11] Apply 3 suggestion(s) to 3 file(s) --- DaVinciSys/scripts/davinci | 2 +- DaVinciSys/tests/test_davinci_script.py | 2 +- Phys/DaVinci/python/DaVinci/utilities_script.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DaVinciSys/scripts/davinci b/DaVinciSys/scripts/davinci index 313287e56..3289b7772 100755 --- a/DaVinciSys/scripts/davinci +++ b/DaVinciSys/scripts/davinci @@ -1,6 +1,6 @@ #!/usr/bin/env python ############################################################################### -# (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration # +# (c) Copyright 2021-2022 CERN for the benefit of the LHCb Collaboration # # # # This software is distributed under the terms of the GNU General Public # # Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # diff --git a/DaVinciSys/tests/test_davinci_script.py b/DaVinciSys/tests/test_davinci_script.py index c21c84436..9b244ca29 100644 --- a/DaVinciSys/tests/test_davinci_script.py +++ b/DaVinciSys/tests/test_davinci_script.py @@ -1,5 +1,5 @@ ############################################################################## -# (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration # +# (c) Copyright 2021-2022 CERN for the benefit of the LHCb Collaboration # # # # This software is distributed under the terms of the GNU General Public # # Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # diff --git a/Phys/DaVinci/python/DaVinci/utilities_script.py b/Phys/DaVinci/python/DaVinci/utilities_script.py index 58575acbf..6e7a9436a 100644 --- a/Phys/DaVinci/python/DaVinci/utilities_script.py +++ b/Phys/DaVinci/python/DaVinci/utilities_script.py @@ -1,5 +1,5 @@ ############################################################################### -# (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration # +# (c) Copyright 2021-2022 CERN for the benefit of the LHCb Collaboration # # # # This software is distributed under the terms of the GNU General Public # # Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # -- GitLab From e527065cab28276d48675c9b9193e9ac3c62ba33 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <eduardo.rodrigues@cern.ch> Date: Mon, 24 Jan 2022 11:10:28 +0100 Subject: [PATCH 08/11] Apply 6 suggestion(s) to 2 file(s) --- DaVinciSys/tests/test_davinci_script.py | 4 ++-- Phys/DaVinci/python/DaVinci/utilities_script.py | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/DaVinciSys/tests/test_davinci_script.py b/DaVinciSys/tests/test_davinci_script.py index 9b244ca29..87ce6e432 100644 --- a/DaVinciSys/tests/test_davinci_script.py +++ b/DaVinciSys/tests/test_davinci_script.py @@ -69,7 +69,7 @@ def test_override_job_option_bis(): def test_run_option_shortcut(): """ - Verify that option values passed to davinci command are accepted also + Verify that the option values passed to the davinci command are accepted also when not using the full name (e.g. --inputfiledb ==> -i). """ cmd = 'davinci --dry-run -o test_run_option_shortcut.opts run-mc -i Upgrade_Bd2KstarMuMu -' @@ -87,7 +87,7 @@ def test_run_option_shortcut(): def test_create_options_templates(): """ - Verify that the 'create_options_templates' command produces a template for the inputfiledb and joboptfileoption files. + Verify that the 'create_options_templates' command produces a template for the inputfiledb and joboptfile option files. """ cmd = 'davinci create-options-templates -f test_inputdb_template.yaml test_jobopt_template.yaml' result = subprocess.run(cmd, shell=True) diff --git a/Phys/DaVinci/python/DaVinci/utilities_script.py b/Phys/DaVinci/python/DaVinci/utilities_script.py index 6e7a9436a..10f2b1168 100644 --- a/Phys/DaVinci/python/DaVinci/utilities_script.py +++ b/Phys/DaVinci/python/DaVinci/utilities_script.py @@ -122,14 +122,15 @@ def get_dummy_config(): def create_jobopt_template(file_name="jobopt_template.yaml", interactive=False): """ - Function for creating simple template for the joboptfile dict. + Function to create a simple template file with job options + as specified by the --joboptfile davinci script option. Args: - - file_name (str): name for the template file. Default = 'joboptfile_template.yaml'. - - interactive (bool): boolean for creating the template interactively. + file_name (str): name for the output template file. Default = 'jobopt_template.yaml'. + interactive (bool): flag to create the template interactively. Output: - - write a template for the job option file with the name defined by the user. + The template file with job options with the name defined by the user. """ if not interactive: template_str = "# Template job option YAML file.\n"\ -- GitLab From 01d3667fa1f6c49f567cc3cbbbd62a66ef6498ff Mon Sep 17 00:00:00 2001 From: Davide Fazzini <davide.fazzini@cern.ch> Date: Mon, 24 Jan 2022 11:25:39 +0100 Subject: [PATCH 09/11] apply suggestions --- DaVinciSys/scripts/davinci | 8 ++++---- DaVinciSys/tests/test_davinci_script.py | 2 +- Phys/DaVinci/python/DaVinci/utilities_script.py | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/DaVinciSys/scripts/davinci b/DaVinciSys/scripts/davinci index 3289b7772..dca6d00bb 100755 --- a/DaVinciSys/scripts/davinci +++ b/DaVinciSys/scripts/davinci @@ -135,9 +135,9 @@ def run_job(configurables, export=None, with_defaults=False, dry_run=None): @click.argument("option_type") @click.option( "-f", - "--filename", + "--filename_output", default="options_template.yaml", - help="Name for the template file.") + help="Name for the output template file.") @click.option( "-i", "--interactive", @@ -169,10 +169,10 @@ def create_options_template(option_type, filename, interactive): ignore_unknown_options=False, allow_extra_args=False)) @click.option( "-f", - "--filenames", + "--filenames_output", default=("inputdb_template.yaml", "jobopt_template.yaml"), nargs=2, - help="Name for the template files: inputfiledb and joboptfile.") + help="Name for the output template files: inputfiledb and joboptfile.") def create_options_templates(filenames): """ Create a template for the two options files to be passed to DaVinci when running a job:\n diff --git a/DaVinciSys/tests/test_davinci_script.py b/DaVinciSys/tests/test_davinci_script.py index 87ce6e432..1617296a7 100644 --- a/DaVinciSys/tests/test_davinci_script.py +++ b/DaVinciSys/tests/test_davinci_script.py @@ -87,7 +87,7 @@ def test_run_option_shortcut(): def test_create_options_templates(): """ - Verify that the 'create_options_templates' command produces a template for the inputfiledb and joboptfile option files. + Verify that the 'create-options-templates' command produces a template for the inputfiledb and joboptfile option files. """ cmd = 'davinci create-options-templates -f test_inputdb_template.yaml test_jobopt_template.yaml' result = subprocess.run(cmd, shell=True) diff --git a/Phys/DaVinci/python/DaVinci/utilities_script.py b/Phys/DaVinci/python/DaVinci/utilities_script.py index 10f2b1168..78d1207b4 100644 --- a/Phys/DaVinci/python/DaVinci/utilities_script.py +++ b/Phys/DaVinci/python/DaVinci/utilities_script.py @@ -88,11 +88,11 @@ def set_testfiledb(inputfiledb): Set the TestFileDB file and the related key to be used in the job. Args: - inputfiledb: pair of values containing information on the TestFileDB and the key chosen by the user. + inputfiledb (str): pair of values containing information on the TestFileDB and the key chosen by the user. Returns: - - key for the TestFileDB. - - file containing the TestFileDB. + key for the TestFileDB. + file containing the TestFileDB. """ assert len( inputfiledb @@ -130,7 +130,7 @@ def create_jobopt_template(file_name="jobopt_template.yaml", interactive (bool): flag to create the template interactively. Output: - The template file with job options with the name defined by the user. + the template file with job options and the name defined by the user. """ if not interactive: template_str = "# Template job option YAML file.\n"\ @@ -178,13 +178,13 @@ def create_jobopt_template(file_name="jobopt_template.yaml", def create_inputdb_template(file_name="inputdb_template.yaml"): """ - Function for creating simple template for the TestFileDB-like file. + Function to create a simple template file for the TestFileDB-like inputdb file. Args: - - file_name (str): name for the template file. Default = 'inputdb_template.yaml'. + file_name (str): name for the output template file. Default = 'inputdb_template.yaml'. Output: - - write a template for the TestFileDB-like input file with the name defined by the user. + the template TestFileDB-like file with input data options and the name defined by the user. """ template_str = "# Template input data YAML file.\n"\ -- GitLab From 7fbfa80fffa1b5330ac22493e844621485c086ad Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <eduardo.rodrigues@cern.ch> Date: Mon, 24 Jan 2022 11:41:40 +0100 Subject: [PATCH 10/11] Apply 1 suggestion(s) to 1 file(s) --- DaVinciSys/tests/test_davinci_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DaVinciSys/tests/test_davinci_script.py b/DaVinciSys/tests/test_davinci_script.py index 1617296a7..7be0a2639 100644 --- a/DaVinciSys/tests/test_davinci_script.py +++ b/DaVinciSys/tests/test_davinci_script.py @@ -100,7 +100,7 @@ def test_create_options_templates(): os.path.expandvars("test_inputdb_template.yaml")) as inputdb_file: assert yaml.safe_load(inputdb_file) - with open(os.path.expandvars("test_inputdb_template.yaml")) as jobopt_file: + with open(os.path.expandvars("test_jobopt_template.yaml")) as jobopt_file: assert yaml.safe_load(jobopt_file) os.remove('test_inputdb_template.yaml') -- GitLab From 582c229cabf28d84ebfa5cd1a655f23031cc9813 Mon Sep 17 00:00:00 2001 From: Davide Fazzini <davide.fazzini@cern.ch> Date: Mon, 24 Jan 2022 16:50:53 +0100 Subject: [PATCH 11/11] fix failing test --- DaVinciSys/scripts/davinci | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/DaVinciSys/scripts/davinci b/DaVinciSys/scripts/davinci index dca6d00bb..8277a839b 100755 --- a/DaVinciSys/scripts/davinci +++ b/DaVinciSys/scripts/davinci @@ -144,7 +144,7 @@ def run_job(configurables, export=None, with_defaults=False, dry_run=None): is_flag=True, default=False, help="Option for creating the template interactively.") -def create_options_template(option_type, filename, interactive): +def create_options_template(option_type, filename_output, interactive): """ Create a template for the options file to be passed to DaVinci when running a job.\n E.g. ./run davinci create-options-template job (for a template of the joboptfile argument).\n @@ -157,9 +157,9 @@ def create_options_template(option_type, filename, interactive): "Use 'job' for creating a template of the joboptfile or 'data' for the inputfiledb." if option_type == "job": - create_jobopt_template(filename, interactive) + create_jobopt_template(filename_output, interactive) else: - create_inputdb_template(filename) + create_inputdb_template(filename_output) return get_dummy_config() @@ -173,7 +173,7 @@ def create_options_template(option_type, filename, interactive): default=("inputdb_template.yaml", "jobopt_template.yaml"), nargs=2, help="Name for the output template files: inputfiledb and joboptfile.") -def create_options_templates(filenames): +def create_options_templates(filenames_output): """ Create a template for the two options files to be passed to DaVinci when running a job:\n - the inputfiledb containing all the information related to the input data;\n @@ -186,8 +186,8 @@ def create_options_templates(filenames): create-options-template as shown in the help. """ - create_inputdb_template(filenames[0]) - create_jobopt_template(filenames[1]) + create_inputdb_template(filenames_output[0]) + create_jobopt_template(filenames_output[1]) return get_dummy_config() -- GitLab