Merge branch 'dfazzini_moving_override_data_options' into 'master'

Making override_data_options a davinci script option.

See merge request !628

DaVinciSys/scripts/davinci

@@ -132,6 +132,7 @@ inputfiledb_helper = "TestFileDB-like file containing job input and conditions i
                       "E.g. for TestFileDB:\n./run davinci [command] --inputfiledb Upgrade_Bd2KstarMuMu -"
 joboptfile_helper = "Option file containing the job information (.yaml, .py)"
 simplejob_helper = "Option for running a simple DaVinci job without any specific configuration (.py)."
+override_data_options_helper = "Allow overriding default data options defined in DaVinci Database."
 
 
 @main.command(
@@ -147,8 +148,13 @@ simplejob_helper = "Option for running a simple DaVinci job without any specific
 @click.option("--joboptfile", default="", help=joboptfile_helper)
 @click.option(
     "--simplejob", is_flag=True, default=False, help=simplejob_helper)
+@click.option(
+    "--override_data_options",
+    is_flag=True,
+    default=False,
+    help=override_data_options_helper)
 @click.pass_context
-def run_mc(ctx, inputfiledb, joboptfile, simplejob):
+def run_mc(ctx, inputfiledb, joboptfile, simplejob, override_data_options):
     """
     DaVinci function for running jobs on simulated samples.
 
@@ -176,7 +182,8 @@ def run_mc(ctx, inputfiledb, joboptfile, simplejob):
         inputfiledb_file,  #file and key for job input and conditions
         joboptfile,  # file for job options
         ctx_args,  # list of extra options to be set in the job
-        simplejob  # flag for running a simplejob
+        simplejob,  # flag for running a simplejob
+        override_data_options  # flag for overriding data options in the job
     )
 
     return config
@@ -195,8 +202,13 @@ def run_mc(ctx, inputfiledb, joboptfile, simplejob):
 @click.option("--joboptfile", default="", help=joboptfile_helper)
 @click.option(
     "--simplejob", is_flag=True, default=False, help=simplejob_helper)
+@click.option(
+    "--override_data_options",
+    is_flag=True,
+    default=False,
+    help=override_data_options_helper)
 @click.pass_context
-def run_data(ctx, inputfiledb, joboptfile, simplejob):
+def run_data(ctx, inputfiledb, joboptfile, simplejob, override_data_options):
     """
     DaVinci function for running jobs on detector data samples.
 
@@ -228,7 +240,8 @@ def run_data(ctx, inputfiledb, joboptfile, simplejob):
         inputfiledb_file,  #file and key for job input and conditions
         joboptfile,  # file for job options
         ctx_args,  # list of extra options to be set in the job
-        simplejob  # flag for running a simplejob
+        simplejob,  # flag for running a simplejob
+        override_data_options  # flag for overriding data options in the job
     )
 
     return config

DaVinciSys/tests/test_davinci_script.py

@@ -34,7 +34,7 @@ def test_override_job_option():
     Verify the overriding of a DaVinci option on the fly.
     No need to actually run the job, hence the dry run.
     """
-    cmd = 'davinci --export test_override_job_option.opts --dry-run run-mc --override_data_options True --process "Turbo" --inputfiledb Upgrade_Bd2KstarMuMu -'
+    cmd = 'davinci --export test_override_job_option.opts --dry-run run-mc --override_data_options --process "Turbo" --inputfiledb Upgrade_Bd2KstarMuMu -'
     result = subprocess.run(cmd, shell=True)
 
     # Just be maniac - the command should work ;-)
@@ -54,7 +54,7 @@ def test_override_job_option_bis():
     (see related test "test_override_job_option").
     No need to actually run the job, hence the dry run.
     """
-    cmd = 'davinci --export test_override_job_option_bis.opts --dry-run run-mc  --process "Turbo" --override_data_options True --inputfiledb Upgrade_Bd2KstarMuMu -'
+    cmd = 'davinci --export test_override_job_option_bis.opts --dry-run run-mc  --process "Turbo" --override_data_options --inputfiledb Upgrade_Bd2KstarMuMu -'
     result = subprocess.run(cmd, shell=True)
 
     # Just be maniac - the command should work ;-)

Phys/DaVinci/python/DaVinci/ConfigurationUpgrade.py

@@ -25,6 +25,7 @@ def run_davinci_app(fileDB_key="",
                     jobOpt_file="",
                     ctx_args=[],
                     simple_job=False,
+                    override_data_options=False,
                     prod_conf=False):
     """
     Run the DaVinci application.
@@ -45,6 +46,8 @@ def run_davinci_app(fileDB_key="",
              Defaults to [].
         simple_job (bool, optional): flag for running job using only the PyConf configurations.
              Defaults to False.
+        override_data_options (bool, optional): boolean for enabling data options override.
+             Defaults to False.
         prod_conf (bool, optional): flag for running DaVinci with ProdConf.
              Defaults to False.
 
@@ -56,9 +59,11 @@ def run_davinci_app(fileDB_key="",
     if fileDB_key and not prod_conf:
         set_input_file_options(options, fileDB_key, fileDB_file)
     if jobOpt_file:
-        set_job_options(options, jobOpt_file, fileDB_key, fileDB_file)
+        set_job_options(options, jobOpt_file, fileDB_key, fileDB_file,
+                        override_data_options)
     if ctx_args:
-        set_args_options(options, ctx_args, fileDB_key, fileDB_file)
+        set_args_options(options, ctx_args, fileDB_key, fileDB_file,
+                         override_data_options)
 
     config = ComponentConfig()
 

Phys/DaVinci/python/DaVinci/configOptions.py

@@ -104,7 +104,7 @@ def set_input_file_options(options, fileDB_key, fileDB_file):
                     set_option_value(options, "input_files", obj)
 
 
-def set_job_options(options, jobOptFile, fileDB_key, fileDB_file):
+def set_job_options(options, jobOptFile, fileDB_key, fileDB_file, override_data_options):
     """
     Set the job properties required by the user. The method checks if there
     are options related to the input files selected with key and file of the
@@ -115,6 +115,7 @@ def set_job_options(options, jobOptFile, fileDB_key, fileDB_file):
        - jobOptFile: file containing the job options chosen by the user.
        - fileDB_key: key in the testfileDB.
        - fileDB_file: file containing the testfileDB.
+       - override_data_options: boolean for enabling data options override.
     """
     if jobOptFile == '':
         log_click("WARNING",
@@ -136,11 +137,11 @@ def set_job_options(options, jobOptFile, fileDB_key, fileDB_file):
                 )
 
             for key, value in config.items():
-                if is_option_settable(options, key, dataOptions):
+                if is_option_settable(options, key, dataOptions, override_data_options):
                     set_option_value(options, key, value)
 
 
-def set_args_options(options, ctx_args, fileDB_key, fileDB_file):
+def set_args_options(options, ctx_args, fileDB_key, fileDB_file, override_data_options):
     """
     Set the extra arguments required by the user.
 
@@ -149,27 +150,20 @@ def set_args_options(options, ctx_args, fileDB_key, fileDB_file):
        - ctx_args: click context.
        - fileDB_key: key in the testfileDB.
        - fileDB_file: file containing the testfileDB.
+       - override_data_options: boolean for enabling data options override.
     """
     dataOptions = list_data_options(fileDB_key.split(":")[0], fileDB_file)
 
-    # First check if the 'override_data_options' property has been defined
-    # If yes then move it at the beginning of the list in order to know
-    # how to deal with the data options properties
-    for i in range(0, len(ctx_args), 2):
-        if ctx_args[i] == '--override_data_options':
-            # Move the properties name and value to the beginning of the list
-            ctx_args = [ctx_args.pop(i), ctx_args.pop(i)] + ctx_args
-
     # Loop over the properties and set their values
     for i in range(0, len(ctx_args), 2):
         key = ctx_args[i][2:]
         value = ctx_args[i + 1]
 
-        if is_option_settable(options, key, dataOptions):
+        if is_option_settable(options, key, dataOptions, override_data_options):
             set_option_value(options, key, value)
 
 
-def is_option_settable(options, name, dataOptions):
+def is_option_settable(options, name, dataOptions, override_data_options):
     """
     Check if the option identified by name is settable.
     This is true for all the job options while for data options
@@ -181,9 +175,10 @@ def is_option_settable(options, name, dataOptions):
        - options: list of DaVinci options.
        - name: name of the option to be checked.
        - dataOptions: list of options related to the input data.
+       - override_data_options: boolean for enabling data options override.
     """
     if name in dataOptions:
-        if get_option_value(options, "override_data_options"):
+        if override_data_options:
             log_click(
                 "INFO",
                 "New value found for the option %s in the job option file. "\