From 6045671c233a8eafad70452c95b4fa18cb1d0d5f Mon Sep 17 00:00:00 2001 From: Jonas Eschle 'Mayou36 <jonas.eschle@cern.ch> Date: Thu, 7 Dec 2017 13:17:36 +0100 Subject: [PATCH 1/6] Add --verbose argument to toys scripts. --- analysis/toys/generate_toys.py | 6 ++++++ analysis/toys/submit_toys.py | 15 +++++++++++---- analysis/toys/submitter.py | 5 ++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/analysis/toys/generate_toys.py b/analysis/toys/generate_toys.py index 0f1d64f..ed4552d 100644 --- a/analysis/toys/generate_toys.py +++ b/analysis/toys/generate_toys.py @@ -206,6 +206,9 @@ def main(): """ parser = argparse.ArgumentParser() + parser.add_argument('-v', '--verbose', + action='store_true', + help="Verbose output") parser.add_argument('--link-from', action='store', type=str, default='', help="Folder to actually store the toy files") @@ -213,6 +216,9 @@ def main(): action='store', type=str, nargs='+', help="Configuration files") args = parser.parse_args() + if args.verbose: + get_logger('analysis').setLevel(1) + logger.setLevel(1) try: run(args.config, args.link_from) exit_status = 0 diff --git a/analysis/toys/submit_toys.py b/analysis/toys/submit_toys.py index 7c875ef..f5a1fd2 100644 --- a/analysis/toys/submit_toys.py +++ b/analysis/toys/submit_toys.py @@ -197,6 +197,9 @@ def main(): return list(sum(list_, typ_)) parser = argparse.ArgumentParser() + parser.add_argument('-v', '--verbose', + action='store_true', + help="Verbose output") parser.add_argument('--link-from', action='store', type=str, help="Folder to actually store the toy files") @@ -210,6 +213,9 @@ def main(): action='store', type=str, nargs='+', help="Configuration file") args = parser.parse_args() + if args.verbose: + get_logger('analysis').setLevel(1) + logger.setLevel(1) try: config = _config.load_config(*args.config) # Which type of toy are we running? @@ -266,10 +272,11 @@ def main(): 'toys', script_to_run) for config_file in config_files: - submitter([config_file], - args.link_from, - args.extend, - args.overwrite).run(script_to_run) + submitter(config_files=[config_file], + link_from=args.link_from, + extend=args.extend, + overwrite=args.overwrite, + verbose=args.verbose).run(script_to_run,) if scan_config: os.remove(config_file) exit_status = 0 diff --git a/analysis/toys/submitter.py b/analysis/toys/submitter.py index b0d6f3d..f14f186 100644 --- a/analysis/toys/submitter.py +++ b/analysis/toys/submitter.py @@ -47,7 +47,7 @@ class ToySubmitter(object): NTOYS_KEY = None NTOYS_PER_JOB_KEY = None - def __init__(self, config_files, link_from, extend, overwrite): + def __init__(self, config_files, link_from, extend, overwrite, verbose=False): """Configure the toy submitter. Arguments: @@ -96,6 +96,7 @@ class ToySubmitter(object): self.link_from = link_from self.extend = extend self.overwrite = overwrite + self.verbose = verbose # Get the batch system self.batch_system = get_batch_system() @@ -152,6 +153,8 @@ class ToySubmitter(object): script_args = [] if self.config['link-from']: script_args.append('--link-from={}'.format(self.config['link-from'])) + if self.verbose: + script_args.append('--verbose') script_args.append(config_file_dest) # Prepare paths # pylint: disable=E1101 -- GitLab From 09d7f7d7f2095e60d0d62e3df43ab1034f29ee57 Mon Sep 17 00:00:00 2001 From: Jonas Eschle 'Mayou36 <jonas.eschle@cern.ch> Date: Thu, 7 Dec 2017 23:32:34 +0100 Subject: [PATCH 2/6] Catch hdf file compression/moving error --- analysis/data/hdf.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/analysis/data/hdf.py b/analysis/data/hdf.py index 0b48518..0298118 100644 --- a/analysis/data/hdf.py +++ b/analysis/data/hdf.py @@ -60,9 +60,14 @@ def modify_hdf(file_name, compress=True): out = subprocess.check_output(cmd) if not os.path.exists(compressed_file): # Something went wrong raise subprocess.CalledProcessError(0, ' '.join(cmd), output=out) - shutil.move(compressed_file, file_name) except subprocess.CalledProcessError as error: logger.warning("Error compressing -> %s", error.output) + else: + try: + shutil.move(compressed_file, file_name) + except IOError as error: + logger.warning("Error moving (copying) compressed file -> %s", error) + finally: if os.path.exists(compressed_file): os.remove(compressed_file) -- GitLab From dc0c9b5a064f2995670dd9dabe85057b2fe3edd1 Mon Sep 17 00:00:00 2001 From: Jonas Eschle 'Mayou36 <jonas.eschle@cern.ch> Date: Fri, 8 Dec 2017 11:11:24 +0100 Subject: [PATCH 3/6] Add possibility to overwrite existing category in fit_toys if existing column contains only one category --- analysis/toys/fit_toys.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/analysis/toys/fit_toys.py b/analysis/toys/fit_toys.py index 2e95fb7..e66ef4e 100644 --- a/analysis/toys/fit_toys.py +++ b/analysis/toys/fit_toys.py @@ -90,10 +90,15 @@ def get_datasets(data_frames, acceptance, fit_models): # Add category column if category: # By default the label is stored in the 'category' column - if 'category' in rows and not all(rows['category']==category): - logger.error("Data %s contains categories not matching the specified category: ", - rows, set(rows['category']) - {category}) - raise ValueError("Data {} has categories different to the specified one".format(data_name)) # TODO: replace with DataError + if 'category' in rows: + if len(set(rows['category'])) > 1: + logger.error("Data %s contains more then one category: %s!", + rows, set(rows['category'])) + raise ValueError("Data {} contains more then one category: ".format(data_name)) # TODO: replace with DataError + elif not rows['category'].iloc[0] == category: + logger.info("Data %s contains a category %s, dropping it and " + "replacing it with the specified one %s", rows, + rows['category'].iloc[0], category) rows['category'] = category elif 'category' in rows: logger.warning("Data %s contains a 'category' column but no category was specified" -- GitLab From e72ef0d4c371d0cee09127ff5dd46136910506a4 Mon Sep 17 00:00:00 2001 From: Jonas Eschle 'Mayou36 <jonas.eschle@cern.ch> Date: Fri, 8 Dec 2017 12:01:17 +0100 Subject: [PATCH 4/6] Refactor temp file creation --- analysis/toys/submit_toys.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/analysis/toys/submit_toys.py b/analysis/toys/submit_toys.py index f5a1fd2..9afce7a 100644 --- a/analysis/toys/submit_toys.py +++ b/analysis/toys/submit_toys.py @@ -256,9 +256,8 @@ def main(): temp_config['name'], ", ".join('{}: {}'.format(*val) for val in values.items())) # Write temp_file - file_ = tempfile.NamedTemporaryFile(delete=False) - file_name = file_.name - file_.close() + with tempfile.NamedTemporaryFile(delete=False) as file_: + file_name = file_.nameCan _config.write_config(_config.fold_config(list(temp_config.items())), file_name) config_files.append(file_name) else: -- GitLab From dc13d917d50c2e554dd663e10c0e1bc78991c705 Mon Sep 17 00:00:00 2001 From: Jonas Eschle 'Mayou36 <jonas.eschle@cern.ch> Date: Mon, 11 Dec 2017 11:30:06 +0100 Subject: [PATCH 5/6] Fix typo --- analysis/toys/submit_toys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analysis/toys/submit_toys.py b/analysis/toys/submit_toys.py index 9afce7a..5a9f4ad 100644 --- a/analysis/toys/submit_toys.py +++ b/analysis/toys/submit_toys.py @@ -257,7 +257,7 @@ def main(): ", ".join('{}: {}'.format(*val) for val in values.items())) # Write temp_file with tempfile.NamedTemporaryFile(delete=False) as file_: - file_name = file_.nameCan + file_name = file_.name _config.write_config(_config.fold_config(list(temp_config.items())), file_name) config_files.append(file_name) else: -- GitLab From a61c7064ca7c9de2ce2a9b1ed6bc7bfd4e42b9e0 Mon Sep 17 00:00:00 2001 From: Michele Atzeni <matzeni@physik.uzh.ch> Date: Wed, 20 Dec 2017 11:29:15 +0100 Subject: [PATCH 6/6] Fix bug in generating toys with given range --- analysis/physics/factory.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/analysis/physics/factory.py b/analysis/physics/factory.py index 6878c11..47cf5e7 100644 --- a/analysis/physics/factory.py +++ b/analysis/physics/factory.py @@ -427,6 +427,8 @@ class BaseFactory(object): new_config[3] = max_ if obs_id in self: self._objects[obs_id].setRange(range_name, min_, max_) + self._objects[obs_id].setMin(min_) + self._objects[obs_id].setMax(max_) if units: new_config[4] = units if obs_id in self: -- GitLab