From 6045671c233a8eafad70452c95b4fa18cb1d0d5f Mon Sep 17 00:00:00 2001
From: Jonas Eschle 'Mayou36 <jonas.eschle@cern.ch>
Date: Thu, 7 Dec 2017 13:17:36 +0100
Subject: [PATCH 1/6] Add --verbose argument to toys scripts.

---
 analysis/toys/generate_toys.py |  6 ++++++
 analysis/toys/submit_toys.py   | 15 +++++++++++----
 analysis/toys/submitter.py     |  5 ++++-
 3 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/analysis/toys/generate_toys.py b/analysis/toys/generate_toys.py
index 0f1d64f..ed4552d 100644
--- a/analysis/toys/generate_toys.py
+++ b/analysis/toys/generate_toys.py
@@ -206,6 +206,9 @@ def main():
 
     """
     parser = argparse.ArgumentParser()
+    parser.add_argument('-v', '--verbose',
+                        action='store_true',
+                        help="Verbose output")
     parser.add_argument('--link-from',
                         action='store', type=str, default='',
                         help="Folder to actually store the toy files")
@@ -213,6 +216,9 @@ def main():
                         action='store', type=str, nargs='+',
                         help="Configuration files")
     args = parser.parse_args()
+    if args.verbose:
+        get_logger('analysis').setLevel(1)
+        logger.setLevel(1)
     try:
         run(args.config, args.link_from)
         exit_status = 0
diff --git a/analysis/toys/submit_toys.py b/analysis/toys/submit_toys.py
index 7c875ef..f5a1fd2 100644
--- a/analysis/toys/submit_toys.py
+++ b/analysis/toys/submit_toys.py
@@ -197,6 +197,9 @@ def main():
         return list(sum(list_, typ_))
 
     parser = argparse.ArgumentParser()
+    parser.add_argument('-v', '--verbose',
+                        action='store_true',
+                        help="Verbose output")
     parser.add_argument('--link-from',
                         action='store', type=str,
                         help="Folder to actually store the toy files")
@@ -210,6 +213,9 @@ def main():
                         action='store', type=str, nargs='+',
                         help="Configuration file")
     args = parser.parse_args()
+    if args.verbose:
+        get_logger('analysis').setLevel(1)
+        logger.setLevel(1)
     try:
         config = _config.load_config(*args.config)
         # Which type of toy are we running?
@@ -266,10 +272,11 @@ def main():
                                      'toys',
                                      script_to_run)
         for config_file in config_files:
-            submitter([config_file],
-                      args.link_from,
-                      args.extend,
-                      args.overwrite).run(script_to_run)
+            submitter(config_files=[config_file],
+                      link_from=args.link_from,
+                      extend=args.extend,
+                      overwrite=args.overwrite,
+                      verbose=args.verbose).run(script_to_run,)
             if scan_config:
                 os.remove(config_file)
         exit_status = 0
diff --git a/analysis/toys/submitter.py b/analysis/toys/submitter.py
index b0d6f3d..f14f186 100644
--- a/analysis/toys/submitter.py
+++ b/analysis/toys/submitter.py
@@ -47,7 +47,7 @@ class ToySubmitter(object):
     NTOYS_KEY = None
     NTOYS_PER_JOB_KEY = None
 
-    def __init__(self, config_files, link_from, extend, overwrite):
+    def __init__(self, config_files, link_from, extend, overwrite, verbose=False):
         """Configure the toy submitter.
 
         Arguments:
@@ -96,6 +96,7 @@ class ToySubmitter(object):
         self.link_from = link_from
         self.extend = extend
         self.overwrite = overwrite
+        self.verbose = verbose
         # Get the batch system
         self.batch_system = get_batch_system()
 
@@ -152,6 +153,8 @@ class ToySubmitter(object):
         script_args = []
         if self.config['link-from']:
             script_args.append('--link-from={}'.format(self.config['link-from']))
+        if self.verbose:
+            script_args.append('--verbose')
         script_args.append(config_file_dest)
         # Prepare paths
         # pylint: disable=E1101
-- 
GitLab


From 09d7f7d7f2095e60d0d62e3df43ab1034f29ee57 Mon Sep 17 00:00:00 2001
From: Jonas Eschle 'Mayou36 <jonas.eschle@cern.ch>
Date: Thu, 7 Dec 2017 23:32:34 +0100
Subject: [PATCH 2/6] Catch hdf file compression/moving error

---
 analysis/data/hdf.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/analysis/data/hdf.py b/analysis/data/hdf.py
index 0b48518..0298118 100644
--- a/analysis/data/hdf.py
+++ b/analysis/data/hdf.py
@@ -60,9 +60,14 @@ def modify_hdf(file_name, compress=True):
             out = subprocess.check_output(cmd)
             if not os.path.exists(compressed_file):  # Something went wrong
                 raise subprocess.CalledProcessError(0, ' '.join(cmd), output=out)
-            shutil.move(compressed_file, file_name)
         except subprocess.CalledProcessError as error:
             logger.warning("Error compressing -> %s", error.output)
+        else:
+            try:
+                shutil.move(compressed_file, file_name)
+            except IOError as error:
+                logger.warning("Error moving (copying) compressed file -> %s", error)
+        finally:
             if os.path.exists(compressed_file):
                 os.remove(compressed_file)
 
-- 
GitLab


From dc0c9b5a064f2995670dd9dabe85057b2fe3edd1 Mon Sep 17 00:00:00 2001
From: Jonas Eschle 'Mayou36 <jonas.eschle@cern.ch>
Date: Fri, 8 Dec 2017 11:11:24 +0100
Subject: [PATCH 3/6] Add possibility to overwrite existing category in
 fit_toys if existing column contains only one category

---
 analysis/toys/fit_toys.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/analysis/toys/fit_toys.py b/analysis/toys/fit_toys.py
index 2e95fb7..e66ef4e 100644
--- a/analysis/toys/fit_toys.py
+++ b/analysis/toys/fit_toys.py
@@ -90,10 +90,15 @@ def get_datasets(data_frames, acceptance, fit_models):
         # Add category column
         if category:
             # By default the label is stored in the 'category' column
-            if 'category' in rows and not all(rows['category']==category):
-                logger.error("Data %s contains categories not matching the specified category: ",
-                             rows, set(rows['category']) - {category})
-                raise ValueError("Data {} has categories different to the specified one".format(data_name))  # TODO: replace with DataError
+            if 'category' in rows:
+                if len(set(rows['category'])) > 1:
+                    logger.error("Data %s contains more then one category: %s!",
+                                 rows, set(rows['category']))
+                    raise ValueError("Data {} contains more then one category: ".format(data_name))  # TODO: replace with DataError
+                elif not rows['category'].iloc[0] == category:
+                    logger.info("Data %s contains a category %s, dropping it and "
+                                "replacing it with the specified one %s", rows,
+                                rows['category'].iloc[0], category)
             rows['category'] = category
         elif 'category' in rows:
             logger.warning("Data %s contains a 'category' column but no category was specified"
-- 
GitLab


From e72ef0d4c371d0cee09127ff5dd46136910506a4 Mon Sep 17 00:00:00 2001
From: Jonas Eschle 'Mayou36 <jonas.eschle@cern.ch>
Date: Fri, 8 Dec 2017 12:01:17 +0100
Subject: [PATCH 4/6] Refactor temp file creation

---
 analysis/toys/submit_toys.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/analysis/toys/submit_toys.py b/analysis/toys/submit_toys.py
index f5a1fd2..9afce7a 100644
--- a/analysis/toys/submit_toys.py
+++ b/analysis/toys/submit_toys.py
@@ -256,9 +256,8 @@ def main():
                              temp_config['name'],
                              ", ".join('{}: {}'.format(*val) for val in values.items()))
                 # Write temp_file
-                file_ = tempfile.NamedTemporaryFile(delete=False)
-                file_name = file_.name
-                file_.close()
+                with tempfile.NamedTemporaryFile(delete=False) as file_:
+                    file_name = file_.nameCan
                 _config.write_config(_config.fold_config(list(temp_config.items())), file_name)
                 config_files.append(file_name)
         else:
-- 
GitLab


From dc13d917d50c2e554dd663e10c0e1bc78991c705 Mon Sep 17 00:00:00 2001
From: Jonas Eschle 'Mayou36 <jonas.eschle@cern.ch>
Date: Mon, 11 Dec 2017 11:30:06 +0100
Subject: [PATCH 5/6] Fix typo

---
 analysis/toys/submit_toys.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/analysis/toys/submit_toys.py b/analysis/toys/submit_toys.py
index 9afce7a..5a9f4ad 100644
--- a/analysis/toys/submit_toys.py
+++ b/analysis/toys/submit_toys.py
@@ -257,7 +257,7 @@ def main():
                              ", ".join('{}: {}'.format(*val) for val in values.items()))
                 # Write temp_file
                 with tempfile.NamedTemporaryFile(delete=False) as file_:
-                    file_name = file_.nameCan
+                    file_name = file_.name
                 _config.write_config(_config.fold_config(list(temp_config.items())), file_name)
                 config_files.append(file_name)
         else:
-- 
GitLab


From a61c7064ca7c9de2ce2a9b1ed6bc7bfd4e42b9e0 Mon Sep 17 00:00:00 2001
From: Michele Atzeni <matzeni@physik.uzh.ch>
Date: Wed, 20 Dec 2017 11:29:15 +0100
Subject: [PATCH 6/6] Fix bug in generating toys with given range

---
 analysis/physics/factory.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/analysis/physics/factory.py b/analysis/physics/factory.py
index 6878c11..47cf5e7 100644
--- a/analysis/physics/factory.py
+++ b/analysis/physics/factory.py
@@ -427,6 +427,8 @@ class BaseFactory(object):
                 new_config[3] = max_
                 if obs_id in self:
                     self._objects[obs_id].setRange(range_name, min_, max_)
+                    self._objects[obs_id].setMin(min_)
+                    self._objects[obs_id].setMax(max_)
             if units:
                 new_config[4] = units
                 if obs_id in self:
-- 
GitLab