Speed-up plotting routines

Description

Implement parallel plotting via multiprocessing for the DAC scans and S-bit rate scan analysis routines.

Related Issue

Fixes #37 (closed).

How Has This Been Tested?

Taking example input files available in /gemdata/data/dac-scan-in-2022081701 and /afs/cern.ch/work/c/cgalloni/public/4Yechan/ for the DAC scans and /gemdata/data/sbit-rate-scan-in-inductionOffHVstandby-2022110801 for the S-bit rate scan.

The changes were tested by running:

gemos analyze dac dac-scan-in-2022081701/fed146* test_dac1/ -m 4Yechan/vfat_map.dat -c 4Yechan/vfat_calibration.txt -p always

gemos analyze sbit sbit-rate-scan-in-inductionOffHVstandby-2022110801/fed1467-slot* test1_sbit/ -p 

Types of changes

• Bug fix (non-breaking change which fixes an issue)
• New feature (non-breaking change which adds functionality)
• Breaking change (fix or feature that would cause existing functionality to change)

Checklist:

• My code follows the code style of this project.
• My change requires a change to the documentation.
• I have updated the documentation accordingly.
• I have read the CONTRIBUTING document.
• I have added tests to cover my changes.
• All new and existing tests passed.

gemos/analysis/dac_scan_analysis.py

-"""DAC scan routines"""
+"""
+@file dac_scan_analysis.py
+@author Antonello Pellecchia (antonello.pallecchia@cern.ch)
+@brief DAC scan routines
+@date 2022-12-06                                     
+"""
+__author__ = "Antonello Pellecchia"
+__maintainer__ = ["Camilla Galloni, Laurent Petre", "Daniel Estrada"]
 
-from enum import Enum
-import os
 
+import os
 import matplotlib.pyplot as plt
+import gc
 import mplhep
-import numpy as np
 import pandas as pd
+from numpy import ceil, argmin
+from enum import Enum
+from multiprocessing import cpu_count, Pool
 
 from gemos.analysis.vfat_calibrations import add_parameters
 
@@ -96,7 +105,7 @@ def analyze_scan(input_df):
         nominal_value = NOMINAL_DAC_VALUES[dac_name][0]
 
         # Calculate chosen ADC value as closest to nominal DAC value
-        chosen_index = np.argmin(abs(adc_values - nominal_value))
+        chosen_index = argmin(abs(adc_values - nominal_value))
         chosen_dac = dac_values[chosen_index]
 
         # Issue warnings
@@ -129,57 +138,50 @@ def analyze_scan(input_df):
     return pd.DataFrame.from_records(output_list).astype(int, errors="ignore")
 
 
-def plot_scan(input_df, output_df, output_dir):
+def plot_scan(vfat_group, output_df, output_dir):
     """Plot all DAC rate scans"""
+    (fed, slot, oh, vfat), vfat_df = vfat_group
 
-    output_df = output_df.set_index(["fed", "slot", "oh", "vfat", "dac-name"]).sort_index()
-
-    vfat_group = input_df.groupby(["fed", "slot", "oh", "vfat"])
+    dac_group = vfat_df.groupby("dac-name")
 
-    for (fed, slot, oh, vfat), vfat_df in vfat_group:
-        dac_group = vfat_df.groupby("dac-name")
+    # Prepare figure for plotting
+    nrows, ncols = 3, int(ceil(len(dac_group) / 3))
+    dac_scan_fig, dac_scan_axs = plt.subplots(nrows, ncols, figsize=(55, 25))
+    dac_scan_axs = dac_scan_axs.flat
 
-        # Prepare figure for plotting
-        nrows, ncols = 3, int(np.ceil(len(dac_group) / 3))
-        dac_scan_fig, dac_scan_axs = plt.subplots(nrows, ncols, figsize=(55, 25))
-        dac_scan_axs = dac_scan_axs.flat
+    # Plot all DAC
+    for group_index, (dac_name, dac_df) in enumerate(dac_group):
+        # Plot DAC scan for current VFAT and register
+        dac_scan_axs[group_index].set_xlabel("DAC units")
+        dac_scan_axs[group_index].set_ylabel(f"{dac_name} ({NOMINAL_DAC_VALUES[dac_name][1]})")
 
-        # Plot all DAC
-        for group_index, (dac_name, dac_df) in enumerate(dac_group):
-            # Plot DAC scan for current VFAT and register
-            dac_scan_axs[group_index].set_xlabel("DAC units")
-            dac_scan_axs[group_index].set_ylabel(f"{dac_name} ({NOMINAL_DAC_VALUES[dac_name][1]})")
+        dac_scan_axs[group_index].plot(dac_df["dac-value"], dac_df["adc-value"], ".", color="black")
 
-            dac_scan_axs[group_index].plot(
-                dac_df["dac-value"], dac_df["adc-value"], ".", color="black"
-            )
-
-            # Draw line pointing to set DAC value
-            nominal_value = NOMINAL_DAC_VALUES[dac_name][0]
-            chosen_dac = output_df.loc[(fed, slot, oh, vfat, dac_name), "dac-value"]
+        # Draw line pointing to set DAC value
+        nominal_value = NOMINAL_DAC_VALUES[dac_name][0]
+        chosen_dac = output_df.loc[(fed, slot, oh, vfat, dac_name), "dac-value"]
 
-            dac_scan_axs[group_index].plot(
-                [chosen_dac, chosen_dac],
-                [min(dac_df["adc-value"]), nominal_value],
-                "--",
-                color="blue",
-            )
-            dac_scan_axs[group_index].plot(
-                [min(dac_df["dac-value"]), chosen_dac],
-                [nominal_value, nominal_value],
-                "--",
-                color="blue",
-            )
+        dac_scan_axs[group_index].plot(
+            [chosen_dac, chosen_dac],
+            [min(dac_df["adc-value"]), nominal_value],
+            "--",
+            color="blue",
+        )
+        dac_scan_axs[group_index].plot(
+            [min(dac_df["dac-value"]), chosen_dac],
+            [nominal_value, nominal_value],
+            "--",
+            color="blue",
+        )
 
-        # Save the figure
-        output_figure = "{}/fed{}-slot{}-oh{}-vfat{}.png".format(output_dir, fed, slot, oh, vfat)
-        dac_scan_fig.savefig(output_figure)
-        print("Scan plot saved to {}".format(output_figure))
+    # Save the figure
+    output_figure = "{}/fed{}-slot{}-oh{}-vfat{}.png".format(output_dir, fed, slot, oh, vfat)
+    dac_scan_fig.savefig(output_figure)
+    print("Scan plot saved to {}".format(output_figure))
 
-        # Delete the figure
-        # FIXME: Reuse the figure
-        plt.close(dac_scan_fig)
-        del dac_scan_fig
+    # Make sure to keep the memory usage reasonable
+    plt.close(dac_scan_fig)
+    gc.collect()
 
 
 def create_configuration(
@@ -225,6 +227,23 @@ def create_configuration(
 
         # Plot!
         os.makedirs("{}/plots".format(output_dir), exist_ok=True)
-        plot_scan(input_df, output_df, output_dir / "plots")
+
+        output_df = output_df.set_index(["fed", "slot", "oh", "vfat", "dac-name"]).sort_index()
+
+        pool = Pool(cpu_count())
+
+        for vfat_group in input_df.groupby(["fed", "slot", "oh", "vfat"]):
+            # one image per vfat
+            pool.apply_async(
+                plot_scan,
+                args=(
+                    vfat_group,
+                    output_df,
+                    output_dir / "plots",
+                ),
+            )
+
+        pool.close()
+        pool.join()
 
     return output_filename