Minor update to BW test handler

Work towards lhcb-dpa/project#268. Link to lhcb-datapkg/PRConfig!353 (merged)

handlers/BandwidthTestHandler.py

@@ -53,7 +53,7 @@ def make_prmon_plots(data, xvar, ylist, process):
     }
     xlabel = 'Wall-time'
     ylabel = 'Memory'
-    xunit = 'Sec'
+    xunit = 's'
     yunit = 'GB'
     xmultiplier = 1.0
     ymultiplier = 1.0 / 1024.0 / 1024.0 # Raw data is in kB
@@ -156,11 +156,12 @@ def make_comparison_page(dfs_for_comparison, ref_version, new_version):
             # Table will be too large; show only those that have changed
             comparison_df = comparison_df[comparison_df['Rate (kHz)']['Ref'] != comparison_df['Rate (kHz)']['New']]
 
-        # Apply highlighting
+        # Apply highlighting and formatting
         styler = comparison_df.style
         for column in columns_to_compare:
             styler = styler.applymap(highlight_vals, subset=[(column, 'Change (%)')])
-        styler = styler.format(precision=2)
+            styler = styler.format('{:.2f}', subset=[(column, 'Change (%)')])
+            styler = styler.format('{:.3g}', subset=[(column, 'New'), (column, 'Ref')])
 
         # Work out what to call the table
         if is_table_per_line:
@@ -239,6 +240,8 @@ def send_gitlab_feedback(
 
     tot_rate_change = (tot_rate_new - tot_rate_ref) / tot_rate_ref
     tot_bandwidth_change = (tot_bandwidth_new - tot_bandwidth_ref) / tot_bandwidth_ref
+    status_msg = status_msg.replace(':bluetick:', ':ballot_box_with_check:') # `bluetick` not supported on GitLab
+    status_msg = status_msg.replace(':alarm:', ':warning:')                  # `alarm` not supported on GitLab
 
     message = (
         f"Bandwidth test [{options}]({web_link_new}): "
@@ -278,7 +281,7 @@ def send_mattermost_feedback(n_low_rate, n_high_rate, tot_rate, tot_bandwidth, o
         f"[{options} {version}]({web_link}):\n"
         f"`Total rate = {tot_rate:.1f} kHz`. `Total bandwidth = {tot_bandwidth:.1f} GB/s`.\n"
         f"Lines with rate of 0 Hz: {n_low_rate}. "
-        f"Lines with rate > {rate_tolerance(process)} Hz: {n_high_rate}. "
+        f"Lines with rate > {rate_tolerance(process)} Hz: {n_high_rate}.\n"
     ) + status_msg
     log.info(
         f"Posting Mattermost feedback:\n message={message}"
@@ -335,8 +338,8 @@ class BandwidthTestHandler(BaseHandler):
         if os.path.exists(prmon_file) and os.path.exists(prmon_json_file):
             monitor_memory = True
             max_rss, max_pss = process_prmon_output(prmon_file, prmon_json_file, process=process)
-            max_rss = f'{max_rss}:.2f'
-            max_pss = f'{max_pss}:.2f'
+            max_rss = f'{max_rss:.2f}'
+            max_pss = f'{max_pss:.2f}'
         else:
             monitor_memory = False
             log.warning('No prmon output files found. Seems this test was not wrapped with `prmon` yet.')
@@ -347,6 +350,8 @@ class BandwidthTestHandler(BaseHandler):
             # for the index page, add comparison for lhcb-master-mr builds
             'comparison': compare_str if slot == 'lhcb-master-mr' else '',
             'dirname': dirname,
+            'start_time': startTime,
+            'end_time': endTime,
             'version': version,
             'platform': platform,
             'hostname': hostname,
@@ -417,7 +422,12 @@ class BandwidthTestHandler(BaseHandler):
         send_mattermost_feedback(n_low_rate, n_high_rate, tot_rate, tot_bandwidth, options, version, targetRootWebDir, status_sentence, process)
 
         # Post GitLab MR feedback; a bit more complex as comparing to a reference build.
-        if (slot == "lhcb-master-mr") and (
+        if (slot 
+            in [
+                "lhcb-master-mr",
+                "lhcb-master",
+            ]
+        ) and (
             options
             in [
                 "Moore_hlt2_bandwidth",
@@ -430,38 +440,36 @@ class BandwidthTestHandler(BaseHandler):
                 try:
 
                     dfs_for_comparison = {version_str(ref): {}, version_str(test): {}}
+                    ref_version = version_str(ref)
+                    new_version = version_str(test)
                     
                     if test == (slot, build_id):
                         # The current build is the *test*, not the reference build
-                        new_version = version_str(test)
-                        for csv_name in csv_rate_table_names:
-                            log.info(f"Trying comparison with {csv_name}.")
-                            dfs_for_comparison[new_version][csv_name] = pd.read_csv(full_output_path(csv_name))
-
                         web_link_new = targetRootWebDir
                         dirname_new = dirname
                         tot_rate_new = tot_rate
                         tot_bandwidth_new = tot_bandwidth
                     
                         # The build we're comparing to is therefore the reference build
-                        ref_version = version_str(ref)
                         dfs_for_comparison[ref_version], web_link_ref, _, tot_rate_ref, tot_bandwidth_ref = get_info_from_comparison_build(ref_version, options, csv_rate_table_names)
 
-                    elif ref == (slot, build_id):
-                        # The current build is the *reference*, not the test build
-                        ref_version = version_str(test)
                         for csv_name in csv_rate_table_names:
                             log.info(f"Trying comparison with {csv_name}.")
-                            dfs_for_comparison[ref_version][csv_name] = pd.read_csv(full_output_path(csv_name))
+                            dfs_for_comparison[new_version][csv_name] = pd.read_csv(full_output_path(csv_name))
 
+                    elif ref == (slot, build_id):
+                        # The current build is the *reference*, not the test build
                         web_link_ref = targetRootWebDir
                         tot_rate_ref = tot_rate
                         tot_bandwidth_ref = tot_bandwidth
 
                         # The build we're comparing to is therefore the test build
-                        new_version = version_str(ref)
                         dfs_for_comparison[new_version], web_link_new, dirname_new, tot_rate_new, tot_bandwidth_new = get_info_from_comparison_build(new_version, options, csv_rate_table_names)
 
+                        for csv_name in csv_rate_table_names:
+                            log.info(f"Trying comparison with {csv_name}.")
+                            dfs_for_comparison[ref_version][csv_name] = pd.read_csv(full_output_path(csv_name))
+
                     else:
                         assert False
                 except dashboard.ResourceNotFound: