From c5663bbd65e29a2ddd1554227442be4731fa0a98 Mon Sep 17 00:00:00 2001
From: amete <serhanmete@gmail.com>
Date: Mon, 8 Jun 2020 12:07:12 +0200
Subject: [PATCH 1/7] Code clean-up for PerfMonMT

---
 .../PerfMonComps/python/PerfMonFlags.py       |   2 +-
 .../share/PerfMonMTSvc_jobOptions.py          |   2 +-
 .../PerfMonComps/src/PerfMonMTSvc.cxx         |  39 +++--
 .../PerfMonComps/src/PerfMonMTUtils.h         | 136 +++++++++---------
 4 files changed, 87 insertions(+), 92 deletions(-)

diff --git a/Control/PerformanceMonitoring/PerfMonComps/python/PerfMonFlags.py b/Control/PerformanceMonitoring/PerfMonComps/python/PerfMonFlags.py
index 5df1cf262176..6dddf2c60f60 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/python/PerfMonFlags.py
+++ b/Control/PerformanceMonitoring/PerfMonComps/python/PerfMonFlags.py
@@ -136,7 +136,7 @@ class doMonitoringMT(JobProperty):
         # Setup PerfMonAlg
         from AthenaCommon.AlgSequence import AthSequencer
         topSequence = AthSequencer("AthAlgSeq")
-        if not hasattr(topSequence, "PerfMonMTSvcAlg"):
+        if not hasattr(topSequence, "PerfMonMTAlg"):
             from PerfMonComps.PerfMonCompsConf import PerfMonMTAlg
             topSequence += PerfMonMTAlg("PerfMonMTAlg")
         return
diff --git a/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_jobOptions.py b/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_jobOptions.py
index dcc0c03ecc1d..014484ca0f0a 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_jobOptions.py
+++ b/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_jobOptions.py
@@ -40,7 +40,7 @@ if not hasattr(svcMgr, 'PerfMonMTSvc'):
 ###############################
 from AthenaCommon.AlgSequence import AthSequencer
 topSequence = AthSequencer("AthAlgSeq")
-if not hasattr(topSequence, "PerfMonMTSvcAlg"):
+if not hasattr(topSequence, "PerfMonMTAlg"):
     from PerfMonComps.PerfMonCompsConf import PerfMonMTAlg
     topSequence += PerfMonMTAlg("PerfMonMTAlg")
     pass
diff --git a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.cxx b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.cxx
index cd52c4c97ace..6257a364028f 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.cxx
+++ b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.cxx
@@ -16,8 +16,6 @@
 // STD includes
 #include <algorithm>
 
-using json = nlohmann::json;  // for convenience
-
 /*
  * Constructor
  */
@@ -60,6 +58,12 @@ StatusCode PerfMonMTSvc::initialize() {
   // Print where we are
   ATH_MSG_INFO("Initializing " << name());
 
+  // Check if /proc exists, if not memory statistics are not available
+  const bool procExists = PMonMT::doesDirectoryExist("/proc");
+  if(!procExists) {
+    ATH_MSG_INFO("The system doesn't support /proc. Therefore, memory measurements are not available");
+  }
+
   // Print some information minimal information about our configuration
   ATH_MSG_INFO("Service is configured for [" << m_numberOfThreads.toString() << "] threads " <<
                "analyzing [" << m_numberOfSlots.toString() << "] events concurrently");
@@ -252,7 +256,7 @@ void PerfMonMTSvc::eventLevelMon() {
   if (m_doEventLoopMonitoring) {
     if (isCheckPoint()) {
       // Capture
-      m_measurement_events.capture_event(m_eventCounter);
+      m_measurement_events.capture_event();
       m_eventLevelData.record_event(m_measurement_events, m_eventCounter);
       // Report instantly - no more than m_eventLoopMsgLimit times
       if(m_eventLoopMsgCounter < m_eventLoopMsgLimit) {
@@ -308,19 +312,13 @@ void PerfMonMTSvc::report2Log() {
   // Header
   report2Log_Description();
 
-  // Detailed tables
-  const bool procExists = doesDirectoryExist("/proc");
-  if (!procExists) {
-    ATH_MSG_INFO("There is no /proc directory in this system, therefore memory monitoring has failed!");
-  }
-
   // Component-level
-  if (m_printDetailedTables && procExists && m_doComponentLevelMonitoring) {
+  if (m_printDetailedTables && m_doComponentLevelMonitoring) {
     report2Log_ComponentLevel();
   }
 
   // Event-level
-  if (m_printDetailedTables && procExists && m_doEventLoopMonitoring) {
+  if (m_printDetailedTables && m_doEventLoopMonitoring) {
     report2Log_EventLevel();
   }
 
@@ -507,21 +505,17 @@ void PerfMonMTSvc::report2Log_CpuInfo() const {
  * Report data to JSON
  */
 void PerfMonMTSvc::report2JsonFile() {
-  json j;
+  nlohmann::json j;
 
   // CPU and Wall-time
   report2JsonFile_Summary(j);  // Snapshots
 
   // Memory
-  const bool procExists = doesDirectoryExist("/proc");
-
-  if (procExists) {
-    if (m_doComponentLevelMonitoring) {
-      report2JsonFile_ComponentLevel(j);  // Component-level
-    }
-    if (m_doEventLoopMonitoring) {
-      report2JsonFile_EventLevel(j);  // Event-level
-    }
+  if (m_doComponentLevelMonitoring) {
+    report2JsonFile_ComponentLevel(j);  // Component-level
+  }
+  if (m_doEventLoopMonitoring) {
+    report2JsonFile_EventLevel(j);  // Event-level
   }
 
   // Write
@@ -746,6 +740,9 @@ std::string PerfMonMTSvc::scaleMem(long memMeas) const {
   return stringObj;
 }
 
+/*
+ * Collect some hardware information
+ */
 std::string PerfMonMTSvc::get_cpu_model_info() const {
   std::string cpu_model;
 
diff --git a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h
index 1436fbf007cc..819c8c7bd3eb 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h
+++ b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h
@@ -24,7 +24,6 @@ typedef std::map<std::string, long> memory_map_t;  // Component : Memory Measure
  * Inline function prototypes
  */
 inline memory_map_t operator-(memory_map_t& map1, memory_map_t& map2);
-inline bool doesDirectoryExist(const std::string dir);
 
 /*
  * Necessary tools
@@ -39,6 +38,8 @@ memory_map_t get_mem_stats();
 // Efficient memory measurements
 double get_malloc();
 double get_vmem();
+// Simple check if directory exists
+bool doesDirectoryExist(const std::string dir);
 
 // Step name and Component name pairs. Ex: Initialize - StoreGateSvc
 struct StepComp {
@@ -53,16 +54,12 @@ struct StepComp {
 
 // Basic Measurement
 struct Measurement {
-  typedef std::map<int, Measurement> event_meas_map_t;  // Event number: Measurement
 
   // Variables to store measurements
   double cpu_time, wall_time; // Timing
   memory_map_t mem_stats; // Memory: Vmem, Rss, Pss, Swap
   double vmem, malloc; // Memory: Vmem, Malloc (faster than above)
 
-  // Event level measurements
-  event_meas_map_t eventLevel_meas_map;  // [Event count so far]: Measurement
-
   // Peak values for Vmem, Rss and Pss
   long vmemPeak = LONG_MIN;
   long rssPeak = LONG_MIN;
@@ -75,9 +72,7 @@ struct Measurement {
     wall_time = get_wall_time();
 
     // Memory 
-    if (doesDirectoryExist("/proc")) {
-      mem_stats = get_mem_stats();
-    }
+    mem_stats = get_mem_stats();
   }
 
   // Capture component-level measurements
@@ -91,32 +86,21 @@ struct Measurement {
 
     // Efficient Memory Measurements
     malloc = get_malloc();
-    if (doesDirectoryExist("/proc")) {
-      vmem = get_vmem();
-    }
+    vmem = get_vmem();
   }
 
   // Capture event-level measurements 
-  void capture_event(int eventCount) {
+  void capture_event() {
     // Timing 
     cpu_time = get_process_cpu_time();
     wall_time = get_wall_time();
-    Measurement meas;
-
-    if (doesDirectoryExist("/proc")) {
-      mem_stats = get_mem_stats();
-      meas.mem_stats = mem_stats;
-
-      if (mem_stats["vmem"] > vmemPeak) vmemPeak = mem_stats["vmem"];
-      if (mem_stats["rss"] > rssPeak) rssPeak = mem_stats["rss"];
-      if (mem_stats["pss"] > pssPeak) pssPeak = mem_stats["pss"];
-    }
 
-    meas.cpu_time = cpu_time;
-    meas.wall_time = wall_time;
+    // Memory
+    mem_stats = get_mem_stats();
 
-    // Capture for event level measurements
-    eventLevel_meas_map[eventCount] = meas;
+    if (mem_stats["vmem"] > vmemPeak) vmemPeak = mem_stats["vmem"];
+    if (mem_stats["rss"] > rssPeak) rssPeak = mem_stats["rss"];
+    if (mem_stats["pss"] > pssPeak) pssPeak = mem_stats["pss"];
   }
 
   Measurement() : cpu_time{0.}, wall_time{0.}, vmem{0.}, malloc{0.} {
@@ -149,7 +133,7 @@ struct MeasurementData {
     m_tmp_wall = meas.wall_time;
 
     // Non-efficient memory measurements
-    if (doesDirectoryExist("/proc")) m_memMon_tmp_map = meas.mem_stats;
+    m_memMon_tmp_map = meas.mem_stats;
   }
 
   // [Component Level Monitoring - Serial Steps] : Record the measurement for the current state
@@ -158,7 +142,7 @@ struct MeasurementData {
     m_delta_wall = meas.wall_time - m_tmp_wall;
 
     // Non-efficient memory measurements
-    if (doesDirectoryExist("/proc")) m_memMon_delta_map = meas.mem_stats - m_memMon_tmp_map;
+    m_memMon_delta_map = meas.mem_stats - m_memMon_tmp_map;
   }
 
   // [Component Level Monitoring] : Start
@@ -172,7 +156,7 @@ struct MeasurementData {
 
     // Efficient memory measurements
     m_tmp_malloc = meas.malloc;
-    if (doesDirectoryExist("/proc")) m_tmp_vmem = meas.vmem;
+    m_tmp_vmem = meas.vmem;
   } 
 
   // [Component Level Monitoring] : Stop
@@ -190,20 +174,17 @@ struct MeasurementData {
 
     // Efficient memory measurements
     m_delta_malloc += meas.malloc - m_tmp_malloc;
-    if (doesDirectoryExist("/proc")) m_delta_vmem += meas.vmem - m_tmp_vmem;
+    m_delta_vmem += meas.vmem - m_tmp_vmem;
   } 
 
   // [Event Level Monitoring - Parallel Steps] : Record the measurement for the current checkpoint
-  void record_event(Measurement& meas, int eventCount) {
-    m_eventLevel_delta_map[eventCount].cpu_time = meas.eventLevel_meas_map[eventCount].cpu_time;
-    m_eventLevel_delta_map[eventCount].wall_time = meas.eventLevel_meas_map[eventCount].wall_time - m_offset_wall;
-
-    if (doesDirectoryExist("/proc")) {
-      m_eventLevel_delta_map[eventCount].mem_stats["vmem"] = meas.eventLevel_meas_map[eventCount].mem_stats["vmem"];
-      m_eventLevel_delta_map[eventCount].mem_stats["rss"] = meas.eventLevel_meas_map[eventCount].mem_stats["rss"];
-      m_eventLevel_delta_map[eventCount].mem_stats["pss"] = meas.eventLevel_meas_map[eventCount].mem_stats["pss"];
-      m_eventLevel_delta_map[eventCount].mem_stats["swap"] = meas.eventLevel_meas_map[eventCount].mem_stats["swap"];
-    }
+  void record_event(const Measurement& meas, int eventCount) {
+    // Timing
+    m_eventLevel_delta_map[eventCount].cpu_time = meas.cpu_time;
+    m_eventLevel_delta_map[eventCount].wall_time = meas.wall_time - m_offset_wall;
+
+    // Memory
+    m_eventLevel_delta_map[eventCount].mem_stats = meas.mem_stats;
   }
 
   void set_wall_time_offset(double wall_time_offset) { m_offset_wall = wall_time_offset; }
@@ -298,48 +279,62 @@ inline double PMonMT::get_wall_time() {
 
 // Read from proc's smaps file. It is costly to do this operation too often.
 inline memory_map_t PMonMT::get_mem_stats() {
+  // Result object
   memory_map_t result;
-  std::string fileName = "/proc/self/smaps";
-  std::ifstream smaps_file(fileName);
-
-  std::string line;
-  std::string key;
-  std::string value;
 
-  while (getline(smaps_file, line)) {
-    std::stringstream ss(line);
-    ss >> key >> value;
-
-    if (key == "Size:") {
-      result["vmem"] += stol(value);
-    }
-    if (key == "Rss:") {
-      result["rss"] += stol(value);
-    }
-    if (key == "Pss:") {
-      result["pss"] += stol(value);
-    }
-    if (key == "Swap:") {
-      result["swap"] += stol(value);
+  // Zero initialize
+  for(auto& stat : result) stat.second = 0;
+
+  // This is the input where we read the stats from
+  static const std::string fileName = "/proc/self/smaps";
+  std::ifstream smaps_file{fileName};
+
+  std::string line{}, key{}, value{};
+
+  // Loop over the file
+  while (smaps_file) {
+    // Read interesting key value pairs
+    smaps_file >> key >> value;
+    smaps_file.ignore(std::numeric_limits<std::streamsize>::max(),'\n');
+
+    if(smaps_file) {
+      if (key == "Size:") {
+        result["vmem"] += std::stol(value);
+      }
+      if (key == "Rss:") {
+        result["rss"] += std::stol(value);
+      }
+      if (key == "Pss:") {
+        result["pss"] += std::stol(value);
+      }
+      if (key == "Swap:") {
+        result["swap"] += std::stol(value);
+      }
     }
   }
+
   return result;
 }
 
 // This operation is less costly than the previous one. Since statm is a much smaller file compared to smaps.
 inline double PMonMT::get_vmem() {
-  const std::string fileName = "/proc/self/statm";
-  std::ifstream statm_file(fileName);
+  // Result
+  double result = 0.;
 
-  std::string vmem_in_pages;  // vmem measured in pages
-  std::string line;
+  // This is where we read the stats from
+  static const std::string fileName = "/proc/self/statm";
+  std::ifstream statm_file{fileName};
+
+  std::string vmem_in_pages{}, line{};  // vmem measured in pages
+
+  // We simply get the first line
   if (getline(statm_file, line)) {
-    std::stringstream ss(line);
+    std::stringstream ss{line};
     ss >> vmem_in_pages;  // The first number in this file is the vmem measured in pages
   }
 
-  const double page_size = sysconf(_SC_PAGESIZE) / 1024.0;  // page size in KB
-  const double result = stod(vmem_in_pages) * page_size;
+  static const double page_size = sysconf(_SC_PAGESIZE) / 1024.0;  // page size in KB
+  result = std::stod(vmem_in_pages) * page_size;
 
   return result;
 }
@@ -391,7 +386,10 @@ inline memory_map_t operator-(memory_map_t& map1, memory_map_t& map2) {
   return result_map;
 }
 
-inline bool doesDirectoryExist(const std::string dir) {
+/*
+ * Simple check if a given directory exists
+ */
+inline bool PMonMT::doesDirectoryExist(const std::string dir) {
   struct stat buffer;
   return (stat(dir.c_str(), &buffer) == 0);
 }
-- 
GitLab


From 5110a317a78989b2e9704c2d36ca2355e63fd095 Mon Sep 17 00:00:00 2001
From: amete <serhanmete@gmail.com>
Date: Mon, 8 Jun 2020 14:09:45 +0200
Subject: [PATCH 2/7] Delete the obselete plotting script that now lives under
 PerfMonAna

---
 .../share/PerfMonMTSvc_plotter.py             | 165 ------------------
 1 file changed, 165 deletions(-)
 delete mode 100644 Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_plotter.py

diff --git a/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_plotter.py b/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_plotter.py
deleted file mode 100644
index c2264f07ad88..000000000000
--- a/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_plotter.py
+++ /dev/null
@@ -1,165 +0,0 @@
-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
-
-# @author: Hasan Ozturk <haozturk@cern.ch>
-
-
-__author__  = "Hasan Ozturk <haozturk@cern.ch"
-__doc__     = "A python module which parses the PerfMonMTSvc results and makes plots"
-
-
-import sys
-import json
-
-import matplotlib
-matplotlib.use('PDF') # use PDF backend
-import matplotlib.pyplot as plt
-import numpy as np
-
-import operator
-
-if ( len(sys.argv) != 2 ):
-  print("Please give result file as an argument!")
-
-# Get the result Json file
-result_file = sys.argv[1] 
-
-
-
-def snapshot_plotter(snapshot_data, plotname):
-  
-
-  snapshot_steps = [ 'Finalize', 'Execute','Initialize']
-  #snapshot_steps = []
-  snapshot_cpu_times = []
-  snapshot_wall_times = []
-
-  # Get rid of code duplication
-  cpu_time = snapshot_data['Finalize']['cpu_time']
-  wall_time = snapshot_data['Finalize']['wall_time']
-  snapshot_cpu_times.append(cpu_time)
-  snapshot_wall_times.append(wall_time)
-
-  cpu_time = snapshot_data['Event_loop']['cpu_time']
-  wall_time = snapshot_data['Event_loop']['wall_time']
-  snapshot_cpu_times.append(cpu_time)
-  snapshot_wall_times.append(wall_time)
-
-  cpu_time = snapshot_data['Initialize']['cpu_time']
-  wall_time = snapshot_data['Initialize']['wall_time']
-  snapshot_cpu_times.append(cpu_time)
-  snapshot_wall_times.append(wall_time)
-
-  ind = np.arange(len(snapshot_steps))
-  width = 0.35
-     
-  fig, ax = plt.subplots()
-
-  rects1 = ax.barh(ind - width/2, snapshot_cpu_times, width, label = 'CPU Time') 
-  rects2 = ax.barh(ind + width/2, snapshot_wall_times, width, label = 'Wall Time') 
-  ax.set_xlabel('Time(ms)')
-  ax.set_ylabel('Steps')
-  ax.set_title('Snapshot Level Monitoring')
-  ax.set_yticks(ind)
-  ax.set_yticklabels(snapshot_steps)
-  ax.legend()
-
-  fig.set_tight_layout( True )
-  fig.savefig(plotname) 
-
-def comp_plotter(complevel_data, plotname):
-  # Plot Component Level Monitoring
- 
-  #fig = plt.figure(figsize=(31,150))
-  fig = plt.figure(figsize=(50,150))
-  stepNum = len(complevel_data)
- 
-  measurement_threshold = 5 # 5 ms
-
-  for i, step in enumerate(complevel_data):
-
-    components = []
-    cpu_times = []
-    wall_times = []
-    
-    for component in complevel_data[step]:
-      cpu_time = complevel_data[step][component]['cpu_time']
-      wall_time = complevel_data[step][component]['wall_time']
-      
-      # Only take components whose measurements are higher a certain threshold
-      if cpu_time + wall_time > measurement_threshold:
-        components.append(component)   
-        cpu_times.append(cpu_time) # Clear!
-        wall_times.append(wall_time)
-
-    # Sort the components 
-
-    # Prepare the necessary data structures for sorting ( could be more efficient!  )
-    cpu_wall_tuple_list = zip(cpu_times, wall_times)
-    comp_dict = dict( zip(components, cpu_wall_tuple_list) )
-    sortby_list = [ cpu + wall for cpu, wall in zip(cpu_times, wall_times)]
-    sort_dict = dict(zip(components, sortby_list))
-    
-    # Sort the components according to wall_time + cpu time
-    sorted_comp_tuple_list = sorted(sort_dict.items() , key = operator.itemgetter(1))
-
-    sorted_components = []
-    sorted_cpu_times = []
-    sorted_wall_times = []
- 
-    # Fill the necessary lists for plotting
-    for idx in sorted_comp_tuple_list:
-      curr_comp = idx[0]
-      curr_cpu = comp_dict[curr_comp][0]
-      curr_wall = comp_dict[curr_comp][1]
-
-      sorted_components.append(curr_comp)
-      sorted_cpu_times.append(curr_cpu)
-      sorted_wall_times.append(curr_wall)
-    
-    # if there is no nonzero measurement in the step, then skip it
-    if len(sorted_components) == 0:
-      continue
-
-    # Horizontal Bar Chart
-    ax = fig.add_subplot(stepNum,1,i+1)
-
-    index = np.arange(len(sorted_components))
-    bar_width = 0.35
-    opacity = 0.8
-
-    rects1 = plt.barh(index + (1.5)*bar_width, sorted_cpu_times,bar_width,
-    alpha=opacity,
-    label='CPU Time')
-
-    rects2 = plt.barh(index + bar_width/2, sorted_wall_times, bar_width,
-    alpha=opacity,
-    label='Wall Time')
-
-    plt.ylabel('Components',fontsize = 35)
-    plt.xlabel('Time(ms)', fontsize = 35)
-    plt.title(step, fontsize = 40, fontweight = "bold")
-    plt.yticks(index + bar_width, sorted_components)
-    plt.legend(prop={'size': 30})
-    
-    ax.tick_params(axis='both', which='major', labelsize=30)
-    ax.tick_params(axis='both', which='minor', labelsize=30)
-
-
-    fig.set_tight_layout( True )
-    
-   
-  fig.savefig(plotname)
-
-
-with open( result_file ) as json_file:
-  data = json.load(json_file)
-
-  snapshot_data = data['Snapshot_level']
-  snapshot_plotter(snapshot_data, 'snapshot_level.pdf')
-  
-  serial_complevel_data = data['Serial_Component_level']
-  comp_plotter(serial_complevel_data, 'serial_complevel.pdf')
-
-  parallel_complevel_data = data['Parallel_Component_level']
-  comp_plotter(parallel_complevel_data, 'parallel_complevel.pdf')
-
-- 
GitLab


From 3ebf55c739e6541b6c2092679818f4317539625a Mon Sep 17 00:00:00 2001
From: amete <serhanmete@gmail.com>
Date: Mon, 8 Jun 2020 16:34:45 +0200
Subject: [PATCH 3/7] Remove redundant lines from PerfMonMTSvc_jobOptions.py,
 modify the event-loop check-point-factor in MTJobOptCfg.py

---
 .../PerfMonComps/python/MTJobOptCfg.py        |  2 +-
 .../share/PerfMonMTSvc_jobOptions.py          | 23 +------------------
 2 files changed, 2 insertions(+), 23 deletions(-)

diff --git a/Control/PerformanceMonitoring/PerfMonComps/python/MTJobOptCfg.py b/Control/PerformanceMonitoring/PerfMonComps/python/MTJobOptCfg.py
index 58650844c3ce..3c5c77584d2f 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/python/MTJobOptCfg.py
+++ b/Control/PerformanceMonitoring/PerfMonComps/python/MTJobOptCfg.py
@@ -48,7 +48,7 @@ class PerfMonMTSvc ( _PerfMonMTSvc  ):
 
         ## Set the monitoring check points
         from AthenaCommon.ConcurrencyFlags import jobproperties as jp
-        handle.checkPointFactor = max(10,jp.ConcurrencyFlags.NumThreads())
+        handle.checkPointFactor = 10*max(1,jp.ConcurrencyFlags.NumThreads())
         handle.numberOfThreads = max(1,jp.ConcurrencyFlags.NumThreads())
         handle.numberOfSlots = max(1,jp.ConcurrencyFlags.NumConcurrentEvents())
 
diff --git a/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_jobOptions.py b/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_jobOptions.py
index 014484ca0f0a..d8505ceed310 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_jobOptions.py
+++ b/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_jobOptions.py
@@ -3,37 +3,16 @@
 ###############################
 from AthenaCommon.Logging import logging
 from AthenaCommon.AppMgr import ServiceMgr as svcMgr
-from AthenaCommon.ConcurrencyFlags import jobproperties as jp
-import os,psutil
 
-log = logging.getLogger("PerfMonMTSvc_jobOptions.py")
+log = logging.getLogger("PerfMonMT")
 log.info("Setting up PerfMonMT...")
 
-
 ###############################
 # Load PerfMonMTSvc
 ###############################
 if not hasattr(svcMgr, 'PerfMonMTSvc'):
     from PerfMonComps.MTJobOptCfg import PerfMonMTSvc
     svcMgr += PerfMonMTSvc("PerfMonMTSvc")
-    # Report results into json by default
-    svcMgr.PerfMonMTSvc.reportResultsToJSON = True
-    # Enable event loop monitoring by default
-    svcMgr.PerfMonMTSvc.doEventLoopMonitoring = True
-    # Disable component level monitoring by default
-    svcMgr.PerfMonMTSvc.doComponentLevelMonitoring = False
-    # Enable detailed table printing by default
-    svcMgr.PerfMonMTSvc.printDetailedTables = True
-    # Print only the top 50 components (sorted by CPU time) by default
-    svcMgr.PerfMonMTSvc.printNComps = 50
-    # Configure the check point sequence in the event loop monitoring.
-    # By default common difference is the number of threads with which the job is running
-    svcMgr.PerfMonMTSvc.checkPointType = "Arithmetic" 
-    svcMgr.PerfMonMTSvc.checkPointFactor = max(10,jp.ConcurrencyFlags.NumThreads())
-    svcMgr.PerfMonMTSvc.wallTimeOffset = psutil.Process(os.getpid()).create_time() * 1000 # Get the job start time in ms
-    # Set number of threads/slots
-    svcMgr.PerfMonMTSvc.numberOfThreads = max(1,jp.ConcurrencyFlags.NumThreads())
-    svcMgr.PerfMonMTSvc.numberOfSlots = max(1,jp.ConcurrencyFlags.NumConcurrentEvents())
 
 ###############################
 # Load PerfMonMTAlg
-- 
GitLab


From 4ce72b7921ad82fecfe32723160213363dd1879d Mon Sep 17 00:00:00 2001
From: amete <serhanmete@gmail.com>
Date: Mon, 8 Jun 2020 18:01:52 +0200
Subject: [PATCH 4/7] Add some information to PerfMonMTUtils.h

---
 .../PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h
index 819c8c7bd3eb..c254a7a7bf58 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h
+++ b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h
@@ -278,6 +278,10 @@ inline double PMonMT::get_wall_time() {
  */
 
 // Read from proc's smaps file. It is costly to do this operation too often.
+// In a realistic RAWtoESD job, the smaps for the the whole application can get large.
+// Therefore, this operation might take about 100 ms per call, which is fairly substantial.
+// However, this is one of the most reliable way to get PSS.
+// Therefore, keep it as is but don't call it too often!
 inline memory_map_t PMonMT::get_mem_stats() {
   // Result object
   memory_map_t result;
-- 
GitLab


From e0d53e9f1c1d59927dd74584fd44a71181830532 Mon Sep 17 00:00:00 2001
From: amete <serhanmete@gmail.com>
Date: Mon, 8 Jun 2020 21:32:25 +0200
Subject: [PATCH 5/7] Reorganize event loop monitoring a bit

---
 .../PerfMonComps/python/MTJobOptCfg.py        |  1 -
 .../PerfMonComps/src/PerfMonMTSvc.cxx         | 35 +++++++++++--------
 .../PerfMonComps/src/PerfMonMTSvc.h           |  2 +-
 3 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/Control/PerformanceMonitoring/PerfMonComps/python/MTJobOptCfg.py b/Control/PerformanceMonitoring/PerfMonComps/python/MTJobOptCfg.py
index 3c5c77584d2f..c3a812866a8f 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/python/MTJobOptCfg.py
+++ b/Control/PerformanceMonitoring/PerfMonComps/python/MTJobOptCfg.py
@@ -48,7 +48,6 @@ class PerfMonMTSvc ( _PerfMonMTSvc  ):
 
         ## Set the monitoring check points
         from AthenaCommon.ConcurrencyFlags import jobproperties as jp
-        handle.checkPointFactor = 10*max(1,jp.ConcurrencyFlags.NumThreads())
         handle.numberOfThreads = max(1,jp.ConcurrencyFlags.NumThreads())
         handle.numberOfSlots = max(1,jp.ConcurrencyFlags.NumConcurrentEvents())
 
diff --git a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.cxx b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.cxx
index 6257a364028f..1d2b36704f7a 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.cxx
+++ b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.cxx
@@ -252,20 +252,20 @@ void PerfMonMTSvc::eventLevelMon() {
   // Lock for data integrity
   std::lock_guard<std::mutex> lock(m_mutex_capture);
 
-  // If enabled, do event level monitoring
-  if (m_doEventLoopMonitoring) {
-    if (isCheckPoint()) {
-      // Capture
-      m_measurement_events.capture_event();
-      m_eventLevelData.record_event(m_measurement_events, m_eventCounter);
-      // Report instantly - no more than m_eventLoopMsgLimit times
-      if(m_eventLoopMsgCounter < m_eventLoopMsgLimit) {
-        report2Log_EventLevel_instant();
-        m_eventLoopMsgCounter++;
-      }
+  // Increment the internal counter
+  incrementEventCounter();
+
+  // Monitor
+  if (m_doEventLoopMonitoring && isCheckPoint()) {
+    // Capture
+    m_measurement_events.capture_event();
+    m_eventLevelData.record_event(m_measurement_events, m_eventCounter);
+    // Report instantly - no more than m_eventLoopMsgLimit times
+    if(m_eventLoopMsgCounter < m_eventLoopMsgLimit) {
+      report2Log_EventLevel_instant();
+      m_eventLoopMsgCounter++;
     }
   }
-  incrementEventCounter();
 }
 
 /*
@@ -278,10 +278,17 @@ void PerfMonMTSvc::incrementEventCounter() { m_eventCounter++; }
  * Is it event-level monitoring check point yet?
  */
 bool PerfMonMTSvc::isCheckPoint() {
+  // Always check 1, 10, 25 for short tests
+  if (m_eventCounter == 1 || m_eventCounter == 10 || m_eventCounter == 25)
+    return true;
+
+  // Check the user settings
   if (m_checkPointType == "Arithmetic")
     return (m_eventCounter % m_checkPointFactor == 0);
-  else
+  else if (m_checkPointType == "Geometric")
     return isPower(m_eventCounter, m_checkPointFactor);
+  else
+    return false;
 }
 
 /*
@@ -432,7 +439,7 @@ void PerfMonMTSvc::report2Log_EventLevel() {
       m_eventLoopMsgCounter++;
     }
     // Add to leak estimate
-    if (it.first >= std::max(uint64_t(10), uint64_t(m_checkPointFactor))) {
+    if (it.first >= 10) {
       m_fit_vmem.addPoint(it.first, it.second.mem_stats.at("vmem"));
       m_fit_pss.addPoint(it.first, it.second.mem_stats.at("pss"));
     }
diff --git a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.h b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.h
index 1981f195d1e4..37ed3e7bf4d6 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.h
+++ b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.h
@@ -136,7 +136,7 @@ class PerfMonMTSvc : virtual public IPerfMonMTSvc, public AthService {
       "Type of the check point sequence: Arithmetic(0, k, 2k...) or Geometric(0,k,k^2...)."};
   /// Frequency of event level monitoring
   Gaudi::Property<uint64_t> m_checkPointFactor{
-      this, "checkPointFactor", 10,
+      this, "checkPointFactor", 50,
       "Common difference if check point sequence is arithmetic, Common ratio if it is Geometric."};
   /// Offset for the wall-time, comes from configuration
   Gaudi::Property<double> m_wallTimeOffset{this, "wallTimeOffset", 0, "Job start wall time in miliseconds."};
-- 
GitLab


From 03b334425d12e43b85fdad635a3e409fb6b4ea88 Mon Sep 17 00:00:00 2001
From: amete <serhanmete@gmail.com>
Date: Mon, 8 Jun 2020 23:49:34 +0200
Subject: [PATCH 6/7] Update leak estimate and zero initialization of memory
 map

---
 Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.cxx | 2 +-
 Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.cxx b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.cxx
index 1d2b36704f7a..8a812befddc3 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.cxx
+++ b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.cxx
@@ -439,7 +439,7 @@ void PerfMonMTSvc::report2Log_EventLevel() {
       m_eventLoopMsgCounter++;
     }
     // Add to leak estimate
-    if (it.first >= 10) {
+    if (it.first >= 25) {
       m_fit_vmem.addPoint(it.first, it.second.mem_stats.at("vmem"));
       m_fit_pss.addPoint(it.first, it.second.mem_stats.at("pss"));
     }
diff --git a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h
index c254a7a7bf58..2b592779bd74 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h
+++ b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTUtils.h
@@ -287,7 +287,7 @@ inline memory_map_t PMonMT::get_mem_stats() {
   memory_map_t result;
 
   // Zero initialize
-  for(auto& stat : result) stat.second = 0;
+  result["vmem"] = result["rss"] = result["pss"] = result["swap"] = 0;
 
   // This is the input where we read the stats from
   static const std::string fileName = "/proc/self/smaps";
-- 
GitLab


From 8d9d981879265f981b578e85264ce3fc95ce8734 Mon Sep 17 00:00:00 2001
From: amete <serhanmete@gmail.com>
Date: Tue, 9 Jun 2020 10:31:37 +0200
Subject: [PATCH 7/7] Re-arrange the default configuration a bit

---
 .../PerfMonComps/python/MTJobOptCfg.py                     | 3 ---
 .../PerfMonComps/share/PerfMonMTSvc_jobOptions.py          | 7 +++++++
 .../PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.h  | 2 +-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/Control/PerformanceMonitoring/PerfMonComps/python/MTJobOptCfg.py b/Control/PerformanceMonitoring/PerfMonComps/python/MTJobOptCfg.py
index c3a812866a8f..35b25d9ba707 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/python/MTJobOptCfg.py
+++ b/Control/PerformanceMonitoring/PerfMonComps/python/MTJobOptCfg.py
@@ -60,9 +60,6 @@ class PerfMonMTSvc ( _PerfMonMTSvc  ):
         if jobproperties.PerfMonFlags.doFullMonMT():
             handle.doComponentLevelMonitoring = True
 
-        ## Turn on JSON reporting
-        handle.reportResultsToJSON = True
-
         return
 
     pass # class PerfMonMTSvc
diff --git a/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_jobOptions.py b/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_jobOptions.py
index d8505ceed310..3936f4053b11 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_jobOptions.py
+++ b/Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_jobOptions.py
@@ -3,6 +3,8 @@
 ###############################
 from AthenaCommon.Logging import logging
 from AthenaCommon.AppMgr import ServiceMgr as svcMgr
+from AthenaCommon.ConcurrencyFlags import jobproperties as jp
+import os,psutil
 
 log = logging.getLogger("PerfMonMT")
 log.info("Setting up PerfMonMT...")
@@ -13,6 +15,11 @@ log.info("Setting up PerfMonMT...")
 if not hasattr(svcMgr, 'PerfMonMTSvc'):
     from PerfMonComps.MTJobOptCfg import PerfMonMTSvc
     svcMgr += PerfMonMTSvc("PerfMonMTSvc")
+    # Set the job start time
+    svcMgr.PerfMonMTSvc.wallTimeOffset = psutil.Process(os.getpid()).create_time() * 1000 # Get the job start time in ms
+    # Set number of threads/slots
+    svcMgr.PerfMonMTSvc.numberOfThreads = max(1,jp.ConcurrencyFlags.NumThreads())
+    svcMgr.PerfMonMTSvc.numberOfSlots = max(1,jp.ConcurrencyFlags.NumConcurrentEvents())
 
 ###############################
 # Load PerfMonMTAlg
diff --git a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.h b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.h
index 37ed3e7bf4d6..5ff69c0e2f51 100644
--- a/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.h
+++ b/Control/PerformanceMonitoring/PerfMonComps/src/PerfMonMTSvc.h
@@ -123,7 +123,7 @@ class PerfMonMTSvc : virtual public IPerfMonMTSvc, public AthService {
       "True if component level monitoring is enabled, false o/w. Component monitoring may cause a decrease in the "
       "performance due to the usage of locks."};
   /// Report results to JSON
-  Gaudi::Property<bool> m_reportResultsToJSON{this, "reportResultsToJSON", false, "Report results into the json file."};
+  Gaudi::Property<bool> m_reportResultsToJSON{this, "reportResultsToJSON", true, "Report results into the json file."};
   /// Name of the JSON file
   Gaudi::Property<std::string> m_jsonFileName{this, "jsonFileName", "PerfMonMTSvc_result.json",
                                               "Name of the JSON file that contains the results."};
-- 
GitLab