reading full mem file and reporting first line of this file to jobReport

Tools/PyJobTransforms/python/trfExe.py

@@ -38,7 +38,7 @@ msg = logging.getLogger(__name__)
 
 from PyJobTransforms.trfJobOptions import JobOptionsTemplate
 from PyJobTransforms.trfUtils import asetupReport, unpackDBRelease, setupDBRelease, cvmfsDBReleaseCheck, forceToAlphaNum
-from PyJobTransforms.trfUtils import ValgrindCommand, isInteractiveEnv, calcCpuTime, calcWallTime
+from PyJobTransforms.trfUtils import ValgrindCommand, isInteractiveEnv, calcCpuTime, calcWallTime, calcMemExcess, calcMemFull
 from PyJobTransforms.trfUtils import bind_port
 from PyJobTransforms.trfExitCodes import trfExit
 from PyJobTransforms.trfLogger import stdLogLevels
@@ -184,6 +184,7 @@ class transformExecutor(object):
         self._exeStart = self._exeStop = None
         self._valStart = self._valStop = None
         self._memStats = {}
+        self._memFullStats = ''
         self._eventCount = None
         self._athenaMP = None
         self._athenaMT = None
@@ -388,6 +389,13 @@ class transformExecutor(object):
         else:
             return None
 
+    @property
+    def memFullEval(self):
+        if self._memFullStats:
+            return calcMemFull(self._memFullStats)
+        else:
+            return 'cant read full mem file'
+
     @property
     def postExeCpuTime(self):
         if self._exeStop and self._valStart:
@@ -724,6 +732,7 @@ class scriptExecutor(transformExecutor):
             if self._memMonitor:
                 try:
                     self._memSummaryFile = 'prmon.summary.' + self._name + '.json'
+                    self._memFullFile = 'prmon.full.' + self._name
                     memMonitorCommand = ['prmon', '--pid', str(p.pid), '--filename', 'prmon.full.' + self._name, 
                                          '--json-summary', self._memSummaryFile, '--interval', '30']
                     mem_proc = subprocess.Popen(memMonitorCommand, shell = False, close_fds=True, **encargs)
@@ -771,7 +780,13 @@ class scriptExecutor(transformExecutor):
                 msg.warning('Failed to load JSON memory summmary file {0}: {1}'.format(self._memSummaryFile, e))
                 self._memMonitor = False
                 self._memStats = {}
-            
+            try:
+                self._memFullStats = open(self._memFullFile)
+            except Exception as e:
+                msg.warning('Failed to load JSON memory full file {0}: {1}'.format(self._memFullFile, e))
+                self._memMonitor = False
+                self._memFullStats = 'could not open mem.full file!!'
+
 
     def validate(self):
         if self._valStart is None:
@@ -812,7 +827,6 @@ class scriptExecutor(transformExecutor):
         msg.debug('valStop time is {0}'.format(self._valStop))
 
 
-
 class athenaExecutor(scriptExecutor):
     _exitMessageLimit = 200 # Maximum error message length to report in the exitMsg
     _defaultIgnorePatternFile = ['atlas_error_mask.db']

Tools/PyJobTransforms/python/trfReports.py

@@ -656,7 +656,14 @@ def exeResourceReport(exe, report):
 
     if exe.memStats:
         exeResource['memory'] = exe.memStats
-        exeResource['MaxVMEM'] = exe.memEval
+        if exe.memEval:
+            exeResource['MaxVMEM'] = exe.memEval
+        else:
+            exeResource['MaxVMEM'] = 'at least report this'
+        if exe.memFullEval:
+            exeResource['FullMem'] = exe.memFullEval
+        else:
+            exeResource['FullMem'] = 'at least report this'
     if exe.eventCount:
         exeResource['nevents'] = exe.eventCount
     if exe.athenaMP:

Tools/PyJobTransforms/python/trfUtils.py

@@ -1298,5 +1298,14 @@ def bind_port(host, port):
     return ret
 
 def calcMemExcess(memStat):
-    maxVmem = memStat['Max']['maxVMEM']
-    return Vmem
+    maxVmem = memStat['Max']['vmem']
+    reportSum = 'couldnt get it'
+    reportSum = maxVmem
+    return reportSum
+
+def calcMemFull(memFullStats):
+    reportFull = 'in utiles: can not read lines!!'
+    for line in memFullStats:
+        reportFull = line
+        break
+    return reportFull