diff --git a/Tools/PyJobTransforms/python/trfExe.py b/Tools/PyJobTransforms/python/trfExe.py index d4e4ee6aa0e377fa9ffb5a9a1434162cebd0ec54..247b3a4bf804f7b9d828409a9bd075f3b426b3fd 100755 --- a/Tools/PyJobTransforms/python/trfExe.py +++ b/Tools/PyJobTransforms/python/trfExe.py @@ -38,7 +38,7 @@ msg = logging.getLogger(__name__) from PyJobTransforms.trfJobOptions import JobOptionsTemplate from PyJobTransforms.trfUtils import asetupReport, unpackDBRelease, setupDBRelease, cvmfsDBReleaseCheck, forceToAlphaNum -from PyJobTransforms.trfUtils import ValgrindCommand, isInteractiveEnv, calcCpuTime, calcWallTime +from PyJobTransforms.trfUtils import ValgrindCommand, isInteractiveEnv, calcCpuTime, calcWallTime, calcMemExcess, calcMemFull from PyJobTransforms.trfUtils import bind_port from PyJobTransforms.trfExitCodes import trfExit from PyJobTransforms.trfLogger import stdLogLevels @@ -184,6 +184,7 @@ class transformExecutor(object): self._exeStart = self._exeStop = None self._valStart = self._valStop = None self._memStats = {} + self._memFullStats = '' self._eventCount = None self._athenaMP = None self._athenaMT = None @@ -388,6 +389,13 @@ class transformExecutor(object): else: return None + @property + def memFullEval(self): + if self._memFullStats: + return calcMemFull(self._memFullStats) + else: + return 'cant read full mem file' + @property def postExeCpuTime(self): if self._exeStop and self._valStart: @@ -724,6 +732,7 @@ class scriptExecutor(transformExecutor): if self._memMonitor: try: self._memSummaryFile = 'prmon.summary.' + self._name + '.json' + self._memFullFile = 'prmon.full.' + self._name memMonitorCommand = ['prmon', '--pid', str(p.pid), '--filename', 'prmon.full.' + self._name, '--json-summary', self._memSummaryFile, '--interval', '30'] mem_proc = subprocess.Popen(memMonitorCommand, shell = False, close_fds=True, **encargs) @@ -771,7 +780,13 @@ class scriptExecutor(transformExecutor): msg.warning('Failed to load JSON memory summmary file {0}: {1}'.format(self._memSummaryFile, e)) self._memMonitor = False self._memStats = {} - + try: + self._memFullStats = open(self._memFullFile) + except Exception as e: + msg.warning('Failed to load JSON memory full file {0}: {1}'.format(self._memFullFile, e)) + self._memMonitor = False + self._memFullStats = 'could not open mem.full file!!' + def validate(self): if self._valStart is None: @@ -812,7 +827,6 @@ class scriptExecutor(transformExecutor): msg.debug('valStop time is {0}'.format(self._valStop)) - class athenaExecutor(scriptExecutor): _exitMessageLimit = 200 # Maximum error message length to report in the exitMsg _defaultIgnorePatternFile = ['atlas_error_mask.db'] diff --git a/Tools/PyJobTransforms/python/trfReports.py b/Tools/PyJobTransforms/python/trfReports.py index eb4036bd303fb6a523025c8ea341344f236038f7..b86f1b499903d1af2b3e6ef7a75cbcc32c147075 100644 --- a/Tools/PyJobTransforms/python/trfReports.py +++ b/Tools/PyJobTransforms/python/trfReports.py @@ -656,7 +656,14 @@ def exeResourceReport(exe, report): if exe.memStats: exeResource['memory'] = exe.memStats - exeResource['MaxVMEM'] = exe.memEval + if exe.memEval: + exeResource['MaxVMEM'] = exe.memEval + else: + exeResource['MaxVMEM'] = 'at least report this' + if exe.memFullEval: + exeResource['FullMem'] = exe.memFullEval + else: + exeResource['FullMem'] = 'at least report this' if exe.eventCount: exeResource['nevents'] = exe.eventCount if exe.athenaMP: diff --git a/Tools/PyJobTransforms/python/trfUtils.py b/Tools/PyJobTransforms/python/trfUtils.py index f27834cedd38f899675983d582a60690ee494fcd..63cf5c79c4c0c0de722b0c4117ed7e7e3e00df47 100644 --- a/Tools/PyJobTransforms/python/trfUtils.py +++ b/Tools/PyJobTransforms/python/trfUtils.py @@ -1298,5 +1298,14 @@ def bind_port(host, port): return ret def calcMemExcess(memStat): - maxVmem = memStat['Max']['maxVMEM'] - return Vmem + maxVmem = memStat['Max']['vmem'] + reportSum = 'couldnt get it' + reportSum = maxVmem + return reportSum + +def calcMemFull(memFullStats): + reportFull = 'in utiles: can not read lines!!' + for line in memFullStats: + reportFull = line + break + return reportFull