From dbf67eb6d58ba5a2c8f1bfe8ee23815c6f082904 Mon Sep 17 00:00:00 2001
From: Mohsen Rezaei Estabragh <mohsen.rezaei.estabragh@cern.ch>
Date: Fri, 13 Mar 2020 18:14:17 +0100
Subject: [PATCH 1/3] elementary modifications, baselines
---
Tools/PyJobTransforms/python/trfExe.py | 7 +++++++
Tools/PyJobTransforms/python/trfReports.py | 1 +
Tools/PyJobTransforms/python/trfUtils.py | 4 ++++
Tools/PyJobTransforms/test/test_trfReports.py | 5 +++++
4 files changed, 17 insertions(+)
diff --git a/Tools/PyJobTransforms/python/trfExe.py b/Tools/PyJobTransforms/python/trfExe.py
index cfb198bfe41a..d4e4ee6aa0e3 100755
--- a/Tools/PyJobTransforms/python/trfExe.py
+++ b/Tools/PyJobTransforms/python/trfExe.py
@@ -381,6 +381,13 @@ class transformExecutor(object):
def memStats(self):
return self._memStats
+ @property
+ def memEval(self):
+ if self._memStats:
+ return calcMemExcess(self._memStats)
+ else:
+ return None
+
@property
def postExeCpuTime(self):
if self._exeStop and self._valStart:
diff --git a/Tools/PyJobTransforms/python/trfReports.py b/Tools/PyJobTransforms/python/trfReports.py
index 63e009b90dee..eb4036bd303f 100644
--- a/Tools/PyJobTransforms/python/trfReports.py
+++ b/Tools/PyJobTransforms/python/trfReports.py
@@ -656,6 +656,7 @@ def exeResourceReport(exe, report):
if exe.memStats:
exeResource['memory'] = exe.memStats
+ exeResource['MaxVMEM'] = exe.memEval
if exe.eventCount:
exeResource['nevents'] = exe.eventCount
if exe.athenaMP:
diff --git a/Tools/PyJobTransforms/python/trfUtils.py b/Tools/PyJobTransforms/python/trfUtils.py
index 230ba083f46d..f27834cedd38 100644
--- a/Tools/PyJobTransforms/python/trfUtils.py
+++ b/Tools/PyJobTransforms/python/trfUtils.py
@@ -1296,3 +1296,7 @@ def bind_port(host, port):
ret=1
s.close()
return ret
+
+def calcMemExcess(memStat):
+ maxVmem = memStat['Max']['maxVMEM']
+ return Vmem
diff --git a/Tools/PyJobTransforms/test/test_trfReports.py b/Tools/PyJobTransforms/test/test_trfReports.py
index d709c2dd5ad6..201461dc933e 100755
--- a/Tools/PyJobTransforms/test/test_trfReports.py
+++ b/Tools/PyJobTransforms/test/test_trfReports.py
@@ -44,6 +44,9 @@ class trfFileReportUnitTests(unittest.TestCase):
self.myMultiFile = trfArgClasses.argFile(['file1', 'file2', 'file3'], name = 'inputTEST_MULTIFile', io = 'input', guid = {'file1': '05ACBDD0-5F5F-4E2E-974A-BBF4F4FE6F0B', 'file2': '1368D295-27C6-4A92-8187-704C2A6A5864', 'file3': 'F5BA4602-6CA7-4111-B3C7-CB06486B30D9'})
self.myMultiFileReport = trfFileReport(self.myMultiFile)
+ self.mytrfReport = trfReport() # trfArgClasses.argFile(['file1'], name = 'inputTEST_SINGLEFile', io = 'input', guid = {'file1': '05ACBDD0-5F5F-4E2E-974A-BBF4F4FE6F0B'}) ##??? should be contain of all keys???
+ self.myJobReport = trfJobReport(self.mytrfReport)
+
def tearDown(self):
for f in 'file1', 'file2', 'file3':
try:
@@ -66,6 +69,8 @@ class trfFileReportUnitTests(unittest.TestCase):
self.assertEqual(self.myMultiFileReport.python(type = 'full'), {'argName': 'inputTEST_MULTIFile', 'subFiles': [{'file_guid': '05ACBDD0-5F5F-4E2E-974A-BBF4F4FE6F0B', 'name': 'file1', 'file_size': 20}, {'file_guid': '1368D295-27C6-4A92-8187-704C2A6A5864', 'name': 'file2', 'file_size': 13}, {'file_guid': 'F5BA4602-6CA7-4111-B3C7-CB06486B30D9', 'name': 'file3', 'file_size': 174}], 'type': None, 'dataset': None})
self.assertEqual(self.myMultiFileReport.python(type = 'name'), {'subFiles': [{'file_guid': '05ACBDD0-5F5F-4E2E-974A-BBF4F4FE6F0B', 'name': 'file1'}, {'file_guid': '1368D295-27C6-4A92-8187-704C2A6A5864', 'name': 'file2'}, {'file_guid': 'F5BA4602-6CA7-4111-B3C7-CB06486B30D9', 'name': 'file3'}], 'nentries': 'UNDEFINED', 'dataset': None})
+ def test_JobReportPython(self):
+ self.assertEqual(self.myJobReport.python(),{} )
class machineReportUnitTests(unittest.TestCase):
# Following tests don't test content, just that the generation of the objects happens correctly
--
GitLab
From 6870e2e2c38bf99aaee6906e217f092adab5642c Mon Sep 17 00:00:00 2001
From: Mohsen Rezaei Estabragh <mohsen.rezaei.estabragh@cern.ch>
Date: Thu, 25 Jun 2020 08:39:00 +0200
Subject: [PATCH 2/3] reading full mem file and reporting first line of this
file to jobReport
---
Tools/PyJobTransforms/python/trfExe.py | 20 +++++++++++++++++---
Tools/PyJobTransforms/python/trfReports.py | 9 ++++++++-
Tools/PyJobTransforms/python/trfUtils.py | 13 +++++++++++--
3 files changed, 36 insertions(+), 6 deletions(-)
diff --git a/Tools/PyJobTransforms/python/trfExe.py b/Tools/PyJobTransforms/python/trfExe.py
index d4e4ee6aa0e3..247b3a4bf804 100755
--- a/Tools/PyJobTransforms/python/trfExe.py
+++ b/Tools/PyJobTransforms/python/trfExe.py
@@ -38,7 +38,7 @@ msg = logging.getLogger(__name__)
from PyJobTransforms.trfJobOptions import JobOptionsTemplate
from PyJobTransforms.trfUtils import asetupReport, unpackDBRelease, setupDBRelease, cvmfsDBReleaseCheck, forceToAlphaNum
-from PyJobTransforms.trfUtils import ValgrindCommand, isInteractiveEnv, calcCpuTime, calcWallTime
+from PyJobTransforms.trfUtils import ValgrindCommand, isInteractiveEnv, calcCpuTime, calcWallTime, calcMemExcess, calcMemFull
from PyJobTransforms.trfUtils import bind_port
from PyJobTransforms.trfExitCodes import trfExit
from PyJobTransforms.trfLogger import stdLogLevels
@@ -184,6 +184,7 @@ class transformExecutor(object):
self._exeStart = self._exeStop = None
self._valStart = self._valStop = None
self._memStats = {}
+ self._memFullStats = ''
self._eventCount = None
self._athenaMP = None
self._athenaMT = None
@@ -388,6 +389,13 @@ class transformExecutor(object):
else:
return None
+ @property
+ def memFullEval(self):
+ if self._memFullStats:
+ return calcMemFull(self._memFullStats)
+ else:
+ return 'cant read full mem file'
+
@property
def postExeCpuTime(self):
if self._exeStop and self._valStart:
@@ -724,6 +732,7 @@ class scriptExecutor(transformExecutor):
if self._memMonitor:
try:
self._memSummaryFile = 'prmon.summary.' + self._name + '.json'
+ self._memFullFile = 'prmon.full.' + self._name
memMonitorCommand = ['prmon', '--pid', str(p.pid), '--filename', 'prmon.full.' + self._name,
'--json-summary', self._memSummaryFile, '--interval', '30']
mem_proc = subprocess.Popen(memMonitorCommand, shell = False, close_fds=True, **encargs)
@@ -771,7 +780,13 @@ class scriptExecutor(transformExecutor):
msg.warning('Failed to load JSON memory summmary file {0}: {1}'.format(self._memSummaryFile, e))
self._memMonitor = False
self._memStats = {}
-
+ try:
+ self._memFullStats = open(self._memFullFile)
+ except Exception as e:
+ msg.warning('Failed to load JSON memory full file {0}: {1}'.format(self._memFullFile, e))
+ self._memMonitor = False
+ self._memFullStats = 'could not open mem.full file!!'
+
def validate(self):
if self._valStart is None:
@@ -812,7 +827,6 @@ class scriptExecutor(transformExecutor):
msg.debug('valStop time is {0}'.format(self._valStop))
-
class athenaExecutor(scriptExecutor):
_exitMessageLimit = 200 # Maximum error message length to report in the exitMsg
_defaultIgnorePatternFile = ['atlas_error_mask.db']
diff --git a/Tools/PyJobTransforms/python/trfReports.py b/Tools/PyJobTransforms/python/trfReports.py
index eb4036bd303f..b86f1b499903 100644
--- a/Tools/PyJobTransforms/python/trfReports.py
+++ b/Tools/PyJobTransforms/python/trfReports.py
@@ -656,7 +656,14 @@ def exeResourceReport(exe, report):
if exe.memStats:
exeResource['memory'] = exe.memStats
- exeResource['MaxVMEM'] = exe.memEval
+ if exe.memEval:
+ exeResource['MaxVMEM'] = exe.memEval
+ else:
+ exeResource['MaxVMEM'] = 'at least report this'
+ if exe.memFullEval:
+ exeResource['FullMem'] = exe.memFullEval
+ else:
+ exeResource['FullMem'] = 'at least report this'
if exe.eventCount:
exeResource['nevents'] = exe.eventCount
if exe.athenaMP:
diff --git a/Tools/PyJobTransforms/python/trfUtils.py b/Tools/PyJobTransforms/python/trfUtils.py
index f27834cedd38..63cf5c79c4c0 100644
--- a/Tools/PyJobTransforms/python/trfUtils.py
+++ b/Tools/PyJobTransforms/python/trfUtils.py
@@ -1298,5 +1298,14 @@ def bind_port(host, port):
return ret
def calcMemExcess(memStat):
- maxVmem = memStat['Max']['maxVMEM']
- return Vmem
+ maxVmem = memStat['Max']['vmem']
+ reportSum = 'couldnt get it'
+ reportSum = maxVmem
+ return reportSum
+
+def calcMemFull(memFullStats):
+ reportFull = 'in utiles: can not read lines!!'
+ for line in memFullStats:
+ reportFull = line
+ break
+ return reportFull
--
GitLab
From 92ecf89f8d74409aa756b478f1db7757ae0de690 Mon Sep 17 00:00:00 2001
From: Mohsen Rezaei Estabragh <mohsen.rezaei.estabragh@cern.ch>
Date: Thu, 2 Jul 2020 09:37:28 +0200
Subject: [PATCH 3/3] removing redundancies
---
Tools/PyJobTransforms/python/trfExe.py | 11 ++---------
Tools/PyJobTransforms/python/trfReports.py | 4 ----
Tools/PyJobTransforms/python/trfUtils.py | 6 ------
Tools/PyJobTransforms/test/test_trfReports.py | 5 -----
4 files changed, 2 insertions(+), 24 deletions(-)
diff --git a/Tools/PyJobTransforms/python/trfExe.py b/Tools/PyJobTransforms/python/trfExe.py
index 247b3a4bf804..4852574ab83d 100755
--- a/Tools/PyJobTransforms/python/trfExe.py
+++ b/Tools/PyJobTransforms/python/trfExe.py
@@ -38,7 +38,7 @@ msg = logging.getLogger(__name__)
from PyJobTransforms.trfJobOptions import JobOptionsTemplate
from PyJobTransforms.trfUtils import asetupReport, unpackDBRelease, setupDBRelease, cvmfsDBReleaseCheck, forceToAlphaNum
-from PyJobTransforms.trfUtils import ValgrindCommand, isInteractiveEnv, calcCpuTime, calcWallTime, calcMemExcess, calcMemFull
+from PyJobTransforms.trfUtils import ValgrindCommand, isInteractiveEnv, calcCpuTime, calcWallTime, calcMemFull
from PyJobTransforms.trfUtils import bind_port
from PyJobTransforms.trfExitCodes import trfExit
from PyJobTransforms.trfLogger import stdLogLevels
@@ -382,13 +382,6 @@ class transformExecutor(object):
def memStats(self):
return self._memStats
- @property
- def memEval(self):
- if self._memStats:
- return calcMemExcess(self._memStats)
- else:
- return None
-
@property
def memFullEval(self):
if self._memFullStats:
@@ -787,7 +780,6 @@ class scriptExecutor(transformExecutor):
self._memMonitor = False
self._memFullStats = 'could not open mem.full file!!'
-
def validate(self):
if self._valStart is None:
self._valStart = os.times()
@@ -827,6 +819,7 @@ class scriptExecutor(transformExecutor):
msg.debug('valStop time is {0}'.format(self._valStop))
+
class athenaExecutor(scriptExecutor):
_exitMessageLimit = 200 # Maximum error message length to report in the exitMsg
_defaultIgnorePatternFile = ['atlas_error_mask.db']
diff --git a/Tools/PyJobTransforms/python/trfReports.py b/Tools/PyJobTransforms/python/trfReports.py
index b86f1b499903..9d7fdee6c110 100644
--- a/Tools/PyJobTransforms/python/trfReports.py
+++ b/Tools/PyJobTransforms/python/trfReports.py
@@ -656,10 +656,6 @@ def exeResourceReport(exe, report):
if exe.memStats:
exeResource['memory'] = exe.memStats
- if exe.memEval:
- exeResource['MaxVMEM'] = exe.memEval
- else:
- exeResource['MaxVMEM'] = 'at least report this'
if exe.memFullEval:
exeResource['FullMem'] = exe.memFullEval
else:
diff --git a/Tools/PyJobTransforms/python/trfUtils.py b/Tools/PyJobTransforms/python/trfUtils.py
index 63cf5c79c4c0..43918b56ed8b 100644
--- a/Tools/PyJobTransforms/python/trfUtils.py
+++ b/Tools/PyJobTransforms/python/trfUtils.py
@@ -1297,12 +1297,6 @@ def bind_port(host, port):
s.close()
return ret
-def calcMemExcess(memStat):
- maxVmem = memStat['Max']['vmem']
- reportSum = 'couldnt get it'
- reportSum = maxVmem
- return reportSum
-
def calcMemFull(memFullStats):
reportFull = 'in utiles: can not read lines!!'
for line in memFullStats:
diff --git a/Tools/PyJobTransforms/test/test_trfReports.py b/Tools/PyJobTransforms/test/test_trfReports.py
index 201461dc933e..d709c2dd5ad6 100755
--- a/Tools/PyJobTransforms/test/test_trfReports.py
+++ b/Tools/PyJobTransforms/test/test_trfReports.py
@@ -44,9 +44,6 @@ class trfFileReportUnitTests(unittest.TestCase):
self.myMultiFile = trfArgClasses.argFile(['file1', 'file2', 'file3'], name = 'inputTEST_MULTIFile', io = 'input', guid = {'file1': '05ACBDD0-5F5F-4E2E-974A-BBF4F4FE6F0B', 'file2': '1368D295-27C6-4A92-8187-704C2A6A5864', 'file3': 'F5BA4602-6CA7-4111-B3C7-CB06486B30D9'})
self.myMultiFileReport = trfFileReport(self.myMultiFile)
- self.mytrfReport = trfReport() # trfArgClasses.argFile(['file1'], name = 'inputTEST_SINGLEFile', io = 'input', guid = {'file1': '05ACBDD0-5F5F-4E2E-974A-BBF4F4FE6F0B'}) ##??? should be contain of all keys???
- self.myJobReport = trfJobReport(self.mytrfReport)
-
def tearDown(self):
for f in 'file1', 'file2', 'file3':
try:
@@ -69,8 +66,6 @@ class trfFileReportUnitTests(unittest.TestCase):
self.assertEqual(self.myMultiFileReport.python(type = 'full'), {'argName': 'inputTEST_MULTIFile', 'subFiles': [{'file_guid': '05ACBDD0-5F5F-4E2E-974A-BBF4F4FE6F0B', 'name': 'file1', 'file_size': 20}, {'file_guid': '1368D295-27C6-4A92-8187-704C2A6A5864', 'name': 'file2', 'file_size': 13}, {'file_guid': 'F5BA4602-6CA7-4111-B3C7-CB06486B30D9', 'name': 'file3', 'file_size': 174}], 'type': None, 'dataset': None})
self.assertEqual(self.myMultiFileReport.python(type = 'name'), {'subFiles': [{'file_guid': '05ACBDD0-5F5F-4E2E-974A-BBF4F4FE6F0B', 'name': 'file1'}, {'file_guid': '1368D295-27C6-4A92-8187-704C2A6A5864', 'name': 'file2'}, {'file_guid': 'F5BA4602-6CA7-4111-B3C7-CB06486B30D9', 'name': 'file3'}], 'nentries': 'UNDEFINED', 'dataset': None})
- def test_JobReportPython(self):
- self.assertEqual(self.myJobReport.python(),{} )
class machineReportUnitTests(unittest.TestCase):
# Following tests don't test content, just that the generation of the objects happens correctly
--
GitLab