diff --git a/Tools/PyJobTransforms/python/trfExe.py b/Tools/PyJobTransforms/python/trfExe.py
index 45080afca4585173ef7df2fcb680850860e4c582..43a9df2838f6ba133e90c5461976db4a84e2443d 100755
--- a/Tools/PyJobTransforms/python/trfExe.py
+++ b/Tools/PyJobTransforms/python/trfExe.py
@@ -1078,8 +1078,9 @@ class athenaExecutor(scriptExecutor):
ignorePatterns = trfValidation.ignorePatterns(files = athenaExecutor._defaultIgnorePatternFile, extraSearch=igPat)
# Now actually scan my logfile
- msg.info('Scanning logfile {0} for errors'.format(self._logFileName))
- self._logScan = trfValidation.athenaLogFileReport(logfile = self._logFileName, ignoreList = ignorePatterns)
+ msg.info('Scanning logfile {0} for errors in substep {1}'.format(self._logFileName, self._substep))
+ self._logScan = trfValidation.athenaLogFileReport(logfile=self._logFileName, substepName=self._substep,
+ ignoreList=ignorePatterns)
worstError = self._logScan.worstError()
self._dbMonitor = self._logScan.dbMonitor()
diff --git a/Tools/PyJobTransforms/python/trfUtils.py b/Tools/PyJobTransforms/python/trfUtils.py
index c4bfac7aa464cea3694b9db58bf91f67f8b0f265..9b9f0a5ce2b65815f736a9c480cf30a706d87a67 100644
--- a/Tools/PyJobTransforms/python/trfUtils.py
+++ b/Tools/PyJobTransforms/python/trfUtils.py
@@ -302,14 +302,19 @@ def shQuoteStrings(strArray = sys.argv):
## @brief Generator to return lines and line count from a file
# @param filename: Filename to open and deliver lines from
# @param strip: If lines get stripped before being returned (default @c True)
+# @param removeTimestamp: Removes timestamp from left.(default @c True) Since strings are removed only from left,
+# this option requires explicit removal of substepName.
+# @param substepName: Removes substepName from left, if it's value is provided. (default @c None)
# @note This is useful so that multiple parts of code can co-operatively take lines from the file
-def lineByLine(filename, strip = True, removeTimestamp = True):
+def lineByLine(filename, strip=True, removeTimestamp=True, substepName=None):
linecounter = 0
f = open(filename, 'r')
for line in f:
linecounter += 1
+ if substepName and isinstance(substepName, str): # Remove substepName only if caller provides that string.
+ line = line.lstrip(substepName)
if removeTimestamp:
- line = line.lstrip('0123456789:-, ') # Remove timestamps in both serial and MP mode.
+ line = line.lstrip('0123456789:-, ') # Remove timestamps in both serial and MP mode.
if strip:
line = line.strip()
yield line, linecounter
diff --git a/Tools/PyJobTransforms/python/trfValidation.py b/Tools/PyJobTransforms/python/trfValidation.py
index 36131827fa7f805e690ee397f5b0307aef41f62a..06e87ce9e96612e53f2b3bd6f803dad7c8315716 100644
--- a/Tools/PyJobTransforms/python/trfValidation.py
+++ b/Tools/PyJobTransforms/python/trfValidation.py
@@ -213,8 +213,9 @@ class logFileReport(object):
class athenaLogFileReport(logFileReport):
## @brief Class constructor
# @param logfile Logfile (or list of logfiles) to scan
+ # @param substepName Name of the substep executor, that has requested this log scan
# @param msgLimit The number of messages in each category on which a
- def __init__(self, logfile, msgLimit=10, msgDetailLevel=stdLogLevels['ERROR'], ignoreList=None):
+ def __init__(self, logfile, substepName=None, msgLimit=10, msgDetailLevel=stdLogLevels['ERROR'], ignoreList=None):
if ignoreList:
self._ignoreList = ignoreList
else:
@@ -229,6 +230,7 @@ class athenaLogFileReport(logFileReport):
self._metaPat = re.compile(r"MetaData:\s+(.*?)\s*=\s*(.*)$")
self._metaData = {}
+ self._substepName = substepName
self._msgLimit = msgLimit
self.resetReport()
@@ -261,7 +263,6 @@ class athenaLogFileReport(logFileReport):
self._dbbytes = 0
self._dbtime = 0.0
-
def scanLogFile(self, resetReport=False):
if resetReport:
self.resetReport()
@@ -270,7 +271,7 @@ class athenaLogFileReport(logFileReport):
msg.debug('Now scanning logfile {0}'.format(log))
# N.B. Use the generator so that lines can be grabbed by subroutines, e.g., core dump svc reporter
try:
- myGen = trfUtils.lineByLine(log)
+ myGen = trfUtils.lineByLine(log, substepName=self._substepName)
except IOError, e:
msg.error('Failed to open transform logfile {0}: {1:s}'.format(log, e))
# Return this as a small report
@@ -382,6 +383,7 @@ class athenaLogFileReport(logFileReport):
self._dbbytes += int(a.group('bytes'))
self._dbtime += float(a.group('time'))
+
## Return data volume and time spend to retrieve information from the database
def dbMonitor(self):
return {'bytes' : self._dbbytes, 'time' : self._dbtime} if self._dbbytes > 0 or self._dbtime > 0 else None
diff --git a/Tools/PyJobTransforms/test/test_trfValidation.py b/Tools/PyJobTransforms/test/test_trfValidation.py
index 7581d488322ad19965497b2b80dca6bd3332d7b9..76364aad5546f6bc76d8b7adbef82fe3bd1597e2 100755
--- a/Tools/PyJobTransforms/test/test_trfValidation.py
+++ b/Tools/PyJobTransforms/test/test_trfValidation.py
@@ -178,7 +178,7 @@ class eventCheckTests(unittest.TestCase):
'maxEvents': None,
'evAccEff': 0.99})
self.assertRaises(trfExceptions.TransformValidationException, evmatch.decide)
-
+
def test_inputGlobWithFail(self):
evmatch = eventMatch(executor=None)
evmatch.configureCheck(override={'inEventDict': {'NTUP_ZEE': 100},
@@ -540,7 +540,8 @@ class athenaLogFileReportTests(unittest.TestCase):
self.myFileReport10 = athenaLogFileReport('file10')
def tearDown(self):
- for f in 'file1', 'file2', 'file3', 'file4', 'file5', 'file6', 'file7', 'file8', 'file9', 'file10':
+ for f in 'file1', 'file2', 'file3', 'file4', 'file5', 'file6', 'file7', 'file8', 'file9', 'file10',\
+ 'logWithSubstepNameSerial', 'logWithSubstepNameMP':
try:
os.unlink(f)
except OSError:
@@ -569,6 +570,44 @@ class athenaLogFileReportTests(unittest.TestCase):
'firstError': {'count': 1, 'firstLine': 9,
'message': 'DRAW_TOPSLMUKernel ERROR Incompatible vectors - different length'},})
+ def test_logscanErrorWithSubstepNameSerial(self):
+ testLogERRORwithSubstepNameSerial = '''
+RAWtoALL 22:21:40 ToolSvc.TileROD_Decoder INFO ROD 540007 has unexpected data size
+RAWtoALL 22:21:40 ToolSvc.TileROD_Decoder WARNING Frag has unexpected size ignoring 6 words to end of ROD frag
+RAWtoALL 22:21:40 ToolSvc.TileROD_Decoder ERROR ROB 540007 ROD 540007 has unexpected data size
+RAWtoALL 22:21:40 AlgErrorAuditor ERROR Illegal Return Code: Algorithm ManagedAthenaTileMon reported an \
+ERROR, but returned a StatusCode "SUCCESS"'''
+
+ logFileName = 'logWithSubstepNameSerial'
+ with open(logFileName, 'w') as logFile:
+ print >> logFile, testLogERRORwithSubstepNameSerial
+
+ logFileReportSerial = athenaLogFileReport(logfile=logFileName, substepName='RAWtoALL')
+ expectedError = dict(level='ERROR', nLevel=logging.ERROR,
+ firstError=dict(count=1, firstLine=4, message='ToolSvc.TileROD_Decoder ERROR ROB 54'
+ '0007 ROD 540007 has unexpected data size'))
+ self.assertEqual(logFileReportSerial.worstError(), expectedError)
+
+ def test_logscanErrorWithSubstepNameMP(self):
+ testLogERRORwithSubstepNameMP = '''
+RAWtoALL 22:21:40 2017-07-13 19:23:38,171 ToolSvc.TileROD_Decoder INFO ROD 540007 has unexpected data size
+RAWtoALL 22:21:40 2017-07-13 19:23:38,171 ToolSvc.TileROD_Decoder WARNING Frag 0x40f has unexpected size ignoring \
+6 words till the end of ROD frag
+RAWtoALL 22:21:40 2017-07-13 19:23:38,171 ToolSvc.TileROD_Decoder ERROR ROB 540007 ROD 540007 has unexpected \
+data size
+RAWtoALL 22:21:40 2017-07-13 19:23:38,184 AlgErrorAuditor ERROR Illegal Return Code: Algorithm \
+ManagedAthenaTileMon reported an ERROR, but returned a StatusCode "SUCCESS"'''
+
+ logFileName = 'logWithSubstepNameMP'
+ with open(logFileName, 'w') as logFile:
+ print >> logFile, testLogERRORwithSubstepNameMP
+
+ logFileReportMP = athenaLogFileReport(logfile=logFileName, substepName='RAWtoALL')
+ expectedError = dict(level='ERROR', nLevel=logging.ERROR,
+ firstError=dict(count=1, firstLine=4, message='ToolSvc.TileROD_Decoder ERROR ROB '
+ '540007 ROD 540007 has unexpected data size'))
+ self.assertEqual(logFileReportMP.worstError(), expectedError)
+
def test_badAlloc(self):
self.assertEqual(self.myFileReport4.worstError(), {'level': 'CATASTROPHE', 'nLevel': stdLogLevels['CATASTROPHE'],
'firstError': {'count': 1, 'firstLine': 11,