From f85b97ef1f9e76d21a837688d54178a58d84215c Mon Sep 17 00:00:00 2001
From: Hass Abouzeid <hass.abouzeid@cern.ch>
Date: Sun, 13 Nov 2016 16:21:15 +0100
Subject: [PATCH] Making tag (PROCTools-00-01-21)
* Fixing 'extras' parameter in RunTier0Tests to actually do what is expected
* Changing default root diff to 'resilient' in RunTier0Tests
* Increase in number of warnings no longer terminates the job
* Included r2a mode for q431 test
2016-11-10 Walter Lampl <walter.lampl@ cern.ch>
* add RecoTimeRAWtoALL to ignore list
* tag PROCTools-00-01-20
2016-09-18 Hass AbouZeid <hass.abouzeid@cern.ch>
* Added derived output test to list of TCT tests not compared by TCTs
Former-commit-id: 54e926155d430270796acdfb6b55a67e739e7cde
---
Tools/PROCTools/python/RunTier0Tests.py | 40 +++++++++++++++++--------
Tools/PROCTools/python/diffTAGTree.py | 2 +-
Tools/PROCTools/python/getFileLists.py | 2 +-
3 files changed, 29 insertions(+), 15 deletions(-)
diff --git a/Tools/PROCTools/python/RunTier0Tests.py b/Tools/PROCTools/python/RunTier0Tests.py
index 05b7cffc395..86912b3c24a 100755
--- a/Tools/PROCTools/python/RunTier0Tests.py
+++ b/Tools/PROCTools/python/RunTier0Tests.py
@@ -24,10 +24,11 @@ formatter = logging.Formatter('%(levelname)-8s %(message)s')
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
-def RunCleanQTest(qtest,pwd,release,extraArg,CleanRunHeadDir,UniqID):
+def RunCleanQTest(qtest,pwd,release,extraArg,CleanRunHeadDir,UniqID, doR2A=False):
q=qtest
- if q != "q221":
- extraArg=""
+ if q == 'q431' and doR2A:
+ extraArg += " --steering='doRAWtoALL'"
+
logging.info("Running clean in rel "+release+" \"Reco_tf.py --AMI "+q+" "+extraArg+"\"")
#Check if CleanRunHead directory exists if not exist with a warning
@@ -38,10 +39,11 @@ def RunCleanQTest(qtest,pwd,release,extraArg,CleanRunHeadDir,UniqID):
logging.info("Finished clean \"Reco_tf.py --AMI "+q+"\"")
pass
-def RunPatchedQTest(qtest,pwd,release,theTestArea,extraArg):
+def RunPatchedQTest(qtest,pwd,release,theTestArea,extraArg, doR2A=False):
q=qtest
- if q != "q221":
- extraArg=""
+ if q == 'q431' and doR2A:
+ extraArg += " --steering='doRAWtoALL'"
+
logging.info("Running patched in rel "+release+" \"Reco_tf.py --AMI "+q+" "+extraArg+"\"")
if 'WorkDir_DIR' in os.environ:
@@ -180,7 +182,7 @@ def RunFrozenTier0PolicyTest(q,inputFormat,maxEvents,CleanRunHeadDir,UniqID):
clean_dir = CleanRunHeadDir+"/clean_run_"+q+"_"+UniqID
- comparison_command = "acmd.py diff-root "+clean_dir+"/my"+inputFormat+".pool.root run_"+q+"/my"+inputFormat+".pool.root --ignore-leaves --ignore-leaves RecoTimingObj_p1_HITStoRDO_timings RecoTimingObj_p1_RAWtoESD_mems RecoTimingObj_p1_RAWtoESD_timings RAWtoESD_mems RAWtoESD_timings ESDtoAOD_mems ESDtoAOD_timings HITStoRDO_mems HITStoRDO_timings --entries "+str(maxEvents)+" > run_"+q+"/diff-root-"+q+"."+inputFormat+".log 2>&1"
+ comparison_command = "acmd.py diff-root "+clean_dir+"/my"+inputFormat+".pool.root run_"+q+"/my"+inputFormat+".pool.root --error-mode resilient --ignore-leaves RecoTimingObj_p1_HITStoRDO_timings RecoTimingObj_p1_RAWtoESD_mems RecoTimingObj_p1_RAWtoESD_timings RAWtoESD_mems RAWtoESD_timings ESDtoAOD_mems ESDtoAOD_timings HITStoRDO_mems HITStoRDO_timings --entries "+str(maxEvents)+" > run_"+q+"/diff-root-"+q+"."+inputFormat+".log 2>&1"
output,error = subprocess.Popen(['/bin/bash', '-c', comparison_command], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
passed_frozen_tier0_test=False
@@ -297,7 +299,6 @@ def RunWARNINGSTest(q,qTestsToRun,CleanRunHeadDir,UniqID):
for w in wn:
logging.error(w)
_Test=False
- sys.exit(0)
elif (len(warnings_test) < len(warnings_ref)):
logging.error("Test log file "+test_file+" has "+str(len(warnings_ref) - len(warnings_test))+" less warnings than the reference log file "+ref_file)
@@ -363,20 +364,24 @@ def main():
parser=OptionParser(usage="\n ./RunTier0Test.py \n")
parser.add_option("-e","--extra" ,type="string" ,dest="extraArgs" ,default="" ,help="define additional args to pass e.g. --preExec 'r2e':'from TriggerJobOpts.TriggerFlags import TriggerFlags;TriggerFlags.triggerMenuSetup=\"MC_pp_v5\"' ")
+ parser.add_option("-a","--r2a" ,action="store_true" ,dest="r2a_flag" ,default=False ,help="r2a option will run q431 test in r2a mode")
parser.add_option("-f","--fast" ,action="store_true" ,dest="fast_flag" ,default=False ,help="fast option will run all q tests simultaneously, such that it will run faster if you have 4 cpu core slots on which to run. Be warned! Only recommended when running on a high performance machine, not lxplus")
parser.add_option("-c","--cleanDir" ,type="string" ,dest="cleanDir" ,default="/tmp/" ,help="specify the head directory for running the clean Tier0 tests. The default is /tmp/${USER}")
parser.add_option("-r","--ref" ,type="string" ,dest="ref" ,default=None ,help="define a particular reference release")
- parser.add_option("-v","--val" ,type="string" ,dest="val" ,default=None ,help="define a particular validation release")
-
+ parser.add_option("-v","--val" ,type="string" ,dest="val" ,default=None ,help="define a particular validation release")
(options,args)=parser.parse_args()
extraArg = ""
if options.extraArgs == "MC_pp_v5":
extraArg = "--preExec 'r2e':'from TriggerJobOpts.TriggerFlags import TriggerFlags;TriggerFlags.triggerMenuSetup=\"MC_pp_v5\"' "
+ else:
+ extraArg = options.extraArgs
RunFast = options.fast_flag
CleanRunHeadDir=options.cleanDir
+ r2aMode = options.r2a_flag
+
# tct_ESD = "root://eosatlas//eos/atlas/atlascerngroupdisk/proj-sit/rtt/prod/tct/"+latest_nightly+"/"+release+"/"+platform+"/offline/Tier0ChainTests/"+q+"/myESD.pool.root"
@@ -424,10 +429,19 @@ def main():
########### Define which q-tests to run
- qTestsToRun = {
+
+ qTestsToRun = {}
+ if r2aMode:
+ qTestsToRun = {
+ 'q221':[ 'HITtoRDO','RAWtoESD','ESDtoAOD','AODtoTAG'],
+ 'q431':[ 'RAWtoALL']
+ }
+ else:
+ qTestsToRun = {
'q221':[ 'HITtoRDO','RAWtoESD','ESDtoAOD','AODtoTAG'],
'q431':[ 'RAWtoESD','ESDtoAOD','AODtoTAG']
}
+
########### Get release info
mysetup = GetReleaseSetup()
@@ -458,11 +472,11 @@ def main():
q=str(qtest)
def mycleanqtest():
- RunCleanQTest(q,mypwd,cleanSetup,extraArg,CleanRunHeadDir,UniqName)
+ RunCleanQTest(q,mypwd,cleanSetup,extraArg,CleanRunHeadDir,UniqName, doR2A=r2aMode)
pass
def mypatchedqtest():
- RunPatchedQTest(q,mypwd,mysetup,myTestArea,extraArg)
+ RunPatchedQTest(q,mypwd,mysetup,myTestArea,extraArg, doR2A=r2aMode)
pass
mythreads[q+"_clean"] = threading.Thread(target=mycleanqtest)
diff --git a/Tools/PROCTools/python/diffTAGTree.py b/Tools/PROCTools/python/diffTAGTree.py
index a9f4aed86b9..e14178462ce 100755
--- a/Tools/PROCTools/python/diffTAGTree.py
+++ b/Tools/PROCTools/python/diffTAGTree.py
@@ -7,7 +7,7 @@ sys.argv += [ '-b' ] # tell ROOT to not use graphics
from ROOT import TFile, TTree
sys.argv.pop()
-setIgnoreLeaves=("Token","StreamESD_ref","StreamRDO_ref","StreamAOD_ref","RecoTimeRAWtoESD","RecoTimeESDtoAOD","JetMissingETWord")
+setIgnoreLeaves=("Token","StreamESD_ref","StreamRDO_ref","StreamAOD_ref","RecoTimeRAWtoESD","RecoTimeESDtoAOD","RecoTimeRAWtoALL","JetMissingETWord")
def diffTTree(tOld,tNew,details=None):
nOld = tOld.GetEntriesFast()
diff --git a/Tools/PROCTools/python/getFileLists.py b/Tools/PROCTools/python/getFileLists.py
index 6bbfb264e29..c8a1e065222 100644
--- a/Tools/PROCTools/python/getFileLists.py
+++ b/Tools/PROCTools/python/getFileLists.py
@@ -137,7 +137,7 @@ class findTCTFiles:
names = self._commonDirs.keys()
for tctname in names:
- if (tctname.startswith("LatestRun") or tctname.endswith("_MP") or tctname.endswith("IDCosmic0") or tctname.endswith("_PHYSVAL")):
+ if (tctname.startswith("LatestRun") or tctname.endswith("_MP") or tctname.endswith("IDCosmic0") or tctname.endswith("_PHYSVAL") or tctname.endswith("Derived_Outputs")):
print "skipping "+tctname
self._commonDirs.pop(tctname)
continue
--
GitLab