From c22c08b63c59e4dc0ce55fc8dcc0fa74fb457513 Mon Sep 17 00:00:00 2001
From: Antonio Limosani <Antonio.Limosani@cern.ch>
Date: Mon, 25 Jul 2016 17:14:01 +0200
Subject: [PATCH] Add warnings mgs test and logger (PROCTools-00-01-18)
* Introduce logger to generate messages
* Add WARNING message check
* Tagging PROCTools-00-01-18
2016-07-22 Hass AbouZeid <hass.abouzeid@cern.ch>
* Update/sync AMI-tag upload scripts and stream output script
* Update to ignored-leaves for TCT comparisons (RAWtoALL timings/mems)
* Tagging PROCTools-00-01-17
Former-commit-id: ae5f05d9820a90161107d1542ca8c6aec0b79d81
---
Tools/PROCTools/python/RunTier0Tests.py | 246 ++++++++++++++++-------
Tools/PROCTools/python/UploadTfAMITag.py | 128 +++++++++---
Tools/PROCTools/python/compareTCTs.py | 2 +-
Tools/PROCTools/python/outputTest_v2.py | 2 +-
4 files changed, 271 insertions(+), 107 deletions(-)
diff --git a/Tools/PROCTools/python/RunTier0Tests.py b/Tools/PROCTools/python/RunTier0Tests.py
index 800d6e46eb7..7f7e8eca129 100755
--- a/Tools/PROCTools/python/RunTier0Tests.py
+++ b/Tools/PROCTools/python/RunTier0Tests.py
@@ -10,29 +10,42 @@ import os
import threading
import time
import uuid
+import logging
+
+### Setup global logging
+logging.basicConfig(level=logging.INFO,
+ format='%(asctime)s %(levelname)-8s %(message)s',
+ datefmt='%m-%d %H:%M',
+ filename='./RunTier0Tests.log',
+ filemode='w')
+console = logging.StreamHandler()
+console.setLevel(logging.INFO)
+formatter = logging.Formatter('%(levelname)-8s %(message)s')
+console.setFormatter(formatter)
+logging.getLogger('').addHandler(console)
def RunCleanQTest(qtest,pwd,release,extraArg,CleanRunHeadDir,UniqID):
q=qtest
if q != "q221":
extraArg=""
- print "Running clean in rel "+release+" \"Reco_tf.py --AMI "+q+" "+extraArg+"\""
+ logging.info("Running clean in rel "+release+" \"Reco_tf.py --AMI "+q+" "+extraArg+"\"")
#Check if CleanRunHead directory exists if not exist with a warning
CleanDirName="clean_run_"+q+"_"+UniqID
cmd = "mkdir -p "+CleanRunHeadDir+" ; cd "+CleanRunHeadDir+"; mkdir -p "+CleanDirName+" ; cd "+CleanDirName+" ; source $AtlasSetup/scripts/asetup.sh "+release+" --testarea `pwd` >& /dev/null ; Reco_tf.py --AMI="+q+" "+extraArg+" > "+q+".log 2>&1"
subprocess.call(cmd,shell=True)
- print "Finished clean \"Reco_tf.py --AMI "+q+"\""
+ logging.info("Finished clean \"Reco_tf.py --AMI "+q+"\"")
pass
def RunPatchedQTest(qtest,pwd,release,theTestArea,extraArg):
q=qtest
if q != "q221":
extraArg=""
- print "Running patched in rel "+release+" \"Reco_tf.py --AMI "+q+" "+extraArg+"\""
+ logging.info("Running patched in rel "+release+" \"Reco_tf.py --AMI "+q+" "+extraArg+"\"")
cmd = "cd "+pwd+"; source $AtlasSetup/scripts/asetup.sh "+release+" --testarea "+theTestArea+" >& /dev/null ; mkdir -p run_"+q+"; cd run_"+q+"; Reco_tf.py --AMI="+q+" "+extraArg+" > "+q+".log 2>&1"
subprocess.call(cmd,shell=True)
- print "Finished patched \"Reco_tf.py --AMI "+q+"\""
+ logging.info("Finished patched \"Reco_tf.py --AMI "+q+"\"")
pass
def pwd():
@@ -73,11 +86,11 @@ def GetReleaseSetup():
platform = os.environ['GEANT4'].split('/')[-1]
if current_nightly != latest_nightly:
- print "Please be aware that you are not testing your tags in the latest available nightly, which is",latest_nightly
+ logging.info("Please be aware that you are not testing your tags in the latest available nightly, which is "+latest_nightly )
setup="%s,%s,%s"%(release,platform.replace("-", ","),current_nightly)
- print "Your tags will be tested in environment ",setup
-
+ logging.info("Your tags will be tested in environment "+setup)
+
return setup
###############################
@@ -85,7 +98,8 @@ def GetReleaseSetup():
def list_patch_packages():
if 'CMTPATH' in os.environ:
if 'TestArea' in os.environ and os.access(os.environ['TestArea'], os.R_OK):
- print "Patch packages in your InstallArea that will be tested are:\n"
+ logging.info("Patch packages in your InstallArea that will be tested are:\n")
+
cmd = ['cmt', 'show', 'packages', os.environ['TestArea']]
cmtProc = subprocess.Popen(cmd, shell = False, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, bufsize = 1)
cmtOut = cmtProc.communicate()[0]
@@ -94,63 +108,64 @@ def list_patch_packages():
if line.strip() == '':
continue
(package, packageVersion, packagePath) = line.split()
- print '\t%s\n' % (packageVersion)
+ logging.info('\t%s\n' % (packageVersion))
except ValueError:
- print "Warning, unusual output from cmt: %s\n" % line
+ logging.info("Warning, unusual output from cmt: %s\n" % line )
elif 'CMAKE_PREFIX_PATH' in os.environ :
- print "Patch packages in your build to be tested:\n"
+ logging.info("Patch packages in your build to be tested:\n")
myfilepath = os.environ['CMAKE_PREFIX_PATH'].split(":")[0]
fname = str(myfilepath) + '/packages.txt'
with open(fname) as fp:
for line in fp:
if '#' not in line:
- print line
+ logging.info(line)
else:
- print "A release has not been setup"
+ logging.info("A release has not been setup")
pass
###############################
########### Was the q test successful? To check simply count the number of lines containing the string "successful run"
def QTestsFailedOrPassed(q,qTestsToRun,CleanRunHeadDir,UniqID):
- print "-----------------------------------------------------"
- print "Did each step of the "+q+" test complete successfully?"
+ logging.info("-----------------------------------------------------" )
+ logging.info("Did each step of the "+q+" test complete successfully?" )
test_dir = CleanRunHeadDir+"/clean_run_"+q+"_"+UniqID
_Test=True
for step in qTestsToRun[q]:
- print ""
+ logging.info("")
cmd = "grep \"successful run\" " + test_dir + "/log."+str(step)
ref = subprocess.Popen(['/bin/bash', '-c',cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
if "successful run" in ref:
- print step+" Reference test successful"
+ logging.info(step+" Reference test successful")
else :
- print step+" Reference test failed"
+ logging.info(step+" Reference test failed")
_Test = False
cmd = "grep \"successful run\" run_"+q+"/log."+str(step)
test = subprocess.Popen(['/bin/bash', '-c',cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
if "successful run" in test:
- print step+" Patched test successful"
+ logging.info(step+" Patched test successful")
else :
- print step+" Patched test failed"
+ logging.info(step+" Patched test failed")
_Test = False
- print ""
+ logging.info("")
if _Test == True:
- print "All "+q+" athena steps completed successfully"
+ logging.info("All "+q+" athena steps completed successfully")
else :
- print "One or more "+q+" Athena steps failed. Please investigate the cause."
+ logging.info("One or more "+q+" Athena steps failed. Please investigate the cause.")
sys.exit()
############### Run Frozen Tier0 Policy Test
def RunFrozenTier0PolicyTest(q,inputFormat,maxEvents,CleanRunHeadDir,UniqID):
- print "---------------------------------------------------------------------------------------"
- print "Running "+q+" Frozen Tier0 Policy Test on "+inputFormat+" for "+str(maxEvents)+" events"
+ logging.info("---------------------------------------------------------------------------------------" )
+ logging.info("Running "+q+" Frozen Tier0 Policy Test on "+inputFormat+" for "+str(maxEvents)+" events" )
+
clean_dir = CleanRunHeadDir+"/clean_run_"+q+"_"+UniqID
comparison_command = "acmd.py diff-root "+clean_dir+"/my"+inputFormat+".pool.root run_"+q+"/my"+inputFormat+".pool.root --ignore-leaves --ignore-leaves RecoTimingObj_p1_HITStoRDO_timings RecoTimingObj_p1_RAWtoESD_mems RecoTimingObj_p1_RAWtoESD_timings RAWtoESD_mems RAWtoESD_timings ESDtoAOD_mems ESDtoAOD_timings HITStoRDO_mems HITStoRDO_timings --entries "+str(maxEvents)+" > run_"+q+"/diff-root-"+q+"."+inputFormat+".log 2>&1"
@@ -162,22 +177,22 @@ def RunFrozenTier0PolicyTest(q,inputFormat,maxEvents,CleanRunHeadDir,UniqID):
for line in f.readlines():
if "WARNING" in line:
- print line
+ logging.info(line)
if "INFO all good." in line :
passed_frozen_tier0_test=True
f.close()
if passed_frozen_tier0_test:
- print "Passed!"
+ logging.info("Passed!")
else:
- print "Your tag breaks the frozen tier0 policy in test "+q+". See run_"+q+"/diff-root-"+q+"."+inputFormat+".log file for more information."
-
+ logging.info("Your tag breaks the frozen tier0 policy in test "+q+". See run_"+q+"/diff-root-"+q+"."+inputFormat+".log file for more information.")
pass
############### Run A Very Simple Test
def RunTest(q,qTestsToRun,TestName,SearchString,MeasurementUnit,FieldNumber,Threshold,CleanRunHeadDir,UniqID):
- print "-----------------------------------------------------"
- print "Running "+q+" "+TestName+" Test"
+
+ logging.info("-----------------------------------------------------")
+ logging.info("Running "+q+" "+TestName+" Test" )
test_dir = CleanRunHeadDir+"/clean_run_"+q+"_"+UniqID
@@ -190,7 +205,7 @@ def RunTest(q,qTestsToRun,TestName,SearchString,MeasurementUnit,FieldNumber,Thre
try:
ref = int(subprocess.Popen(['/bin/bash', '-c',cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].split()[FieldNumber])
except:
- print "No data available in "+ test_dir + "/log."+str(step)+" . Job failed."
+ logging.info("No data available in "+ test_dir + "/log."+str(step)+" . Job failed.")
return
cmd = "grep \""+SearchString+"\" run_"+q+"/log."+str(step)
@@ -198,7 +213,7 @@ def RunTest(q,qTestsToRun,TestName,SearchString,MeasurementUnit,FieldNumber,Thre
try:
test = int(subprocess.Popen(['/bin/bash', '-c',cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].split()[FieldNumber])
except:
- print "No data available in run_"+q+"/log."+str(step)+" . Job failed."
+ logging.info("No data available in run_"+q+"/log."+str(step)+" . Job failed.")
return
@@ -206,27 +221,100 @@ def RunTest(q,qTestsToRun,TestName,SearchString,MeasurementUnit,FieldNumber,Thre
factor = float(test) / float(ref)
if factor > float(1+Threshold) or factor < float(1-Threshold) :
- print SearchString+" in the",step," step with(out) your tag is",test,"(",ref,") "+MeasurementUnit
- print "Your tag changes "+SearchString+" by a factor "+str(factor)
- print "Is this an expected outcome of your tag(s)?"
+ logging.info(SearchString+" in the",step," step with(out) your tag is",test,"(",ref,") "+MeasurementUnit )
+ logging.info("Your tag changes "+SearchString+" by a factor "+str(factor) )
+ logging.info("Is this an expected outcome of your tag(s)?" )
_Test=False
- print step+" : "+TestName
- print "ref "+str(ref)+" "+str(MeasurementUnit)
- print "test "+str(test)+" "+str(MeasurementUnit)
+ logging.info(step+" : "+TestName )
+ logging.info("ref "+str(ref)+" "+str(MeasurementUnit) )
+ logging.info("test "+str(test)+" "+str(MeasurementUnit))
+
+
if _Test:
- print "Passed!"
+ logging.info("Passed!")
else :
- print "Failed!"
+ logging.info("Failed!" )
+
+ pass
+
+############### Run a WARNING helper function
+def warnings_count(file_name):
+ warnings=[]
+ with open(file_name, "r") as file:
+ lines= file.readlines()
+ for line in lines:
+ if "WARNING" in line:
+ if not "| WARNING |" in line:
+ warnings.append(line)
+ return warnings
+
+
+############### Run a WARNINGs TEST
+def RunWARNINGSTest(q,qTestsToRun,CleanRunHeadDir,UniqID):
+ logging.info("-----------------------------------------------------")
+ logging.info("Running "+q+" WARNINGS Test" )
+ logging.info("")
+
+ test_dir = str(CleanRunHeadDir+"/clean_run_"+q+"_"+UniqID)
+
+ _Test=True
+ for step in qTestsToRun[q]:
+
+
+ ref_file=str(test_dir + "/log."+str(step))
+ test_file = str("run_"+q+"/log."+str(step))
+ warnings_ref = warnings_count(ref_file)
+ warnings_test = warnings_count (test_file)
+
+ wr=[]
+ for w in warnings_ref:
+ wr.append(w[9:])
+ wt=[]
+ for w in warnings_test:
+ wt.append(w[9:])
+
+ wn = list(set(wt)-set(wr))
+ wo = list(set(wr)-set(wt))
+
+
+ if (len(warnings_test) > len(warnings_ref)):
+ logging.error("Test log file "+test_file+" has "+str(len(warnings_test) - len(warnings_ref))+" more warning(s) than the reference log file "+ref_file)
+ logging.error("Please remove the new warning message(s):")
+ for w in wn:
+ logging.error(w)
+ _Test=False
+ sys.exit(0)
+
+ elif (len(warnings_test) < len(warnings_ref)):
+ logging.error("Test log file "+test_file+" has "+str(len(warnings_ref) - len(warnings_test))+" less warnings than the reference log file "+ref_file)
+ logging.error("The reduction of unnecessary WARNINGs is much appreciated. Is it expected?")
+ logging.error("The following warning messages have been removed:")
+ for w in wo:
+ logging.error(w)
+ _Test=True
+ else :
+ logging.info("Test log file "+test_file+" has the same number of warnings as the reference log file "+ref_file)
+ _Test=True
+
+
+ logging.info("")
+
+ if _Test:
+ logging.info("Passed!")
+ else :
+ logging.info("Failed!" )
+
+ pass
pass
##########################################################################
def RunHistTest(q,CleanRunHeadDir,UniqID):
- print "-----------------------------------------------------"
- print "Running "+q+" HIST Comparison Test"
+ logging.info("-----------------------------------------------------" )
+ logging.info("Running "+q+" HIST Comparison Test" )
ref_file = CleanRunHeadDir+"/clean_run_"+q+"_"+UniqID+"/myHIST.root"
test_file = "./run_"+q+"/myHIST.root"
@@ -245,13 +333,17 @@ def RunHistTest(q,CleanRunHeadDir,UniqID):
output,error = subprocess.Popen(['/bin/bash', '-c', comparison_command], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
f = open("run_"+q+"/diff."+q+".HIST.log", 'r')
- print "The following are changes you've made to the HIST file: "
+ logging.info("The following are changes you've made to the HIST file: ")
for line in f.readlines():
- print line
+ logging.info(line)
f.close()
- print "-----------------------------------------------------"
+ logging.info("-----------------------------------------------------" )
pass
+
+
+
+
##########################################################################
def main():
@@ -283,41 +375,44 @@ def main():
CleanRunHeadDir = "/tmp/"+str(myUser)
if os.path.exists(CleanRunHeadDir):
- print
- print "The head directory for the output of the clean Tier0 q-tests will be "+CleanRunHeadDir
- print
+ logging.info("")
+ logging.info("The head directory for the output of the clean Tier0 q-tests will be "+CleanRunHeadDir)
+ logging.info("")
else:
- print
- print "Exit. Please specify a directory that exists for the argument of the -c or --cleanDir option"
- print
- print "RunTier0Tests.py --cleanDir <ExistingDirectory>"
- print
+ logging.info("")
+ logging.info("Exit. Please specify a directory that exists for the argument of the -c or --cleanDir option")
+ logging.info("")
+ logging.info("RunTier0Tests.py --cleanDir <ExistingDirectory>")
+ logging.info("")
sys.exit(0)
########### Is an ATLAS release setup?
if 'AtlasPatchVersion' not in os.environ and 'AtlasArea' not in os.environ and 'AtlasBaseDir' not in os.environ:
- print "Exit. Please setup the an ATLAS release"
+ logging.info("Exit. Please setup the an ATLAS release")
sys.exit(0)
elif 'TestArea' not in os.environ :
- print "Exit. The environment variable ${TESTAREA} is not defined."
- print "Please re-setup the release with the argument \"here\" in the execution of the asetup command"
- print "to specify the TestArea to be the directory from which you setup the release"
- print "E.g. "
- print " asetup 20.7.X.Y-VAL,rel_5,AtlasProduction,here"
- print
- print "or use the --testarea <TestArea> option of asetup to explicitly define the TestArea"
- print "E.g. "
- print " asetup 20.7.X.Y-VAL,rel_5,AtlasProduction --testarea `pwd`"
- print
+ logging.info("Exit. The environment variable ${TESTAREA} is not defined." )
+ logging.info("Please re-setup the release with the argument \"here\" in the execution of the asetup command" )
+ logging.info("to specify the TestArea to be the directory from which you setup the release" )
+ logging.info("E.g. " )
+ logging.info(" asetup 20.7.X.Y-VAL,rel_5,AtlasProduction,here" )
+ logging.info( )
+ logging.info("or use the --testarea <TestArea> option of asetup to explicitly define the TestArea" )
+ logging.info("E.g. " )
+ logging.info(" asetup 20.7.X.Y-VAL,rel_5,AtlasProduction --testarea `pwd`" )
+ logging.info( )
+
+
+
sys.exit(0)
elif not os.path.exists(os.environ['TestArea']):
- print "Exit. The path for your TestArea "+os.environ['TestArea']+" does not exist!."
+ logging.info("Exit. The path for your TestArea "+os.environ['TestArea']+" does not exist!." )
else:
if 'AtlasPatchVersion' not in os.environ and 'AtlasArea' not in os.environ and 'AtlasBaseDir' in os.environ:
- print "Please be aware that you are running in a base release rather than a Tier0 release, where in general q-tests are not guaranteed to work."
-
+ logging.info("Please be aware that you are running in a base release rather than a Tier0 release, where in general q-tests are not guaranteed to work.")
+
########### Define which q-tests to run
qTestsToRun = {
@@ -335,8 +430,8 @@ def main():
if options.ref and options.val:
cleanSetup = options.ref
mysetup = options.val
- print "WARNING: You have specified a dedicated release as reference %s and as validation %s release, Your local testare will not be considered!!!" %(cleanSetup, mysetup)
- print "this option is mainly designed for comparing release versions!!"
+ logging.info("WARNING: You have specified a dedicated release as reference %s and as validation %s release, Your local testare will not be considered!!!" %(cleanSetup, mysetup))
+ logging.info("this option is mainly designed for comparing release versions!!")
else:
list_patch_packages()
@@ -346,7 +441,7 @@ def main():
########### Define and run jobs
mythreads={}
# mysetup=mysetup+",builds"
- print "------------------ Run Athena q-test jobs---------------"
+ logging.info("------------------ Run Athena q-test jobs---------------" )
if RunFast:
@@ -390,10 +485,11 @@ def main():
mythreads[q+"_patched"].join()
#Run post-processing tests
+
for qtest in qTestsToRun:
q=str(qtest)
- print "-----------------------------------------------------"
- print "----------- Post-processing of "+q+" Test -----------"
+ logging.info("-----------------------------------------------------" )
+ logging.info("----------- Post-processing of "+q+" Test -----------" )
QTestsFailedOrPassed(q,qTestsToRun,CleanRunHeadDir,UniqName)
@@ -411,8 +507,10 @@ def main():
RunTest(q,qTestsToRun,"Virtual Memory" ,"VmSize" ,"kBytes" ,4,0.2,CleanRunHeadDir,UniqName)
RunTest(q,qTestsToRun,"Memory Leak" ,"leakperevt_evt11","kBytes/event",7,0.05,CleanRunHeadDir,UniqName)
+
+ RunWARNINGSTest(q,qTestsToRun,CleanRunHeadDir,UniqName)
-# RunHistTest(q,CleanRunHeadDir,UniqName)
+# RunHistTest(q,CleanRunHeadDir,UniqName)
if __name__ == '__main__':
main()
diff --git a/Tools/PROCTools/python/UploadTfAMITag.py b/Tools/PROCTools/python/UploadTfAMITag.py
index df39412d310..914aa4cc944 100755
--- a/Tools/PROCTools/python/UploadTfAMITag.py
+++ b/Tools/PROCTools/python/UploadTfAMITag.py
@@ -17,6 +17,7 @@ cmProjTag = "data[0-9][0-9]_(cos|1beam|comm)" # only for commissioning
cpProjTag = "data[0-9][0-9]_(cos|1beam|.*eV|comm)" #For commissioning or pp but not HI
pcProjTag = "data[0-9][0-9]_(1beam|.*eV|comm)" #For commissioning or pp but not cosmics
cphipProjTag = "data[0-9][0-9]_(cos|1beam|.*eV|comm|hip)" # Above plus hip, i.e. all bar Pb-Pb HI
+cphiProjTag = "data[0-9][0-9]_(cos|1beam|.*eV|comm|hi)" # Above plus hi, HI
pphipProjTag = "data[0-9][0-9]_(.*eV|hip)" # pp or hip, for hip outputs in pp
#Setup script locations
@@ -30,42 +31,106 @@ specialT0Setup = specialT0Setup_Oracle # By Default
OutputsVsStreams = {
# The basics:
- 'outputESDFile': {'dstype': '!replace RAW ESD', 'ifMatch': '(?!.*DRAW.*)', 'HumanOutputs': 'always produced, except for DRAW input'},
- 'outputAODFile': {'dstype': '!replace RAW AOD', 'ifMatch': cpProjTag, 'HumanOutputs': 'always produced except for ZeroBias stream.'},
+ ## modification for pp reference run - only produce for the express stream
+ #'outputESDFile': {'dstype': '!replace RAW ESD', 'ifMatch': '(?!.*DRAW.*)', 'HumanOutputs': 'always produced, except for DRAW input'},
+ ## special version for 2 part of HI
+ #'outputESDFile': {'dstype': '!replace RAW ESD', 'ifMatch': '(?!.*DRAW.*)(.*express.*|.*L1Calo.*|.*ZeroBias.*|.*CosmicCalo.*|.*Late.*)', 'HumanOutputs': 'always produced, except for DRAW input'},
+ ## previous pp setup
+ ##'outputESDFile': {'dstype': '!replace RAW ESD', 'ifMatch': '(?!.*DRAW.*)(?!.*physics_Main.*)', 'HumanOutputs': 'always produced, except for DRAW input and physics_Main'},
+ ## setup for cosmics
+ #'outputESDFile': {'dstype': '!replace RAW ESD', 'ifMatch': '(?!.*DRAW.*)(?!.*IDCosmic.*)', 'HumanOutputs': 'always produced, except for DRAW input and IDCosmic'},
+ #'outputESDFile': {'dstype': '!replace RAW ESD', 'ifMatch': '(?!.*DRAW.*)', 'HumanOutputs': 'always produced, except for DRAW'},
+ #'outputESDFile': {'dstype': '!replace RAW ESD', 'ifMatch': '(?!.*DRAW.*)(?!.*physics_Main.*)', 'HumanOutputs': 'always produced, except for DRAW input and physics_Main'},
+ 'outputESDFile': {'dstype': '!replace RAW ESD', 'ifMatch': '(?!.*DRAW.*)(?!.(.*physics_Main\..*|.*Background.*|.*L1Topo.*))', 'HumanOutputs': 'always produced, except for DRAW input and physics_Main'},
+ 'outputAODFile': {'dstype': '!replace RAW AOD', 'ifMatch': cphiProjTag+'(?!.*DRAW_RPVLL.*)(?!.*Background.*)', 'HumanOutputs': 'always produced except for DRAW_RPVLL.'},
'outputTAGFile': {'dstype': 'TAG', 'ifMatch': HIProjTag+'(?!.(.*DRAW.*))', 'HumanOutputs': 'Produced in AOD merging'},
- 'outputHISTFile': {'dstype': 'HIST', 'ifMatch': '(?!.(.*DRAW.*|.*debugrec.*))', 'HumanOutputs': 'always produced except for debug stream'}, # note was disabled for Pb-Pb HardProbes
-
+ 'outputHISTFile': {'dstype': 'HIST', 'ifMatch': '(?!.(.*DRAW.*|.*debugrec.*))', 'HumanOutputs': 'always produced except for DRAW and debugrec'}, # note was disabled for Pb-Pb HardProbes
+ 'outputRDOFile': {'dstype': 'RDO', 'ifMatch': cphiProjTag, 'HumanOutputs': 'always produced.'},
# NTuples
- 'outputNTUP_MUONCALIBFile': {'dstype': 'NTUP_MUONCALIB', 'ifMatch': cpProjTag+'(?!.*DRAW.*)(.*physics_IDCosmic\..*|.*physics_CosmicMuons\..*)',
- 'HumanOutputs': 'produced for IDCosmic and CosmicMuons streams'},
+# 'outputNTUP_MUONCALIBFile': {'dstype': 'NTUP_MUONCALIB', 'ifMatch': cpProjTag+'(?!.*DRAW.*)(.*physics_IDCosmic\..*|.*physics_CosmicMuons\..*)',
+# 'HumanOutputs': 'produced for IDCosmic and CosmicMuons streams'},
+
+### put it to all streams for M10 tags
+# 'outputNTUP_MUONCALIBFile': {'dstype': 'NTUP_MUONCALIB', 'ifMatch': pcProjTag+'(?!.*DRAW.*)(?!.*Background.*)',
+# 'HumanOutputs': 'always produced except for DRAW'},
# DRAW production
- 'outputDRAW_ZMUMUFile': {'dstype': 'DRAW_ZMUMU', 'ifMatch': pcProjTag+'(?!.*DRAW.*)(.*physics_Main\..*)',
+ 'outputDRAW_ZMUMUFile': {'dstype': 'DRAW_ZMUMU', 'ifMatch': ppProjTag+'(?!.*DRAW.*)(.*physics_Main\..*)',
'HumanOutputs': 'produced for collision runs, for the physics_Main stream.'},
- 'outputDRAW_TAUMUHFile' : {'dstype': 'DRAW_TAUMUH', 'ifMatch': pcProjTag+'(?!.*DRAW.*)(.*physics_Main\..*)',
+ 'outputDRAW_TAUMUHFile' : {'dstype': 'DRAW_TAUMUH', 'ifMatch': ppProjTag+'(?!.*DRAW.*)(.*physics_Main\..*)',
'HumanOutputs': 'produced for collision runs, for the physics_Main stream.'},
### These don't work in 20.1.5.4, code updates from Paul coming soon
### Added three new outputs in 20.1.5.5 - DRAW_EGZ, DRAW_EGJPSI, DRAW_EMU
- 'outputDRAW_EGZFile' : {'dstype': 'DRAW_EGZ', 'ifMatch': pcProjTag+'(?!.*DRAW.*)(.*physics_Main\..*)',
+ 'outputDRAW_EGZFile' : {'dstype': 'DRAW_EGZ', 'ifMatch': ppProjTag+'(?!.*DRAW.*)(.*physics_Main\..*)',
'HumanOutputs': 'produced for collision runs, for the physics_Main stream.'},
#'outputDRAW_EGJPSIFile' : {'dstype': 'DRAW_EGJPSI', 'ifMatch': pcProjTag+'(?!.*DRAW.*)(.*physics_Main\..*)',
# 'HumanOutputs': 'produced for collision runs, for the physics_Main stream.'},
- 'outputDRAW_EMUFile' : {'dstype': 'DRAW_EMU', 'ifMatch': pcProjTag+'(?!.*DRAW.*)(.*physics_Main\..*)',
- 'HumanOutputs': 'produced for collision runs, for the physics_Main stream.'},
+ 'outputDRAW_EMUFile' : {'dstype': 'DRAW_EMU', 'ifMatch': ppProjTag+'(?!.*DRAW.*)(.*physics_Main\..*)',
+ 'HumanOutputs': 'produced for collision runs, for the physics_Main stream.'},
+
+ 'outputDRAW_RPVLLFile' : {'dstype': 'DRAW_RPVLL', 'ifMatch': ppProjTag+'(?!.*DRAW.*)(.*physics_Main\..*)',
+ 'HumanOutputs': 'produced for collision runs, for the physics_Main stream.'},
# Special reconstruction outputs for DRAW
- #'outputDESDM_MSPerfFile' : {'dstype': 'DESDM_ZMCP', 'ifMatch': cpProjTag+'(.*DRAW_ZMUMU.*)',
- # 'HumanOutputs': 'produced when reconstructing DRAW_ZMUMU'},
- 'outputDESDM_ALLCELLSFile': {'dstype': 'replace RAW ESDM', 'ifMatch': cpProjTag+'(.*DRAW.*)(?!.*TAUMUH.*)',
+# 'outputDESDM_MSPerfFile' : {'dstype': '!replace RAW DESDM_MCP DDESDM_MCP_ZMUMU DESDM_ZMCP', 'ifMatch': pcProjTag+'(.*physics_Main\..*|.*CosmicMuons.*)(.*\.RAW\.*|.*DRAW_ZMUMU.*)',
+# 'HumanOutputs': 'produced when reconstructing DRAW_ZMUMU'},
+# 'outputDESDM_MSPerfFile' : {'dstype': 'DESDM_ZMCP', 'ifMatch': pcProjTag+'(.*DRAW_ZMUMU.*)',
+# 'HumanOutputs': 'produced when reconstructing DRAW_ZMUMU'},
+ 'outputDESDM_ALLCELLSFile': {'dstype': '!replace RAW ESDM', 'ifMatch': pcProjTag+'(?!.*DRAW_TAUMUH.*)(.*DRAW.*)',
'HumanOutputs': 'produced when reconstructing all DRAW formats'},
+# 'outputDESDM_MSPerfFile' : {'dstype': 'DESDM_MCP', 'ifMatch': pcProjTag+'(?!.*DRAW.*)(.*physics_Main\..*|.*CosmicMuons.*)',
+# 'HumanOutputs': 'produced when reconstructing RAW'},
+ 'outputDESDM_MCPFile' : {'dstype': 'DESDM_MCP', 'ifMatch': pcProjTag+'(?!.*DRAW.*)(.*physics_Main\..*|.*CosmicMuons.*)',
+ 'HumanOutputs': 'produced when reconstructing RAW'},
+
+ 'outputDESDM_EXOTHIPFile' : {'dstype': 'DESDM_EXOTHIP', 'ifMatch': pcProjTag+'(?!.*DRAW.*)(.*physics_Main\..*)',
+ 'HumanOutputs': 'produced when reconstructing RAW'},
+
+# 'outputDESDM_IDALIGNFile' : {'dstype': '!replace RAW ESDM', 'ifMatch': pcProjTag+'(.*\.DRAW_EGZ\.*|.*DRAW_ZMUMU.*)(.*physics_Main.*)',
+ 'outputDESDM_IDALIGNFile' : {'dstype': '!replace RAW ESDM', 'ifMatch': pcProjTag+'(.*physics_Main.*)(.*\.DRAW_EGZ\.*|.*DRAW_ZMUMU.*)',
+ 'HumanOutputs': 'produced when reconstructing DRAW_EGZ and DRAW_ZMUMU formats'},
+
# DESDs made from full ESD files
+ 'outputDAOD_IDNCBFile' : {'dstype': 'DAOD_IDNCB', 'ifMatch': cpProjTag+'(?!.*DRAW.*)(.*Background.*)',
+ 'HumanOutputs': 'produced for Background streams.'},
+
'outputDAOD_IDTRKVALIDFile' : {'dstype': 'DAOD_IDTRKVALID', 'ifMatch': cpProjTag+'(?!.*DRAW.*)(.*MinBias.*|.*IDCosmic.*)',
'HumanOutputs': 'produced for MinBias and IDCosmic streams.'},
- 'outputDAOD_IDTRKLUMIFile' : {'dstype': 'DAOD_IDTRKLUMI', 'ifMatch': pcProjTag+'(?!.*DRAW.*)(.*VdM.*|.*PixelBeam.*)',
+
+ 'outputDAOD_IDTRKLUMIFile' : {'dstype': 'DAOD_IDTRKLUMI', 'ifMatch': pcProjTag+'(?!.*DRAW.*)(.*VdM.*|.*PixelBeam*.)',
'HumanOutputs': 'produced for calibration streams.'},
+
+ 'outputDAOD_IDPIXLUMIFile' : {'dstype': 'DAOD_IDPIXLUMI', 'ifMatch': pcProjTag+'(?!.*DRAW.*)(.*VdM.*|.*PixelBeam*.)',
+ 'HumanOutputs': 'produced for PixelBeam and VdM calibration streams.'},
+
+ 'outputDAOD_SCTVALIDFile': {'dstype': 'DAOD_SCTVALID', 'ifMatch': cphiProjTag+'(?!.*DRAW.*)(.*express_express.*)',
+ 'HumanOutputs': 'produced for express stream only'},
+
+
+ 'outputDAOD_IDTIDEFile' : {'dstype': 'DAOD_IDTIDE', 'ifMatch': ppProjTag+'(?!.*DRAW.*)(.*physics_Main.*)',
+ 'HumanOutputs': 'produced for collision runs, for physics_Main stream.'},
+
'outputDESDM_RPVLLFile': {'dstype': 'DESDM_RPVLL','ifMatch': ppProjTag+'(?!.*DRAW.*)(.*physics_Main.*)',
- 'HumanOutputs': 'produced for collision runs, for physics_JetTauEtMiss, physics_Egamma and physics_Muons streams.'},
+ 'HumanOutputs': 'produced for collision runs, for physics_Main stream.'},
+
+ 'outputDESDM_SLTTMUFile': {'dstype': 'DESDM_SLTTMU','ifMatch': ppProjTag+'(?!.*DRAW.*)(.*physics_Main.*)',
+ 'HumanOutputs': 'produced for collision runs, for physics_Main stream.'},
+
+ 'outputDESDM_PHOJETFile': {'dstype': 'DESDM_PHOJET','ifMatch': ppProjTag+'(?!.*DRAW.*)(.*physics_Main.*)',
+ 'HumanOutputs': 'produced for collision runs, for physics_Main stream.'},
+
+ 'outputDESDM_CALJETFile': {'dstype': 'DESDM_CALJET','ifMatch': ppProjTag+'(?!.*DRAW.*)(.*physics_Main.*)',
+ 'HumanOutputs': 'produced for collision runs, for physics_Main.'},
+
+ 'outputDESDM_SGLELFile': {'dstype': 'DESDM_SGLEL','ifMatch': ppProjTag+'(?!.*DRAW.*)(.*physics_Main.*)',
+ 'HumanOutputs': 'produced for collision runs, for physics_Main stream.'},
+
+ 'outputDESDM_EGAMMAFile': {'dstype': 'DESDM_EGAMMA','ifMatch': ppProjTag+'(?!.*DRAW.*)(.*physics_Main.*)',
+ 'HumanOutputs': 'produced for collision runs, for physics_Main stream.'},
+
+ 'outputDESDM_TILEMUFile': {'dstype': 'DESDM_TILEMU','ifMatch': ppProjTag+'(?!.*DRAW.*)(.*physics_Main.*)',
+ 'HumanOutputs': 'produced for collision runs, for physics_Main stream.'},
}
@@ -118,14 +183,14 @@ def GetProcessConfigs(release,patcharea):
# AOD merging
processConfigs['aodmerge'] = {
'inputs': {'inputAODFile': {}},
- 'outputs': {'outputAOD_MRGFile': {'dstype': 'AOD'},
- 'outputTAGFile': {'dstype': 'TAG'}
+ 'outputs': {'outputAOD_MRGFile': {'dstype': 'AOD'}
+ #'outputTAGFile': {'dstype': 'TAG', 'ifMatch': '(?!.*calibration.*)'}
},
'phconfig': {'ignoreErrors': 'True', 'autoConfiguration': 'everything'},
'transformation': 'AODMerge_tf.py',
'tasktransinfo': {'trfpath': 'AODMerge_tf.py',
'trfsetupcmd': setupScript+' '+pa+' '+rel+' '+specialT0Setup },
- 'description': 'Produces merged AODs plus associated physics TAGs with AODMerge_tf.py. '}
+ 'description': 'Produces merged AODs with AODMerge_tf.py. '}
# DAOD merging
processConfigs['daodmerge'] = {
@@ -135,17 +200,17 @@ def GetProcessConfigs(release,patcharea):
'transformation': 'AODMerge_tf.py',
'tasktransinfo': {'trfpath': 'AODMerge_tf.py',
'trfsetupcmd': setupScript+' '+pa+' '+rel+' '+specialT0Setup },
- 'description': 'Produces merged DAODs plus associated physics TAGs with AODMerge_tf.py. '}
+ 'description': 'Produces merged DAODs with AODMerge_tf.py. '}
# DPD merging
processConfigs['dpdmerge'] = {
'inputs': {'inputESDFile': {}},
- 'outputs': {'outputESDFile': {'dstype': '!likeinput' }},
- 'phconfig': {'ignoreerrors': 'ALL', 'autoConfiguration': 'everything', 'preExec': 'rec.doDPD.set_Value_and_Lock(False)'},
- 'transformation': 'Merging_tf.py',
- 'tasktransinfo': {'trfpath': 'Merging_tf.py',
+ 'outputs': {'outputESD_MRGFile': {'dstype': '!likeinput' }},
+ 'phconfig': {'ignoreErrors': 'True', 'autoConfiguration': 'everything'},
+ 'transformation': 'ESDMerge_tf.py',
+ 'tasktransinfo': {'trfpath': 'ESDMerge_tf.py',
'trfsetupcmd': setupScript+' '+pa+' '+rel+' '+specialT0Setup },
- 'description': 'Produces merged DPDs with Merging_trf. ' }
+ 'description': 'Produces merged DPDs with ESDMerge_tf. ' }
# Fast Physics Monitoring Ntuple
@@ -290,17 +355,16 @@ if __name__ == '__main__':
print startingDic
- from PATJobTransforms.Configuration import ConfigDic
didConditionsUpdate = False
for key in startingDic.keys():
- if ConfigDic.has_key(key) and hasattr(ConfigDic[key],"isInput"):
+ if key.startswith("input"):
if amiTag.startswith("q"): # only write the input file for q-tags
inputDic[key]=startingDic[key]
else:
inputDic[key]={}
#print "inputDic[%s] = %s" % (key, inputDic[key])
- elif ConfigDic.has_key(key) and hasattr(ConfigDic[key],"isOutput") and (key!="tmpESD" and key!="tmpAOD"):
+ elif key.startswith("output"):
try:
#print "key: " , key , " isOutput"
outputDic[key]=OutputsVsStreams[key]
@@ -401,9 +465,11 @@ if __name__ == '__main__':
c['trfsetupcmd']=str(moreInfoDic['tasktransinfo']['trfsetupcmd'])
c['moreInfo']=str(moreInfoDic.__str__())
if amiTag.startswith("f") or amiTag.startswith("x") or amiTag.startswith("q") or amiTag.startswith("v") or amiTag.startswith("c"):
- c['Geometry']='Auto-config. See dataset info.'
- if not amiTag.startswith("x"):
- c['ConditionsTag']='Auto-config. See dataset info.'
+ c['Geometry']=startingDic['geometryVersion']['all']
+ #c['Geometry']='Auto-config. See dataset info.'
+ #if not amiTag.startswith("x"):
+ c['ConditionsTag']=configDic['conditionsTag']['all']
+ #c['ConditionsTag']='Auto-config. See dataset info.'
#print c['Human-readable outputs']
diff --git a/Tools/PROCTools/python/compareTCTs.py b/Tools/PROCTools/python/compareTCTs.py
index f14565406bb..6ed745f2e98 100755
--- a/Tools/PROCTools/python/compareTCTs.py
+++ b/Tools/PROCTools/python/compareTCTs.py
@@ -243,7 +243,7 @@ if __name__ == "__main__":
if(v.startswith("/eos")): v = "root://eosatlas/"+v
if not diffroot: stat=diffPoolFiles(r,v,details)
else:
- stat=os.system("acmd.py diff-root "+r+" "+v+" --error-mode resilient --ignore-leaves HITStoRDO_timings RecoTimingObj_p1_HITStoRDO_timings RecoTimingObj_p1_RAWtoESD_mems RecoTimingObj_p1_RAWtoESD_timings RAWtoESD_mems RAWtoESD_timings ESDtoAOD_mems ESDtoAOD_timings --entries 10 > tmp.txt")
+ stat=os.system("acmd.py diff-root "+r+" "+v+" --error-mode resilient --ignore-leaves HITStoRDO_timings RecoTimingObj_p1_HITStoRDO_timings RecoTimingObj_p1_RAWtoESD_mems RecoTimingObj_p1_RAWtoESD_timings RAWtoESD_mems RAWtoESD_timings ESDtoAOD_mems ESDtoAOD_timings RAWtoALL_mems RAWtoALL_timings RecoTimingObj_p1_RAWtoALL_mems RecoTimingObj_p1_RAWtoALL_timings --entries 10 > tmp.txt")
os.system("cat tmp.txt|grep -v sync")
os.system("rm -f tmp.txt")
identical=not stat
diff --git a/Tools/PROCTools/python/outputTest_v2.py b/Tools/PROCTools/python/outputTest_v2.py
index 651c1e2d6ea..1e710820cae 100755
--- a/Tools/PROCTools/python/outputTest_v2.py
+++ b/Tools/PROCTools/python/outputTest_v2.py
@@ -9,7 +9,7 @@ from optparse import OptionParser
physicsStreams = ["physics_Main","physics_MinBias","physics_MinBiasOverlay","physics_ZeroBias","physics_HardProbes",
"physics_TauOverlay","physics_CosmicCalo", "physics_CosmicMuons","physics_IDCosmic",
- "physics_HLT_IDCosmic","physics_Standby","physics_Late","physics_L1Calo",
+ "physics_HLT_IDCosmic","physics_Standby","physics_Late","physics_L1Calo","physics_L1Topo",
"express_express"]
--
GitLab