Skip to content
Snippets Groups Projects
Commit c6084a00 authored by Antonio Limosani's avatar Antonio Limosani Committed by Graeme Stewart
Browse files

make it work in cmake build, add HIST test but don't allow it to run, add RDO...

make it work in cmake build, add HIST test but don't allow it to run, add RDO test (PROCTools-00-01-15)
parent e40e352e
No related branches found
No related tags found
No related merge requests found
......@@ -7,5 +7,5 @@ atlas_subdir( PROCTools )
# Install files from the package:
atlas_install_python_modules( python/*.py )
atlas_install_scripts( python/compareTCTs.py python/UploadAMITag.py python/UploadTfAMITag.py python/getCurrentCOMCONDTag.py python/RunTier0Tests.py python/ExtractEvents.py python/checkUPD1.py python/runDiffRootOnChanged.py python/provideTwikiSummary.py python/evaluateDiffRoot.py python/outputTest_v2.py python/CreateTierZeroArgdict.py )
atlas_install_scripts( python/compareTCTs.py python/UploadAMITag.py python/UploadTfAMITag.py python/getCurrentCOMCONDTag.py python/RunTier0Tests.py python/ExtractEvents.py python/checkUPD1.py python/runDiffRootOnChanged.py python/provideTwikiSummary.py python/evaluateDiffRoot.py python/outputTest_v2.py python/CreateTierZeroArgdict.py python/root_lsr_rank.py)
......@@ -7,5 +7,5 @@ use AtlasPolicy AtlasPolicy-*
private
apply_pattern declare_python_modules files="*.py"
apply_pattern declare_scripts files="-s=$(PROCTools_root)/python compareTCTs.py UploadAMITag.py UploadTfAMITag.py getCurrentCOMCONDTag.py RunTier0Tests.py ExtractEvents.py checkUPD1.py runDiffRootOnChanged.py provideTwikiSummary.py evaluateDiffRoot.py outputTest_v2.py CreateTierZeroArgdict.py"
apply_pattern declare_scripts files="-s=$(PROCTools_root)/python compareTCTs.py UploadAMITag.py UploadTfAMITag.py getCurrentCOMCONDTag.py RunTier0Tests.py ExtractEvents.py checkUPD1.py runDiffRootOnChanged.py provideTwikiSummary.py evaluateDiffRoot.py outputTest_v2.py CreateTierZeroArgdict.py root_lsr_rank.py"
end_private
......@@ -83,22 +83,32 @@ def GetReleaseSetup():
###############################
########### List patch packages
def list_patch_packages():
if 'TestArea' in os.environ and os.access(os.environ['TestArea'], os.R_OK):
print "Patch packages in your InstallArea that will be tested are:\n"
cmd = ['cmt', 'show', 'packages', os.environ['TestArea']]
cmtProc = subprocess.Popen(cmd, shell = False, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, bufsize = 1)
cmtOut = cmtProc.communicate()[0]
for line in cmtOut.split('\n'):
try:
if line.strip() == '':
continue
(package, packageVersion, packagePath) = line.split()
print '\t%s\n' % (packageVersion)
except ValueError:
print "Warning, unusual output from cmt: %s\n" % line
if 'CMTPATH' in os.environ:
if 'TestArea' in os.environ and os.access(os.environ['TestArea'], os.R_OK):
print "Patch packages in your InstallArea that will be tested are:\n"
cmd = ['cmt', 'show', 'packages', os.environ['TestArea']]
cmtProc = subprocess.Popen(cmd, shell = False, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, bufsize = 1)
cmtOut = cmtProc.communicate()[0]
for line in cmtOut.split('\n'):
try:
if line.strip() == '':
continue
(package, packageVersion, packagePath) = line.split()
print '\t%s\n' % (packageVersion)
except ValueError:
print "Warning, unusual output from cmt: %s\n" % line
elif 'CMAKE_PREFIX_PATH' in os.environ :
print "Patch packages in your build to be tested:\n"
myfilepath = os.environ['CMAKE_PREFIX_PATH'].split(":")[0]
fname = str(myfilepath) + '/packages.txt'
with open(fname) as fp:
for line in fp:
if '#' not in line:
print line
else:
print "A release has not been setup"
pass
###############################
########### Was the q test successful? To check simply count the number of lines containing the string "successful run"
......@@ -213,6 +223,35 @@ def RunTest(q,qTestsToRun,TestName,SearchString,MeasurementUnit,FieldNumber,Thre
pass
##########################################################################
def RunHistTest(q,CleanRunHeadDir,UniqID):
print "-----------------------------------------------------"
print "Running "+q+" HIST Comparison Test"
ref_file = CleanRunHeadDir+"/clean_run_"+q+"_"+UniqID+"/myHIST.root"
test_file = "./run_"+q+"/myHIST.root"
ref_outfile = CleanRunHeadDir+"/clean_run_"+q+"_"+UniqID+"/ref."+q+".HIST.chk.log"
test_outfile = "./run_"+q+"/test."+q+".HIST.chk.log"
ref_cmd = "root_lsr_rank.py "+ ref_file +" --hash >& "+ref_outfile
test_cmd = "root_lsr_rank.py "+ test_file +" --hash >& "+test_outfile
ref_out = subprocess.Popen(['/bin/bash', '-c',ref_cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
test_out = subprocess.Popen(['/bin/bash', '-c',test_cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
comparison_command = "diff "+ref_outfile+" "+test_outfile+" >& run_"+q+"/diff."+q+".HIST.log"
output,error = subprocess.Popen(['/bin/bash', '-c', comparison_command], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
f = open("run_"+q+"/diff."+q+".HIST.log", 'r')
print "The following are changes you've made to the HIST file: "
for line in f.readlines():
print line
f.close()
print "-----------------------------------------------------"
pass
##########################################################################
def main():
......@@ -357,6 +396,9 @@ def main():
print "----------- Post-processing of "+q+" Test -----------"
QTestsFailedOrPassed(q,qTestsToRun,CleanRunHeadDir,UniqName)
if 'q221' in q:
RunFrozenTier0PolicyTest(q,"RDO",10,CleanRunHeadDir,UniqName)
RunFrozenTier0PolicyTest(q,"ESD",10,CleanRunHeadDir,UniqName)
......@@ -370,6 +412,7 @@ def main():
RunTest(q,qTestsToRun,"Memory Leak" ,"leakperevt_evt11","kBytes/event",7,0.05,CleanRunHeadDir,UniqName)
# RunHistTest(q,CleanRunHeadDir,UniqName)
if __name__ == '__main__':
main()
......
#!/usr/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
## root_lsr_rank.py - Brief description of the purpose of this script (Has to be in PROC tools)
# When run as
# root_lsr_rank.py <HIST file> --hash
#
# it produces a listing of all objects in the HIST file and a hash value for each of them. This output can then be compared between the clean and patched runs to look
# for any changes in HIST output. Since it's entirely possible to change the HIST output unintentionally without changing the AOD/ESD etc., this catches a new class of potential errors.
# $Id$
import ROOT
import sys, os, operator
import argparse
import zlib
parser=argparse.ArgumentParser()
parser.add_argument('filename',
help='Input HIST file name')
parser.add_argument('-r', '--rankorder', default='onfile',
choices=['onfile', 'uncompressed', 'name'],
help='rankorder is "onfile" (default), "uncompressed" or "name"')
parser.add_argument('-p', '--path',
help='Only look under this directory')
parser.add_argument('--hash', action='store_true',
help='Print hashes of objects')
parser.add_argument('--metadata', action='store_true',
help='Include metadata trees')
args=parser.parse_args()
ordering = args.rankorder
accounting = {}; hashes = {}; types = {}
ROOT.gInterpreter.LoadText("UInt_t bufferhash(TKey* key) { key->SetBuffer(); key->ReadFile(); UInt_t rv = TString::Hash(key->GetBuffer()+key->GetKeylen(), key->GetNbytes()-key->GetKeylen()); key->DeleteBuffer(); return rv; }")
ROOT.gInterpreter.LoadText("void* getbuffer(TKey* key) { key->SetBuffer(); key->ReadFile(); return (void*) (key->GetBuffer()+key->GetKeylen()); }")
ROOT.gInterpreter.LoadText("UInt_t bufferhash2(TKey* key) { TObject* obj = key->ReadObj(); TMessage msg(kMESS_OBJECT); msg.WriteObject(obj); UInt_t rv = TString::Hash(msg.Buffer(), msg.Length()); delete obj; return rv; }")
def dumpdir(d):
thispath = d.GetPath()
if ':' in thispath:
thispath = thispath.split(':', 1)[1]
#print thispath
subdirs = []
for k in d.GetListOfKeys():
if not args.metadata and k.GetName() == 'metadata' and k.GetClassName() == 'TTree':
continue
if k.GetClassName().startswith('TDirectory'):
subdirs.append(k)
else:
if args.hash:
#lhash = ROOT.bufferhash(k)
#objsize = (k.GetNbytes()-k.GetKeylen())/8
#print (k.GetNbytes()-k.GetKeylen())/8.
#buf = ROOT.getbuffer(k); buf.SetSize(objsize)
#print buf[objsize-1], objsize
#lhash = zlib.adler32(str(buf))
#k.DeleteBuffer()
#obj=k.ReadObj();
#tm=ROOT.TMessage(ROOT.TMessage.kMESS_OBJECT)
#tm.WriteObject(obj)
lhash = ROOT.bufferhash2(k)
#obj.IsA().Destructor(obj)
#lhash = 0
else:
lhash = 0
idxname = os.path.join(thispath, k.GetName())
accounting[idxname] = (k.GetObjlen(), k.GetNbytes()-k.GetKeylen())
hashes[idxname] = lhash
types[idxname] = k.GetClassName()
#print '%s,' % os.path.join(thispath, k.GetName()),
#obj = k.ReadObj(); obj.IsA().Destructor(obj)
#print 'OK'
for k in subdirs:
dumpdir(k.ReadObj())
f = ROOT.TFile.Open(args.filename)
if args.path:
d = f.Get(args.path.rstrip('/'))
if not d:
print "Can't access path", args.path, "- exiting"
sys.exit(1)
else:
d = f
dumpdir(d)
#sortedl = sorted(accounting.items(), key=operator.itemgetter(0,1), reverse=True)
if ordering == 'onfile':
key=lambda x: (x[1][1], x[1][0], x[0])
elif ordering == 'uncompressed':
key=lambda x: (x[1][0], x[1][1], x[0])
else:
key=lambda x: (x[0], x[1][1], x[1][0])
sortedl = sorted(accounting.items(), key=key, reverse=True)
if args.hash:
print '\n'.join('%s %s: %d uncompressed, %d on file (hash %s)' % (types[a], a, b, c, hashes[a]) for a, (b, c) in sortedl)
else:
print '\n'.join('%s %s: %d uncompressed, %d on file' % (types[a], a, b, c) for a, (b, c) in sortedl)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment