diff --git a/Control/AthenaMonitoring/CMakeLists.txt b/Control/AthenaMonitoring/CMakeLists.txt index 5829d66fa4c0a7d6f5b95072ebe0e9ed8f9eafa6..1d27ee52c1779e53a97650d4fae2bb1e9cc68cb7 100644 --- a/Control/AthenaMonitoring/CMakeLists.txt +++ b/Control/AthenaMonitoring/CMakeLists.txt @@ -78,4 +78,4 @@ atlas_add_component( atlas_install_python_modules( python/*.py POST_BUILD_CMD ${ATLAS_FLAKE8} ) atlas_install_joboptions( share/*.py ) -atlas_install_scripts( share/Run3DQTestingDriver.py ) \ No newline at end of file +atlas_install_scripts( share/Run3DQTestingDriver.py share/hist_file_dump.py share/hist_diff.sh ) diff --git a/Control/AthenaMonitoring/share/hist_diff.sh b/Control/AthenaMonitoring/share/hist_diff.sh new file mode 100755 index 0000000000000000000000000000000000000000..f75ff83fd00970d487cfd1223ae10244b62d6673 --- /dev/null +++ b/Control/AthenaMonitoring/share/hist_diff.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +if [[ ! -f $1 ]] ; then { echo "$1 does not exist"; exit 1; } ; fi +if [[ ! -f $2 ]] ; then { echo "$2 does not exist"; exit 1; } ; fi + +LOG1=$(mktemp FILE1.XXXXXXX) +LOG2=$(mktemp FILE2.XXXXXXX) + +hist_file_dump.py $1 --hash > $LOG1 +RV=$? +if [ $RV != 0 ]; then { echo "Failure dumping $1"; rm -f $LOG1 $LOG2; exit $RV ; }; fi +hist_file_dump.py $2 --hash > $LOG2 +RV=$? +if [ $RV != 0 ]; then { echo "Failure dumping $2"; rm -f $LOG1 $LOG2; exit $RV ; }; fi + +diff $LOG1 $LOG2 +RV=$? +if [ $RV != 0 ]; then + echo "$1 <" + echo "$2 >" + echo "Files differ" +else + echo "Files match" +fi + +rm -f $LOG1 $LOG2 +exit $RV diff --git a/Control/AthenaMonitoring/share/hist_file_dump.py b/Control/AthenaMonitoring/share/hist_file_dump.py new file mode 100755 index 0000000000000000000000000000000000000000..2c4109135d834a7acc3feaf206daeb9153cd6648 --- /dev/null +++ b/Control/AthenaMonitoring/share/hist_file_dump.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python + +import ROOT +import sys, os, operator +import argparse +import zlib + +parser=argparse.ArgumentParser() +parser.add_argument('filename', + help='Input HIST file name') +parser.add_argument('-r', '--rankorder', default='onfile', + choices=['onfile', 'uncompressed', 'name'], + help='rankorder is "onfile" (default), "uncompressed" or "name"') +parser.add_argument('-p', '--path', + help='Only look under this directory') +parser.add_argument('--hash', action='store_true', + help='Print hashes of objects') +parser.add_argument('--metadata', action='store_true', + help='Include metadata trees') +parser.add_argument('--no_onfile', action='store_true', + help="Don't show on file size") +parser.add_argument('--no_inmem', action='store_true', + help="Don't show in memory size") +args=parser.parse_args() + +ordering = args.rankorder + +accounting = {}; hashes = {}; types = {} + +ROOT.gInterpreter.LoadText("UInt_t bufferhash(TKey* key) { key->SetBuffer(); key->ReadFile(); UInt_t rv = TString::Hash(key->GetBuffer()+key->GetKeylen(), key->GetNbytes()-key->GetKeylen()); key->DeleteBuffer(); return rv; }") +ROOT.gInterpreter.LoadText("void* getbuffer(TKey* key) { key->SetBuffer(); key->ReadFile(); return (void*) (key->GetBuffer()+key->GetKeylen()); }") +ROOT.gInterpreter.LoadText("UInt_t bufferhash2(TKey* key) { TObject* obj = key->ReadObj(); TMessage msg(kMESS_OBJECT); msg.WriteObject(obj); UInt_t rv = TString::Hash(msg.Buffer(), msg.Length()); delete obj; return rv; }") +ROOT.gInterpreter.LoadText("UInt_t bufferhash3(TKey* key) { TObject* obj = key->ReadObj(); UInt_t rv = obj->Hash(); delete obj; return rv; }") +ROOT.gSystem.Load('libDataQualityUtils') + +def dumpdir(d): + thispath = d.GetPath() + if ':' in thispath: + thispath = thispath.split(':', 1)[1] + #print thispath + subdirs = [] + for k in d.GetListOfKeys(): + if not args.metadata and k.GetName() == 'metadata' and k.GetClassName() == 'TTree': + continue + if k.GetClassName().startswith('TDirectory'): + subdirs.append(k) + else: + if args.hash: + #lhash = ROOT.bufferhash(k) + #objsize = (k.GetNbytes()-k.GetKeylen())/8 + #print (k.GetNbytes()-k.GetKeylen())/8. + #buf = ROOT.getbuffer(k); buf.SetSize(objsize) + #print buf[objsize-1], objsize + #lhash = zlib.adler32(str(buf)) + #k.DeleteBuffer() + #obj=k.ReadObj(); + #tm=ROOT.TMessage(ROOT.TMessage.kMESS_OBJECT) + #tm.WriteObject(obj) + # This is what we _were_ doing + #lhash = ROOT.bufferhash2(k) + # How about this? + #lhash = ROOT.bufferhash3(k) + obj = k.ReadObj(); lhash = obj.Hash(); del obj + else: + lhash = 0 + idxname = os.path.join(thispath, k.GetName()) + accounting[idxname] = (k.GetObjlen(), k.GetNbytes()-k.GetKeylen()) + hashes[idxname] = lhash + types[idxname] = k.GetClassName() + #print '%s,' % os.path.join(thispath, k.GetName()), + #obj = k.ReadObj(); obj.IsA().Destructor(obj) + #print 'OK' + for k in subdirs: + dumpdir(k.ReadObj()) + +f = ROOT.TFile.Open(args.filename) +if args.path: + d = f.Get(args.path.rstrip('/')) + if not d: + print "Can't access path", args.path, "- exiting" + sys.exit(1) +else: + d = f +dumpdir(d) + +#sortedl = sorted(accounting.items(), key=operator.itemgetter(0,1), reverse=True) +if ordering == 'onfile': + key=lambda x: (x[1][1], x[1][0], x[0]) +elif ordering == 'uncompressed': + key=lambda x: (x[1][0], x[1][1], x[0]) +else: + key=lambda x: (x[0], x[1][1], x[1][0]) +sortedl = sorted(accounting.items(), key=key, reverse=True) +if args.hash: + print '\n'.join(('%s %s: ' + + ('%d uncompressed' % b if not args.no_inmem else '') + + (', %d on file ' % c if not args.no_onfile else '') + + '(hash %s)') + % (types[a], a, hashes[a]) for a, (b, c) in sortedl) +else: + print '\n'.join(('%s %s: ' + + ('%d uncompressed' % b if not args.no_inmem else '') + + (', %d on file' % c if not args.no_onfile else '')) + % (types[a], a) for a, (b, c) in sortedl) + #print '\n'.join('%s %s: %d uncompressed, %d on file' % (types[a], a, b, c) for a, (b, c) in sortedl) + diff --git a/Control/AthenaMonitoring/test/test_run3dq_r21_esd.sh b/Control/AthenaMonitoring/test/test_run3dq_r21_esd.sh new file mode 100755 index 0000000000000000000000000000000000000000..85055a596f6a9d1ad0a1eedc2a50507f30ea2ffc --- /dev/null +++ b/Control/AthenaMonitoring/test/test_run3dq_r21_esd.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# art-description: ESD->HIST, R21 data ESD +# art-type: grid +# art-include: master/Athena +# art-output: ExampleMonitorOutput.root + +Run3DQTestingDriver.py 'Input.Files=["/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/Tier0ChainTests/q431/21.0/myESD.pool.root"]' DQ.Steering.doHLTMon=False --postExec 'cfg.getEventAlgo("LArCollisionTimeAlg").cutIteration=False' + +echo "art-result: $? HIST_Creation" + +ArtPackage=$1 +ArtJobName=$2 +art.py download ${ArtPackage} ${ArtJobName} +hist_diff.sh ExampleMonitorOutput.root ./ref-*/ExampleMonitorOutput.root +echo "art-result: $? HIST_Diff" diff --git a/Control/AthenaMonitoring/test/test_run3dq_r21_esd_mc.sh b/Control/AthenaMonitoring/test/test_run3dq_r21_esd_mc.sh new file mode 100755 index 0000000000000000000000000000000000000000..f8f4ac0c73c56241222fbbdd4618e8f181fd3621 --- /dev/null +++ b/Control/AthenaMonitoring/test/test_run3dq_r21_esd_mc.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# art-description: ESD->HIST, R21 MC ESD +# art-type: grid +# art-include: master/Athena +# art-output: ExampleMonitorOutput.root + +Run3DQTestingDriver.py 'Input.Files=["/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/Tier0ChainTests/q221/21.0/myESD.pool.root"]' DQ.Steering.doHLTMon=False --postExec 'cfg.getEventAlgo("LArCollisionTimeAlg").cutIteration=False' + +echo "art-result: $? HIST_Creation" + +ArtPackage=$1 +ArtJobName=$2 +art.py download ${ArtPackage} ${ArtJobName} +hist_diff.sh ExampleMonitorOutput.root ./ref-*/ExampleMonitorOutput.root +echo "art-result: $? HIST_Diff" diff --git a/Control/AthenaMonitoring/test/test_run3dq_r21_esd_mt.sh b/Control/AthenaMonitoring/test/test_run3dq_r21_esd_mt.sh new file mode 100755 index 0000000000000000000000000000000000000000..9e580205c6e64675405a87b63e30b822ea440851 --- /dev/null +++ b/Control/AthenaMonitoring/test/test_run3dq_r21_esd_mt.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# art-description: ESD->HIST, R21 data ESD, MT +# art-type: grid +# art-include: master/Athena +# art-output: ExampleMonitorOutput.root + +Run3DQTestingDriver.py 'Input.Files=["/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/Tier0ChainTests/q431/21.0/myESD.pool.root"]' DQ.Steering.doHLTMon=False Concurrency.NumThreads=1 Concurrency.NumConcurrentEvents=1 --postExec 'cfg.getEventAlgo("LArCollisionTimeAlg").cutIteration=False' + +echo "art-result: $? HIST_Creation" + +ArtPackage=$1 +ArtJobName=$2 +art.py download ${ArtPackage} ${ArtJobName} +hist_diff.sh ExampleMonitorOutput.root ./ref-*/ExampleMonitorOutput.root +echo "art-result: $? HIST_Diff" diff --git a/Control/AthenaMonitoring/test/test_run3dq_r22_aod_trigger.sh b/Control/AthenaMonitoring/test/test_run3dq_r22_aod_trigger.sh new file mode 100755 index 0000000000000000000000000000000000000000..8e7b5ce43a837e661edd85ab70cae312f111c29e --- /dev/null +++ b/Control/AthenaMonitoring/test/test_run3dq_r22_aod_trigger.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# art-description: AOD->HIST, R22 MC, Trigger Only +# art-type: grid +# art-include: master/Athena +# art-output: ExampleMonitorOutput.root + +art.py download TrigAnalysisTest test_trigAna_q221_RDOtoAOD_mt1_grid.sh +AODFILE=(./ref-*/AOD.pool.root) +Run3DQTestingDriver.py 'Input.Files=["'${AODFILE}'"]' DQ.Steering.doHLTMon=True --dqOffByDefault + +echo "art-result: $? HIST_Creation" +rm -rf ref-* + +ArtPackage=$1 +ArtJobName=$2 +art.py download ${ArtPackage} ${ArtJobName} +hist_diff.sh ExampleMonitorOutput.root ./ref-*/ExampleMonitorOutput.root +echo "art-result: $? HIST_Diff" diff --git a/Control/AthenaMonitoring/test/test_run3dq_r22_esd.sh b/Control/AthenaMonitoring/test/test_run3dq_r22_esd.sh new file mode 100755 index 0000000000000000000000000000000000000000..59c55230ab015d1ca7b7a33e0dc4f95ffbe9807d --- /dev/null +++ b/Control/AthenaMonitoring/test/test_run3dq_r22_esd.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# art-description: ESD->HIST, R22 data ESD +# art-type: grid +# art-include: master/Athena +# art-output: ExampleMonitorOutput.root + +art.py download Tier0ChainTests test_q431.sh +ESDFILE=(./ref-*/myESD.pool.root) +Run3DQTestingDriver.py 'Input.Files=["'${ESDFILE}'"]' DQ.Steering.doHLTMon=False + +echo "art-result: $? HIST_Creation" +rm -rf ref-* + +ArtPackage=$1 +ArtJobName=$2 +art.py download ${ArtPackage} ${ArtJobName} +hist_diff.sh ExampleMonitorOutput.root ./ref-*/ExampleMonitorOutput.root +echo "art-result: $? HIST_Diff"