From 008883b562c147a7cfb914974ba984122959dafa Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Tue, 5 Nov 2019 05:51:30 +0100 Subject: [PATCH 1/2] Scripts to help compare HIST files (focus on histograms, not trees) --- Control/AthenaMonitoring/CMakeLists.txt | 2 +- Control/AthenaMonitoring/share/hist_diff.sh | 27 +++++ .../AthenaMonitoring/share/hist_file_dump.py | 106 ++++++++++++++++++ 3 files changed, 134 insertions(+), 1 deletion(-) create mode 100755 Control/AthenaMonitoring/share/hist_diff.sh create mode 100755 Control/AthenaMonitoring/share/hist_file_dump.py diff --git a/Control/AthenaMonitoring/CMakeLists.txt b/Control/AthenaMonitoring/CMakeLists.txt index 5829d66fa4c..1d27ee52c17 100644 --- a/Control/AthenaMonitoring/CMakeLists.txt +++ b/Control/AthenaMonitoring/CMakeLists.txt @@ -78,4 +78,4 @@ atlas_add_component( atlas_install_python_modules( python/*.py POST_BUILD_CMD ${ATLAS_FLAKE8} ) atlas_install_joboptions( share/*.py ) -atlas_install_scripts( share/Run3DQTestingDriver.py ) \ No newline at end of file +atlas_install_scripts( share/Run3DQTestingDriver.py share/hist_file_dump.py share/hist_diff.sh ) diff --git a/Control/AthenaMonitoring/share/hist_diff.sh b/Control/AthenaMonitoring/share/hist_diff.sh new file mode 100755 index 00000000000..f75ff83fd00 --- /dev/null +++ b/Control/AthenaMonitoring/share/hist_diff.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +if [[ ! -f $1 ]] ; then { echo "$1 does not exist"; exit 1; } ; fi +if [[ ! -f $2 ]] ; then { echo "$2 does not exist"; exit 1; } ; fi + +LOG1=$(mktemp FILE1.XXXXXXX) +LOG2=$(mktemp FILE2.XXXXXXX) + +hist_file_dump.py $1 --hash > $LOG1 +RV=$? +if [ $RV != 0 ]; then { echo "Failure dumping $1"; rm -f $LOG1 $LOG2; exit $RV ; }; fi +hist_file_dump.py $2 --hash > $LOG2 +RV=$? +if [ $RV != 0 ]; then { echo "Failure dumping $2"; rm -f $LOG1 $LOG2; exit $RV ; }; fi + +diff $LOG1 $LOG2 +RV=$? +if [ $RV != 0 ]; then + echo "$1 <" + echo "$2 >" + echo "Files differ" +else + echo "Files match" +fi + +rm -f $LOG1 $LOG2 +exit $RV diff --git a/Control/AthenaMonitoring/share/hist_file_dump.py b/Control/AthenaMonitoring/share/hist_file_dump.py new file mode 100755 index 00000000000..2c4109135d8 --- /dev/null +++ b/Control/AthenaMonitoring/share/hist_file_dump.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python + +import ROOT +import sys, os, operator +import argparse +import zlib + +parser=argparse.ArgumentParser() +parser.add_argument('filename', + help='Input HIST file name') +parser.add_argument('-r', '--rankorder', default='onfile', + choices=['onfile', 'uncompressed', 'name'], + help='rankorder is "onfile" (default), "uncompressed" or "name"') +parser.add_argument('-p', '--path', + help='Only look under this directory') +parser.add_argument('--hash', action='store_true', + help='Print hashes of objects') +parser.add_argument('--metadata', action='store_true', + help='Include metadata trees') +parser.add_argument('--no_onfile', action='store_true', + help="Don't show on file size") +parser.add_argument('--no_inmem', action='store_true', + help="Don't show in memory size") +args=parser.parse_args() + +ordering = args.rankorder + +accounting = {}; hashes = {}; types = {} + +ROOT.gInterpreter.LoadText("UInt_t bufferhash(TKey* key) { key->SetBuffer(); key->ReadFile(); UInt_t rv = TString::Hash(key->GetBuffer()+key->GetKeylen(), key->GetNbytes()-key->GetKeylen()); key->DeleteBuffer(); return rv; }") +ROOT.gInterpreter.LoadText("void* getbuffer(TKey* key) { key->SetBuffer(); key->ReadFile(); return (void*) (key->GetBuffer()+key->GetKeylen()); }") +ROOT.gInterpreter.LoadText("UInt_t bufferhash2(TKey* key) { TObject* obj = key->ReadObj(); TMessage msg(kMESS_OBJECT); msg.WriteObject(obj); UInt_t rv = TString::Hash(msg.Buffer(), msg.Length()); delete obj; return rv; }") +ROOT.gInterpreter.LoadText("UInt_t bufferhash3(TKey* key) { TObject* obj = key->ReadObj(); UInt_t rv = obj->Hash(); delete obj; return rv; }") +ROOT.gSystem.Load('libDataQualityUtils') + +def dumpdir(d): + thispath = d.GetPath() + if ':' in thispath: + thispath = thispath.split(':', 1)[1] + #print thispath + subdirs = [] + for k in d.GetListOfKeys(): + if not args.metadata and k.GetName() == 'metadata' and k.GetClassName() == 'TTree': + continue + if k.GetClassName().startswith('TDirectory'): + subdirs.append(k) + else: + if args.hash: + #lhash = ROOT.bufferhash(k) + #objsize = (k.GetNbytes()-k.GetKeylen())/8 + #print (k.GetNbytes()-k.GetKeylen())/8. + #buf = ROOT.getbuffer(k); buf.SetSize(objsize) + #print buf[objsize-1], objsize + #lhash = zlib.adler32(str(buf)) + #k.DeleteBuffer() + #obj=k.ReadObj(); + #tm=ROOT.TMessage(ROOT.TMessage.kMESS_OBJECT) + #tm.WriteObject(obj) + # This is what we _were_ doing + #lhash = ROOT.bufferhash2(k) + # How about this? + #lhash = ROOT.bufferhash3(k) + obj = k.ReadObj(); lhash = obj.Hash(); del obj + else: + lhash = 0 + idxname = os.path.join(thispath, k.GetName()) + accounting[idxname] = (k.GetObjlen(), k.GetNbytes()-k.GetKeylen()) + hashes[idxname] = lhash + types[idxname] = k.GetClassName() + #print '%s,' % os.path.join(thispath, k.GetName()), + #obj = k.ReadObj(); obj.IsA().Destructor(obj) + #print 'OK' + for k in subdirs: + dumpdir(k.ReadObj()) + +f = ROOT.TFile.Open(args.filename) +if args.path: + d = f.Get(args.path.rstrip('/')) + if not d: + print "Can't access path", args.path, "- exiting" + sys.exit(1) +else: + d = f +dumpdir(d) + +#sortedl = sorted(accounting.items(), key=operator.itemgetter(0,1), reverse=True) +if ordering == 'onfile': + key=lambda x: (x[1][1], x[1][0], x[0]) +elif ordering == 'uncompressed': + key=lambda x: (x[1][0], x[1][1], x[0]) +else: + key=lambda x: (x[0], x[1][1], x[1][0]) +sortedl = sorted(accounting.items(), key=key, reverse=True) +if args.hash: + print '\n'.join(('%s %s: ' + + ('%d uncompressed' % b if not args.no_inmem else '') + + (', %d on file ' % c if not args.no_onfile else '') + + '(hash %s)') + % (types[a], a, hashes[a]) for a, (b, c) in sortedl) +else: + print '\n'.join(('%s %s: ' + + ('%d uncompressed' % b if not args.no_inmem else '') + + (', %d on file' % c if not args.no_onfile else '')) + % (types[a], a) for a, (b, c) in sortedl) + #print '\n'.join('%s %s: %d uncompressed, %d on file' % (types[a], a, b, c) for a, (b, c) in sortedl) + -- GitLab From 25f30aad95781b3ef921e106ad38afbd73fee2fb Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Tue, 5 Nov 2019 05:52:24 +0100 Subject: [PATCH 2/2] Add ART tests for Run 3 AthenaMonitoring --- .../test/test_run3dq_r21_esd.sh | 15 +++++++++++++++ .../test/test_run3dq_r21_esd_mc.sh | 15 +++++++++++++++ .../test/test_run3dq_r21_esd_mt.sh | 15 +++++++++++++++ .../test/test_run3dq_r22_aod_trigger.sh | 18 ++++++++++++++++++ .../test/test_run3dq_r22_esd.sh | 18 ++++++++++++++++++ 5 files changed, 81 insertions(+) create mode 100755 Control/AthenaMonitoring/test/test_run3dq_r21_esd.sh create mode 100755 Control/AthenaMonitoring/test/test_run3dq_r21_esd_mc.sh create mode 100755 Control/AthenaMonitoring/test/test_run3dq_r21_esd_mt.sh create mode 100755 Control/AthenaMonitoring/test/test_run3dq_r22_aod_trigger.sh create mode 100755 Control/AthenaMonitoring/test/test_run3dq_r22_esd.sh diff --git a/Control/AthenaMonitoring/test/test_run3dq_r21_esd.sh b/Control/AthenaMonitoring/test/test_run3dq_r21_esd.sh new file mode 100755 index 00000000000..85055a596f6 --- /dev/null +++ b/Control/AthenaMonitoring/test/test_run3dq_r21_esd.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# art-description: ESD->HIST, R21 data ESD +# art-type: grid +# art-include: master/Athena +# art-output: ExampleMonitorOutput.root + +Run3DQTestingDriver.py 'Input.Files=["/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/Tier0ChainTests/q431/21.0/myESD.pool.root"]' DQ.Steering.doHLTMon=False --postExec 'cfg.getEventAlgo("LArCollisionTimeAlg").cutIteration=False' + +echo "art-result: $? HIST_Creation" + +ArtPackage=$1 +ArtJobName=$2 +art.py download ${ArtPackage} ${ArtJobName} +hist_diff.sh ExampleMonitorOutput.root ./ref-*/ExampleMonitorOutput.root +echo "art-result: $? HIST_Diff" diff --git a/Control/AthenaMonitoring/test/test_run3dq_r21_esd_mc.sh b/Control/AthenaMonitoring/test/test_run3dq_r21_esd_mc.sh new file mode 100755 index 00000000000..f8f4ac0c73c --- /dev/null +++ b/Control/AthenaMonitoring/test/test_run3dq_r21_esd_mc.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# art-description: ESD->HIST, R21 MC ESD +# art-type: grid +# art-include: master/Athena +# art-output: ExampleMonitorOutput.root + +Run3DQTestingDriver.py 'Input.Files=["/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/Tier0ChainTests/q221/21.0/myESD.pool.root"]' DQ.Steering.doHLTMon=False --postExec 'cfg.getEventAlgo("LArCollisionTimeAlg").cutIteration=False' + +echo "art-result: $? HIST_Creation" + +ArtPackage=$1 +ArtJobName=$2 +art.py download ${ArtPackage} ${ArtJobName} +hist_diff.sh ExampleMonitorOutput.root ./ref-*/ExampleMonitorOutput.root +echo "art-result: $? HIST_Diff" diff --git a/Control/AthenaMonitoring/test/test_run3dq_r21_esd_mt.sh b/Control/AthenaMonitoring/test/test_run3dq_r21_esd_mt.sh new file mode 100755 index 00000000000..9e580205c6e --- /dev/null +++ b/Control/AthenaMonitoring/test/test_run3dq_r21_esd_mt.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# art-description: ESD->HIST, R21 data ESD, MT +# art-type: grid +# art-include: master/Athena +# art-output: ExampleMonitorOutput.root + +Run3DQTestingDriver.py 'Input.Files=["/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/Tier0ChainTests/q431/21.0/myESD.pool.root"]' DQ.Steering.doHLTMon=False Concurrency.NumThreads=1 Concurrency.NumConcurrentEvents=1 --postExec 'cfg.getEventAlgo("LArCollisionTimeAlg").cutIteration=False' + +echo "art-result: $? HIST_Creation" + +ArtPackage=$1 +ArtJobName=$2 +art.py download ${ArtPackage} ${ArtJobName} +hist_diff.sh ExampleMonitorOutput.root ./ref-*/ExampleMonitorOutput.root +echo "art-result: $? HIST_Diff" diff --git a/Control/AthenaMonitoring/test/test_run3dq_r22_aod_trigger.sh b/Control/AthenaMonitoring/test/test_run3dq_r22_aod_trigger.sh new file mode 100755 index 00000000000..8e7b5ce43a8 --- /dev/null +++ b/Control/AthenaMonitoring/test/test_run3dq_r22_aod_trigger.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# art-description: AOD->HIST, R22 MC, Trigger Only +# art-type: grid +# art-include: master/Athena +# art-output: ExampleMonitorOutput.root + +art.py download TrigAnalysisTest test_trigAna_q221_RDOtoAOD_mt1_grid.sh +AODFILE=(./ref-*/AOD.pool.root) +Run3DQTestingDriver.py 'Input.Files=["'${AODFILE}'"]' DQ.Steering.doHLTMon=True --dqOffByDefault + +echo "art-result: $? HIST_Creation" +rm -rf ref-* + +ArtPackage=$1 +ArtJobName=$2 +art.py download ${ArtPackage} ${ArtJobName} +hist_diff.sh ExampleMonitorOutput.root ./ref-*/ExampleMonitorOutput.root +echo "art-result: $? HIST_Diff" diff --git a/Control/AthenaMonitoring/test/test_run3dq_r22_esd.sh b/Control/AthenaMonitoring/test/test_run3dq_r22_esd.sh new file mode 100755 index 00000000000..59c55230ab0 --- /dev/null +++ b/Control/AthenaMonitoring/test/test_run3dq_r22_esd.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# art-description: ESD->HIST, R22 data ESD +# art-type: grid +# art-include: master/Athena +# art-output: ExampleMonitorOutput.root + +art.py download Tier0ChainTests test_q431.sh +ESDFILE=(./ref-*/myESD.pool.root) +Run3DQTestingDriver.py 'Input.Files=["'${ESDFILE}'"]' DQ.Steering.doHLTMon=False + +echo "art-result: $? HIST_Creation" +rm -rf ref-* + +ArtPackage=$1 +ArtJobName=$2 +art.py download ${ArtPackage} ${ArtJobName} +hist_diff.sh ExampleMonitorOutput.root ./ref-*/ExampleMonitorOutput.root +echo "art-result: $? HIST_Diff" -- GitLab