From ccb732a87f63d3af145b9a00ad5dd07e3fa5099f Mon Sep 17 00:00:00 2001
From: James Beacham <j.beacham@cern.ch>
Date: Fri, 16 Mar 2018 19:23:45 +0000
Subject: [PATCH] Merge branch '21.0-directIO-art-implementation' into '21.0'
New package for direct IO ART tests
See merge request atlas/athena!9615
(cherry picked from commit a8022ad86e64c0a23a7e731a0750c726fea0385b [formerly ca5643cce58d64b9d84f330649d09cf83ce22204])
e754159a First commit of Tools/DirectIOART
Former-commit-id: 3ada6767734038f52efe402d370addc979479e30
---
Tools/DirectIOART/CMakeLists.txt | 16 ++
Tools/DirectIOART/generateARTtests.py | 160 ++++++++++++++++++
.../test/test_DirectIOART_TFileOpen.py | 28 +++
3 files changed, 204 insertions(+)
create mode 100644 Tools/DirectIOART/CMakeLists.txt
create mode 100755 Tools/DirectIOART/generateARTtests.py
create mode 100755 Tools/DirectIOART/test/test_DirectIOART_TFileOpen.py
diff --git a/Tools/DirectIOART/CMakeLists.txt b/Tools/DirectIOART/CMakeLists.txt
new file mode 100644
index 00000000000..356791473f5
--- /dev/null
+++ b/Tools/DirectIOART/CMakeLists.txt
@@ -0,0 +1,16 @@
+################################################################################
+# Package: DirectIOART
+################################################################################
+
+# Declare the package name:
+atlas_subdir( DirectIOART )
+
+# Declare the package's dependencies:
+atlas_depends_on_subdirs( PRIVATE
+ TestPolicy )
+
+# Install files from the package:
+atlas_install_scripts( test/*.sh test/*.py )
+
+### # Install joboptions
+### atlas_install_joboptions( share/*.py )
diff --git a/Tools/DirectIOART/generateARTtests.py b/Tools/DirectIOART/generateARTtests.py
new file mode 100755
index 00000000000..531d75d3c91
--- /dev/null
+++ b/Tools/DirectIOART/generateARTtests.py
@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+
+import os
+
+HAMMERCLOUD_FILE = "mc15_13TeV.423202.Pythia8B_A14_CTEQ6L1_Jpsie3e13.merge.AOD.e3869_s2608_s2183_r6630_r6264"
+PROJECTS = ["AthAnalysis", "AthDerivation", "AthSimulation", "Athena", "AnalysisBase"]
+
+#PWD = os.environ['PWD']
+#DIRNAME = "test"
+
+TFILE_OPEN_TURLs = [
+ {"ROOT":
+ ("root://lcg-lrz-rootd.grid.lrz.de:1094/pnfs/lrz-muenchen.de/data/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # LRZ-LMU
+ "root://grid-dc.rzg.mpg.de:1094//pnfs/rzg.mpg.de/data/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # MPPMU
+ "root://dcache-atlas-xrootd.desy.de:1094//pnfs/desy.de/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # DESY-Hamburg
+ "root://lcg-se0.ifh.de:1094//pnfs/ifh.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # DESY-Zeuthen
+ "root://atlasxrootd-kit.gridka.de:1094//pnfs/gridka.de/atlas/disk-only/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # Karlsruhe
+ "root://sedoor1.bfg.uni-freiburg.de:1094//pnfs/bfg.uni-freiburg.de/data/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # Freiburg
+ "root://se-goegrid.gwdg.de:1094//pnfs/gwdg.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # Goettingen
+ "root://grid-se.physik.uni-wuppertal.de:1094//pnfs/physik.uni-wuppertal.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # Wuppertal
+ )
+ },
+ {"DAVS":
+ ("davs://lcg-lrz-http.grid.lrz.de:443/pnfs/lrz-muenchen.de/data/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # LRZ-LMU
+ "davs://grid-dav.rzg.mpg.de:2880/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", #MPPMU
+ "davs://dcache-atlas-webdav.desy.de:2880/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # DESY-Hamburg
+ "davs://lcg-se0.ifh.de:2880/pnfs/ifh.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # DESY-Zeuthen
+ "davs://atlaswebdav-kit.gridka.de:2880/pnfs/gridka.de/atlas/disk-only/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # Karlsruhe
+ "davs://webdav.bfg.uni-freiburg.de:2880/pnfs/bfg.uni-freiburg.de/data/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # Freiburg
+ "davs://se-goegrid.gwdg.de:2880/pnfs/gwdg.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # Goettingen
+ "davs://grid-se.physik.uni-wuppertal.de:2881/pnfs/physik.uni-wuppertal.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # Wuppertal
+ )
+ },
+ {"HTTPS":
+ ("https://lcg-lrz-http.grid.lrz.de:443/pnfs/lrz-muenchen.de/data/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # LRZ-LMU
+ "https://grid-dav.rzg.mpg.de:2880/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # MPPMU
+ "https://dcache-atlas-webdav.desy.de:2880/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # DESY-Hamburg
+ "https://lcg-se0.ifh.de:2880/pnfs/ifh.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # DESY-Zeuthen
+ "https ://atlaswebdav-kit.gridka.de:2880/pnfs/gridka.de/atlas/disk-only/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # Karlsruhe
+ "https://webdav.bfg.uni-freiburg.de:2880/pnfs/bfg.uni-freiburg.de/data/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # Freiburg
+ "https://se-goegrid.gwdg.de:2880/pnfs/gwdg.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # Goettingen
+ "https://grid-se.physik.uni-wuppertal.de:2881/pnfs/physik.uni-wuppertal.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1", # Wuppertal
+ )
+ },
+]
+
+#def writeTFileOpen(athena_project):
+ #name = os.path.join("test", "test_DirectIO" + athena_project + "ART_TFileOpen.py")
+ #print "\tGenerating ...",name
+ #outfile = open(name,'w')
+
+ ## generate frame of test scripts
+ #outstring = """#!/usr/bin/env python\n
+## art-description: DirectIO{project}ART TFile::Open
+## art-type: grid
+## art-include: {branch}/{project}\n
+#import ROOT
+#import os\n""".format(project=athena_project, branch='21.0' if athena_project == 'Athena' else '21.3' if athena_project == 'AthSimulation' else '21.2')
+ ## loop over turl list
+ #for i in xrange(len(TFILE_OPEN_TURLs)):
+ ## get values of dictionaries
+ #for key, value in TFILE_OPEN_TURLs[i].items():
+ ## add turl to filenames string
+ #filenames = ""
+ #for j in xrange(len(value)):
+ #filenames += value[j]
+ #if j < len(value)-1: filenames += " | "
+ #outstring += """
+#f = ROOT.TFile.Open(\"{turl}\")
+#n = f.GetName()
+#if f: print(\"art-result: 0 DirectIO{project}ART_TFileOpen_protocol_{protocol}\")
+#else: print(\"art-result: 1 DirectIO{project}ART_TFileOpen_protocol_{protocol}\")
+#del f
+#os.system(\"pool_insertFileToCatalog {{}}\".format(n))\n""".format(turl=filenames, project=athena_project, protocol=key)
+ #filenames = ""
+
+ ## back to main test scripts
+ #outfile.write(outstring)
+ #outfile.close()
+ #os.system("chmod +x " + name)
+
+#def parse_options():
+ #import argparse
+ #parser = argparse.ArgumentParser()
+
+ #parser.add_argument("athena_project", help="Specify the Athena release flavour (Default=AthAnalysis)", type=str, nargs="*" )
+ #parser.add_argument("-o", "--tfile-open", help="Generate ART scripts for TFile::Open tests", action="store_true", dest="open_tests")
+
+ #options = parser.parse_args()
+ #return options
+
+def writeTFileOpen():
+ name = os.path.join("test", "test_DirectIOART_TFileOpen.py")
+ print "\tGenerating ...",name
+ outfile = open(name,'w')
+
+ # generate frame of test scripts
+ outstring = """#!/usr/bin/env python\n
+# art-description: DirectIOART TFile::Open
+# art-type: grid\n
+import ROOT
+import os\n"""
+ # loop over turl list
+ for i in xrange(len(TFILE_OPEN_TURLs)):
+ # get values of dictionaries
+ for key, value in TFILE_OPEN_TURLs[i].items():
+ # add turl to filenames string
+ filenames = ""
+ for j in xrange(len(value)):
+ filenames += value[j]
+ if j < len(value)-1: filenames += " | "
+ outstring += """
+f = ROOT.TFile.Open(\"{turl}\")
+n = f.GetName()
+if f: print(\"art-result: 0 DirectIOART_TFileOpen_protocol_{protocol}\")
+else: print(\"art-result: 1 DirectIOART_TFileOpen_protocol_{protocol}\")
+del f
+os.system(\"pool_insertFileToCatalog {{}}\".format(n))\n""".format(turl=filenames, protocol=key)
+ filenames = ""
+
+ # back to main test scripts
+ outfile.write(outstring)
+ outfile.close()
+ os.system("chmod +x " + name)
+
+def parse_options():
+ import argparse
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument("athena_project", help="Specify the Athena release flavour (Default=AthAnalysis)", type=str, nargs="*" )
+ parser.add_argument("-o", "--tfile-open", help="Generate ART scripts for TFile::Open tests", action="store_true", dest="open_tests")
+
+ options = parser.parse_args()
+ return options
+
+def main():
+ #TODO: Extend to AthAnalysis & AthDerivation algorithms
+
+ opts = parse_options()
+
+ if not os.path.exists("./test/"):
+ os.makedirs("./test/")
+
+ for i in xrange(len(opts.athena_project)):
+ if not opts.athena_project[i] in PROJECTS:
+ print "\tAthena project \"",opts.athena_project[i],"\" not intended for DirectIOART!"
+ continue
+
+ #if opts.open_tests:
+ #writeTFileOpen(opts.athena_project[i])
+
+ if opts.open_tests:
+ writeTFileOpen()
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except KeyboardInterrupt:
+ print "\nexiting"
\ No newline at end of file
diff --git a/Tools/DirectIOART/test/test_DirectIOART_TFileOpen.py b/Tools/DirectIOART/test/test_DirectIOART_TFileOpen.py
new file mode 100755
index 00000000000..e640e380818
--- /dev/null
+++ b/Tools/DirectIOART/test/test_DirectIOART_TFileOpen.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+# art-description: DirectIOART TFile::Open
+# art-type: grid
+
+import ROOT
+import os
+
+f = ROOT.TFile.Open("root://lcg-lrz-rootd.grid.lrz.de:1094/pnfs/lrz-muenchen.de/data/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | root://grid-dc.rzg.mpg.de:1094//pnfs/rzg.mpg.de/data/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | root://dcache-atlas-xrootd.desy.de:1094//pnfs/desy.de/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | root://lcg-se0.ifh.de:1094//pnfs/ifh.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | root://atlasxrootd-kit.gridka.de:1094//pnfs/gridka.de/atlas/disk-only/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | root://sedoor1.bfg.uni-freiburg.de:1094//pnfs/bfg.uni-freiburg.de/data/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | root://se-goegrid.gwdg.de:1094//pnfs/gwdg.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | root://grid-se.physik.uni-wuppertal.de:1094//pnfs/physik.uni-wuppertal.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1")
+n = f.GetName()
+if f: print("art-result: 0 DirectIOART_TFileOpen_protocol_ROOT")
+else: print("art-result: 1 DirectIOART_TFileOpen_protocol_ROOT")
+del f
+os.system("pool_insertFileToCatalog {}".format(n))
+
+f = ROOT.TFile.Open("davs://lcg-lrz-http.grid.lrz.de:443/pnfs/lrz-muenchen.de/data/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | davs://grid-dav.rzg.mpg.de:2880/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | davs://dcache-atlas-webdav.desy.de:2880/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | davs://lcg-se0.ifh.de:2880/pnfs/ifh.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | davs://atlaswebdav-kit.gridka.de:2880/pnfs/gridka.de/atlas/disk-only/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | davs://webdav.bfg.uni-freiburg.de:2880/pnfs/bfg.uni-freiburg.de/data/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | davs://se-goegrid.gwdg.de:2880/pnfs/gwdg.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | davs://grid-se.physik.uni-wuppertal.de:2881/pnfs/physik.uni-wuppertal.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1")
+n = f.GetName()
+if f: print("art-result: 0 DirectIOART_TFileOpen_protocol_DAVS")
+else: print("art-result: 1 DirectIOART_TFileOpen_protocol_DAVS")
+del f
+os.system("pool_insertFileToCatalog {}".format(n))
+
+f = ROOT.TFile.Open("https://lcg-lrz-http.grid.lrz.de:443/pnfs/lrz-muenchen.de/data/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | https://grid-dav.rzg.mpg.de:2880/atlas/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | https://dcache-atlas-webdav.desy.de:2880/dq2/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | https://lcg-se0.ifh.de:2880/pnfs/ifh.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | https ://atlaswebdav-kit.gridka.de:2880/pnfs/gridka.de/atlas/disk-only/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | https://webdav.bfg.uni-freiburg.de:2880/pnfs/bfg.uni-freiburg.de/data/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | https://se-goegrid.gwdg.de:2880/pnfs/gwdg.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1 | https://grid-se.physik.uni-wuppertal.de:2881/pnfs/physik.uni-wuppertal.de/data/atlas/atlasdatadisk/rucio/mc15_13TeV/ed/68/AOD.05536542._000001.pool.root.1")
+n = f.GetName()
+if f: print("art-result: 0 DirectIOART_TFileOpen_protocol_HTTPS")
+else: print("art-result: 1 DirectIOART_TFileOpen_protocol_HTTPS")
+del f
+os.system("pool_insertFileToCatalog {}".format(n))
--
GitLab