diff --git a/Database/AthenaPOOL/AthenaPoolCnvSvc/python/PoolAttributeHelper.py b/Database/AthenaPOOL/AthenaPoolCnvSvc/python/PoolAttributeHelper.py
new file mode 100644
index 0000000000000000000000000000000000000000..d8bc8678181cb1745b4a98744757d2d2de4462b1
--- /dev/null
+++ b/Database/AthenaPOOL/AthenaPoolCnvSvc/python/PoolAttributeHelper.py
@@ -0,0 +1,88 @@
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
+""" A basic module that helps with setting various common Pool Attributes """
+
+def setPoolAttribute( **kwargs ):
+ """ The low-level function that builds the requested Pool Attribure string. """
+
+ # Resolve the variables for convenience
+ fileName = kwargs.get('fileName')
+ contName = kwargs.get('contName')
+ attrName = kwargs.get('attrName')
+ attrValue = kwargs.get('attrValue')
+
+ # Now let's build the string
+ result = ""
+
+ # First set the domain attribute
+ if not attrName or not attrValue:
+ return result
+ result += f"{attrName} = '{attrValue}';"
+
+ # Then set database and container level attributes
+ # These are optional: can set both, only one, or neither
+ if contName:
+ result = f"ContainerName = '{contName}'; {result}"
+ if fileName:
+ result = f"DatabaseName = '{fileName}'; {result}"
+
+ # Finally return the result
+ return result
+
+def setFileCompAlg( fileName = None, compAlg = None ):
+ """ Convenience method for setting the compression algorithm for a given file. """
+
+ return setPoolAttribute( fileName = fileName,
+ attrName = "COMPRESSION_ALGORITHM",
+ attrValue = compAlg )
+
+def setFileCompLvl( fileName = None, compLvl = None ):
+ """ Convenience method for setting the compression level for a given file. """
+
+ return setPoolAttribute( fileName = fileName,
+ attrName = "COMPRESSION_LEVEL",
+ attrValue = compLvl )
+
+def setTreeAutoFlush( fileName = None, treeName = None, autoFlush = None ):
+ """ Convenience method for setting the AutoFlush for a tree in a given file. """
+
+ return setPoolAttribute( fileName = fileName,
+ contName = f"TTree={treeName}",
+ attrName = "TREE_AUTO_FLUSH",
+ attrValue = autoFlush )
+
+def setContainerSplitLevel( fileName = None, treeName = None, splitLvl = None ):
+ """ Convenience method for setting the split level for a tree in a given file. """
+
+ return setPoolAttribute( fileName = fileName,
+ contName = f"TTree={treeName}",
+ attrName = "CONTAINER_SPLITLEVEL",
+ attrValue = splitLvl )
+
+def setBranchBasketSize( fileName = None, treeName = None, basketSize = None ):
+ """ Convenience method for setting the branch basket size for a tree in a given file. """
+
+ return setPoolAttribute( fileName = fileName,
+ contName = f"TTree={treeName}",
+ attrName = "BRANCH_BASKET_SIZE",
+ attrValue = basketSize )
+
+# Main Function: Only to check the basic functionality
+# Can be run via python PoolAttributeHelper.py
+if "__main__" in __name__:
+
+ # Test Basic Functionality
+ attrs = []
+
+ # High-level
+ attrs += [ setFileCompAlg( "AOD.pool.root", 2 ) ]
+ attrs += [ setFileCompLvl( "AOD.pool.root", 1 ) ]
+ attrs += [ setTreeAutoFlush( "AOD.pool.root", "CollectionTree", 10 ) ]
+ attrs += [ setContainerSplitLevel( None, "POOLContainerForm(DataHeaderForm)", 99 ) ]
+
+ # Low-level
+ attrs += [ setPoolAttribute( attrName = "DEFAULT_SPLITLEVEL", attrValue = 0) ]
+
+ # Print the configuration
+ for attr in attrs:
+ print( f"{attr}" )
diff --git a/Database/AthenaPOOL/AthenaPoolCnvSvc/python/WriteAthenaPool.py b/Database/AthenaPOOL/AthenaPoolCnvSvc/python/WriteAthenaPool.py
index 785c4881f37d41d0cf61a43931151eaeb1f20476..461988021486c0351446c6b97c0c7b98dbc87b4b 100644
--- a/Database/AthenaPOOL/AthenaPoolCnvSvc/python/WriteAthenaPool.py
+++ b/Database/AthenaPOOL/AthenaPoolCnvSvc/python/WriteAthenaPool.py
@@ -40,10 +40,8 @@ def _configureWriteAthenaPool():
# Increase default BasketSize to 32K, ROOT default (but overwritten by POOL)
svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "DEFAULT_BUFFERSIZE = '32000'" ]
- # Turn off auto_flush for DataHeader container to avoid basket optimization
- svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "ContainerName = 'POOLContainer(DataHeader)'; BRANCH_BASKET_SIZE = '256000'" ]
- svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "ContainerName = 'POOLContainerForm(DataHeaderForm)'; BRANCH_BASKET_SIZE = '1024000'" ]
- svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "ContainerName = 'TTree=POOLContainerForm(DataHeaderForm)'; CONTAINER_SPLITLEVEL = '99'" ]
+ # Set POOLContainerForm(DataHeaderForm) split level to 0
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "ContainerName = 'TTree=POOLContainerForm(DataHeaderForm)'; CONTAINER_SPLITLEVEL = '0'" ]
svcMgr.AthenaPoolCnvSvc.TopLevelContainerName = ""
svcMgr.AthenaPoolCnvSvc.SubLevelBranchName = "<type>/<key>"
diff --git a/Reconstruction/RecExample/RecExCommon/share/RecoUtils.py b/Reconstruction/RecExample/RecExCommon/share/RecoUtils.py
index 61fce65d89efa0e0df12c2817c95169a5c4f6b36..9b7a99627d2704b7c37b04ab39f5997fd3459291 100644
--- a/Reconstruction/RecExample/RecExCommon/share/RecoUtils.py
+++ b/Reconstruction/RecExample/RecExCommon/share/RecoUtils.py
@@ -151,27 +151,39 @@ if rec.doTimeLimit():
# rather use default athenapool value
if rec.doPersistencyOptimization() and hasattr(svcMgr, 'AthenaPoolCnvSvc'):
+ from AthenaPoolCnvSvc import PoolAttributeHelper as pah
+
svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "TREE_BRANCH_OFFSETTAB_LEN ='100'" ]
if rec.doWriteRDO():
from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
- ServiceMgr.AthenaPoolCnvSvc.PoolAttributes += [ "DatabaseName = '" + athenaCommonFlags.PoolRDOOutput() + "'; COMPRESSION_ALGORITHM = '2'" ]
- ServiceMgr.AthenaPoolCnvSvc.PoolAttributes += [ "DatabaseName = '" + athenaCommonFlags.PoolRDOOutput() + "'; COMPRESSION_LEVEL = '1'" ]
- svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "DatabaseName = '" + athenaCommonFlags.PoolRDOOutput() + "'; ContainerName = 'TTree=CollectionTree'; TREE_AUTO_FLUSH = '1'" ]
+ # Use LZMA w/ Level 1
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setFileCompAlg( athenaCommonFlags.PoolRDOOutput(), 2 ) ]
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setFileCompLvl( athenaCommonFlags.PoolRDOOutput(), 1 ) ]
+ # Flush the CollectionTree, POOLContainer, and POOLContainerForm to disk at every 1 events
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( athenaCommonFlags.PoolRDOOutput(), "CollectionTree", 1 ) ]
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( athenaCommonFlags.PoolRDOOutput(), "POOLContainer", 1 ) ]
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( athenaCommonFlags.PoolRDOOutput(), "POOLContainerForm", 1 ) ]
if rec.doWriteESD():
from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
- svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "DatabaseName = '" + athenaCommonFlags.PoolESDOutput() + "'; COMPRESSION_ALGORITHM = '2'" ]
- svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "DatabaseName = '" + athenaCommonFlags.PoolESDOutput() + "'; COMPRESSION_LEVEL = '1'" ]
- # Optimize Basket Sizes to store data for 10 entries/events
- svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "DatabaseName = '" + athenaCommonFlags.PoolESDOutput() + "'; ContainerName = 'TTree=CollectionTree'; TREE_AUTO_FLUSH = '10'" ]
+ # Use LZMA w/ Level 1
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setFileCompAlg( athenaCommonFlags.PoolESDOutput(), 2 ) ]
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setFileCompLvl( athenaCommonFlags.PoolESDOutput(), 1 ) ]
+ # Flush the CollectionTree, POOLContainer, and POOLContainerForm to disk at every 10 events
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( athenaCommonFlags.PoolESDOutput(), "CollectionTree", 10 ) ]
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( athenaCommonFlags.PoolESDOutput(), "POOLContainer", 10 ) ]
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( athenaCommonFlags.PoolESDOutput(), "POOLContainerForm", 10 ) ]
if rec.doWriteAOD():
from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
- svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "DatabaseName = '" + athenaCommonFlags.PoolAODOutput() + "'; COMPRESSION_ALGORITHM = '2'" ]
- svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "DatabaseName = '" + athenaCommonFlags.PoolAODOutput() + "'; COMPRESSION_LEVEL = '1'" ]
- # Optimize Basket Sizes to store data for 100 entries/events
- svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "DatabaseName = '" + athenaCommonFlags.PoolAODOutput() + "'; ContainerName = 'TTree=CollectionTree'; TREE_AUTO_FLUSH = '100'"]
+ # Use LZMA w/ Level 1
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setFileCompAlg( athenaCommonFlags.PoolAODOutput(), 2 ) ]
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setFileCompLvl( athenaCommonFlags.PoolAODOutput(), 1 ) ]
+ # Flush the CollectionTree, POOLContainer, and POOLContainerForm to disk at every 100 events
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( athenaCommonFlags.PoolAODOutput(), "CollectionTree", 100 ) ]
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( athenaCommonFlags.PoolAODOutput(), "POOLContainer", 100 ) ]
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( athenaCommonFlags.PoolAODOutput(), "POOLContainerForm", 100 ) ]
# Base the xAOD branch names just on the SG keys:
StreamAOD.WritingTool.SubLevelBranchName = "<key>"
diff --git a/Simulation/SimuJobTransforms/share/skeleton.EVGENtoHIT_ISF.py b/Simulation/SimuJobTransforms/share/skeleton.EVGENtoHIT_ISF.py
index 59db6eb01d0afe0f08ea0231b7b51fb543de5921..11ce0fa1af471696871e1e0d8980f213ffe0507f 100644
--- a/Simulation/SimuJobTransforms/share/skeleton.EVGENtoHIT_ISF.py
+++ b/Simulation/SimuJobTransforms/share/skeleton.EVGENtoHIT_ISF.py
@@ -268,6 +268,15 @@ if jobproperties.Beam.beamType.get_Value() == 'cosmics':
else:
svcMgr.EventSelector.FirstEvent = 0
+# Set AutoFlush to 10 as per ATLASSIM-4274
+# These outputs are meant to be read sequentially
+if athenaCommonFlags.PoolHitsOutput():
+ from AthenaPoolCnvSvc import PoolAttributeHelper as pah
+ Out = athenaCommonFlags.PoolHitsOutput()
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( Out, "CollectionTree", 10 ) ]
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( Out, "POOLContainer", 10 ) ]
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( Out, "POOLContainerForm", 10 ) ]
+
## Post-include
if hasattr(runArgs, "postInclude"):
for fragment in runArgs.postInclude:
diff --git a/Simulation/SimuJobTransforms/share/skeleton.FilterHit.py b/Simulation/SimuJobTransforms/share/skeleton.FilterHit.py
index 6123f8a102d5833c64ccf9ce89d02a9cfa556051..9ed07af610a7e501f6bb888e3392a7ac897ee785 100644
--- a/Simulation/SimuJobTransforms/share/skeleton.FilterHit.py
+++ b/Simulation/SimuJobTransforms/share/skeleton.FilterHit.py
@@ -120,6 +120,13 @@ except:
# The next line is an example on how to exclude clid's if they are causing a problem
#Stream1.ExcludeList = ['6421#*']
+# Set AutoFlush to 1 as per ATLASSIM-4274
+# These outputs are meant to be read randomly
+from AthenaPoolCnvSvc import PoolAttributeHelper as pah
+ServiceMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( Stream1.OutputFile , "CollectionTree", 1 ) ]
+ServiceMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( Stream1.OutputFile , "POOLContainer", 1 ) ]
+ServiceMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( Stream1.OutputFile , "POOLContainerForm", 1 ) ]
+
#--------------------------------------------------------------
# Specify collections for output HIT files, as not all are required.
#--------------------------------------------------------------
diff --git a/Simulation/SimuJobTransforms/share/skeleton.HITSMerge.py b/Simulation/SimuJobTransforms/share/skeleton.HITSMerge.py
index 2aacca3666c64d9690eaea9562fd2e3a604aefc2..04e2c2e0148f69c0583a3bd2901935749bd2546b 100644
--- a/Simulation/SimuJobTransforms/share/skeleton.HITSMerge.py
+++ b/Simulation/SimuJobTransforms/share/skeleton.HITSMerge.py
@@ -157,6 +157,15 @@ if not hasattr(ServiceMgr.IOVDbSvc, 'GlobalTag') or not ServiceMgr.IOVDbSvc.Glob
#--------------------------------------------------------------
+# Set AutoFlush to 10 as per ATLASSIM-4274
+# These outputs are meant to be read sequentially
+from AthenaPoolCnvSvc import PoolAttributeHelper as pah
+ServiceMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( Out, "CollectionTree", 10 ) ]
+ServiceMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( Out, "POOLContainer", 10 ) ]
+ServiceMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( Out, "POOLContainerForm", 10 ) ]
+
+#--------------------------------------------------------------
+
## Post-include
if hasattr(runArgs,"postInclude"):
for fragment in runArgs.postInclude:
diff --git a/Simulation/SimuJobTransforms/share/skeleton.HITtoRDO.py b/Simulation/SimuJobTransforms/share/skeleton.HITtoRDO.py
index 5fd42e4bc53bcff86931840b1dab4a65551a7dc1..7b4c4499cdb187f95bae71de4c31f0a5cfe39986 100644
--- a/Simulation/SimuJobTransforms/share/skeleton.HITtoRDO.py
+++ b/Simulation/SimuJobTransforms/share/skeleton.HITtoRDO.py
@@ -423,10 +423,19 @@ if hasattr(runArgs,"AMITag"):
#==========================================================
# Use ZLIB for compression of all temporary outputs
#==========================================================
-from AthenaCommon.AppMgr import ServiceMgr as svcMgr; import AthenaPoolCnvSvc.AthenaPool
-if hasattr(runArgs, "outputRDOFile") and ('_000' in runArgs.outputRDOFile or 'tmp.' in runArgs.outputRDOFile):
- svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "DatabaseName = '" + athenaCommonFlags.PoolRDOOutput()+ "'; COMPRESSION_ALGORITHM = '1'" ]
- svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ "DatabaseName = '" + athenaCommonFlags.PoolRDOOutput()+ "'; COMPRESSION_LEVEL = '1'" ]
+from AthenaCommon.AppMgr import ServiceMgr as svcMgr
+import AthenaPoolCnvSvc.AthenaPool # noqa: F401
+from AthenaPoolCnvSvc import PoolAttributeHelper as pah
+Out = athenaCommonFlags.PoolRDOOutput()
+if hasattr(runArgs, "outputRDOFile") and ('_000' in runArgs.outputRDOFile or 'tmp.' in runArgs.outputRDOFile): # noqa: F821
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setFileCompAlg( Out, 1 ) ]
+ svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setFileCompLvl( Out, 1 ) ]
+
+# Set AutoFlush to 1 as per ATLASSIM-4274
+# This helps with the overall HITtoRDO memory footprint
+svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( Out, "CollectionTree", 1 ) ]
+svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( Out, "POOLContainer", 1 ) ]
+svcMgr.AthenaPoolCnvSvc.PoolAttributes += [ pah.setTreeAutoFlush( Out, "POOLContainerForm", 1 ) ]
## Post-include
if hasattr(runArgs,"postInclude"):