Merge branch 'athena_unitest' into '24.0'

AthenaCommon: package and unit test cleanup

See merge request !69685

Control/AthenaCommon/CMakeLists.txt

-# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
 
 # Declare the package name:
 atlas_subdir( AthenaCommon )
@@ -7,8 +7,7 @@ if (XAOD_STANDALONE)
     # Install files from the package:
     atlas_install_python_modules( python/SystemOfUnits.py
                                   python/Logging.py
-                                  python/Constants.py
-                                  POST_BUILD_CMD ${ATLAS_FLAKE8} )
+                                  python/Constants.py )
 else()
     # Set up specific Athena runtime environment:
     set( AthenaCommonEnvironment_DIR ${CMAKE_CURRENT_SOURCE_DIR}
@@ -16,17 +15,13 @@ else()
     find_package( AthenaCommonEnvironment )
     
     # Install files from the package:
-    atlas_install_python_modules( python/*.py python/Utils
-                                  POST_BUILD_CMD ${ATLAS_FLAKE8} )
-    
+    atlas_install_python_modules( python/*.py python/Utils )
+
     atlas_install_joboptions( share/Preparation.py
                               share/Execution.py
                               share/Atlas.UnixStandardJob.py
-                              share/zeroJO.py
                               share/Atlas_Gen.UnixStandardJob.py
-                              share/MemTraceInclude.py
-                              share/runbatch.py
-                              test/*.py )
+                              share/runbatch.py )
     
     atlas_install_scripts( share/athena_preload.sh
                            share/ThinCAWrapper.sh )
@@ -40,25 +35,23 @@ else()
     atlas_add_alias( athena "athena.py" )
     
     # Tests:
-    atlas_add_test( AthAppMgrUnitTests SCRIPT test/test_AthAppMgrUnitTests.sh
-       PROPERTIES TIMEOUT 300
-       LOG_IGNORE_PATTERN "Warning in <TFile::Init>: no StreamerInfo found|^Ran .* tests in|built on" )
-    
-    atlas_add_test( ConfigurableUnitTests SCRIPT test/test_ConfigurableUnitTests.sh
-       PROPERTIES TIMEOUT 300
-       LOG_IGNORE_PATTERN "Warning in <TFile::Init>: no StreamerInfo found|^Ran .* tests in" )
-    
-    atlas_add_test( JobPropertiesUnitTests SCRIPT test/test_JobPropertiesUnitTests.sh
-       LOG_IGNORE_PATTERN "Warning in <TFile::Init>: no StreamerInfo found|^Ran .* tests in" )
-    
-    atlas_add_test( KeyStoreUnitTests SCRIPT test/test_KeyStoreUnitTests.sh
-       LOG_IGNORE_PATTERN "Warning in <TFile::Init>: no StreamerInfo found|^Ran .* tests in|^outFileName: " )
-    
-    atlas_add_test( CFElementsTest SCRIPT python -m unittest -v AthenaCommon.CFElements
-       POST_EXEC_SCRIPT nopost.sh )
-    
+    atlas_add_test( AthAppMgr SCRIPT test/AthAppMgrUnitTests.py
+       POST_EXEC_SCRIPT noerror.sh )
+
+    atlas_add_test( Configurable SCRIPT test/ConfigurableUnitTests.py
+       POST_EXEC_SCRIPT noerror.sh )
+
+    atlas_add_test( JobProperties SCRIPT test/JobPropertiesUnitTests.py
+       POST_EXEC_SCRIPT noerror.sh )
+
+    atlas_add_test( KeyStore SCRIPT test/KeyStoreUnitTests.py
+       POST_EXEC_SCRIPT noerror.sh )
+
+    atlas_add_test( CFElements SCRIPT python -m unittest -v AthenaCommon.CFElements
+       POST_EXEC_SCRIPT noerror.sh )
+
     if (NOT XAOD_ANALYSIS)
-       atlas_add_test( JobOptionsUnitTests SCRIPT test/test_JobOptionsUnitTests.sh
-          LOG_IGNORE_PATTERN "Warning in <TFile::Init>: no StreamerInfo found|^Ran .* tests in" )
+       atlas_add_test( JobOptionsUnitTests SCRIPT test/JobOptionsUnitTests.py
+          POST_EXEC_SCRIPT noerror.sh )
     endif()
 endif()

Control/AthenaCommon/python/BFieldFlags.py

-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
-
-#=======================================================================
-# File:   AthenaCommon/python/BeamFlags.py
-#=======================================================================
-""" Flags for beam structure
-"""
-#
-#
-__author__  = 'D. Rousseau'
-__version__="$Revision: 1.2 $"
-__doc__="magnetic field flags "
-
-
-
-
-#=======================================================================
-# imports
-#=======================================================================
-from AthenaCommon.JobProperties import JobProperty, JobPropertyContainer
-from AthenaCommon.JobProperties import jobproperties
-
-class override(JobProperty):
-    """ True wether magnetic filed is taken from these flags rather than cond db (when available)
-    """
-    statusOn=True
-    allowedTypes=['bool']
-    StoredValue=True
-
-class solenoidOn (JobProperty):
-    """ True wether solenoid is on
-    """
-    statusOn=True
-    allowedTypes=['bool']
-    StoredValue=True
-
-class barrelToroidOn (JobProperty):
-    """ True wether barrel toroid is on
-    """
-    statusOn=True
-    allowedTypes=['bool']
-    StoredValue=True
-
-class endcapToroidOn (JobProperty):
-    """ True wether endcap toroid are on
-    """
-    statusOn=True
-    allowedTypes=['bool']
-    StoredValue=True
-
-
-
-
-
- 
-# Defines a sub-container for the algorithm switches
-class BField(JobPropertyContainer):
-    """ Beam information """
-    # computes on the fly the expected luminosity
-    def allToroidOn(self):
-        return jobproperties.BField.barrelToroidOn() and  jobproperties.BField.endcapToroidOn()
-        
-# add the beam flags container to the top container 
-jobproperties.add_Container(BField)
-
-# I want always the following flags in the BField container  
-for j in [override,solenoidOn,barrelToroidOn,endcapToroidOn]: 
-    jobproperties.BField.add_JobProperty(j)
-
-
-
-#=======================================================================
-

Control/AthenaCommon/python/DumpProperties.py

-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
-
-# File: AthenaCommon/python/DumpProperties.py
-# Author: Wim Lavrijsen (WLavrijsen@lbl.gov)
-
-"""Dump all properties from Gaudi objects in a given namespace."""
-
-import os
-import sys
-import GaudiPython
-
-
-### data ---------------------------------------------------------------------
-__version__ = '1.2.0'
-__author__  = 'Wim Lavrijsen (WLavrijsen@lbl.gov)'
-
-__all__ = [ 'pprint', 'dump', 'dumpSet', 'dumpAlgorithms', 'dumpServices' ]
-
-indent = '   '
-
-
-### logging and messages -----------------------------------------------------
-import logging
-log = logging.getLogger( 'DumpProperties' )
-
-
-### helper -------------------------------------------------------------------
-def _printFromLookup( ns, listing, lookup, extra, klass ):
-   """<internal>"""
- 
-   if 'theApp' not in ns:
-      log.error( 'given namespace does not contain "theApp"' )
-      return
-
-   if lookup not in ns:
-      log.error( 'given namespace does not contain "%s" lookup', lookup )
-      return
-
-   app = ns[ 'theApp' ]
-   if app.state() != app.State.CONFIGURED:
-      listing = []
-      for name, obj in ns.items():
-         if isinstance( obj, klass ):
-            listing.append( name )
-   else:
-      listing = getattr( app, listing )()
-
-   if extra and type(extra) != list:
-      extra = getattr( app, extra )
-
-   extra = [ e for e in extra if e not in listing ]
-
-   lookup = ns[ lookup ]
-   for name in listing:
-      pprint( lookup( name ) )
-
-   for name in extra:
-      pprint( klass( name ) )
-
-
-### function pprint ---------------------------------------------------------
-def pprint( obj, stream = sys.stdout ):
-   """Pretty print the properties of the given Gaudi object."""
-
-   from AthenaCommon.Configurable import Configurable
-   if isinstance( obj, Configurable ):
-      stream.write( str(obj) )
-      stream.write( '\n' )
-      return
-
-   try:
-      stream.write( obj.name() + os.linesep )
-   except TypeError:
-      print (obj)
-   for prop, value in obj.properties().items():
-      if not obj._ip:
-         try:
-            value = eval( value )
-         except Exception:
-            pass
-
-      if value and type(value) == list:
-         stream.write( indent + '%s = [\n' % prop )
-         nv = len(value)
-         for i in range(nv):
-            v = value[i]
-            if type(v) == str:
-               v = '"%s"' % v
-            stream.write( 2*indent + str(v) + ( i == nv-1 and "\n" or ",\n" ) )
-         stream.write( indent + ']\n' )
-         continue
-
-      if hasattr( value, 'value' ) and callable( value.value ):
-         value = value.value()
-
-      if type(value) == str:
-         value = '"%s"' % value
-
-      stream.write( indent + '%-20s = %s\n' % (prop,value) )
-   stream.write( os.linesep )
-
-
-### function dump -----------------------------------------------------------
-def dump( opt = [ 'set' ], ns = None ):
-   """Dump properties of Gaudi objects; opt can be 'set', 'algorithms', 'services',
-'all', or a list of a combination of these. The given namespace must contain the
-application manager."""
-
-   if ns is None:
-      import __main__
-      ns = __main__.__dict__ 
-
-   if hasattr( opt, 'properties' ) and hasattr( opt, 'name' ):
-      pprint( opt )
-      return
-
-   if type(opt) == str:
-      opt = [ opt ]
-
-   if 'set' in opt:
-      log.info( '  ***** dumping properties of configured objects *****  ' )
-      dumpSet( ns )
-
-   all = 'all' in opt
-   if all or 'algorithms' in opt:
-      log.info( '  ***** dumping algorithm properties *****  ' )
-      dumpAlgorithms( ns )
-
-   if all or 'services' in opt:
-      log.info( '  ***** dumping service properties   *****  ' )
-      dumpServices( ns )
- 
-
-### selected dump functions -------------------------------------------------
-def dumpSet( ns = None ):
-   """Dump all Gaudi objects that have had their properties set. The namespace must
-contain the application manager."""
-
-   if ns is None:
-      import __main__
-      ns = __main__.__dict__
-
-   for name, obj in ns.items():
-      if isinstance( obj, GaudiPython.Bindings.iProperty ) or name == "StoreGate":
-         pprint( obj )
-
-
-def dumpAlgorithms( ns = None ):
-   """Dump all algorithm properties. The namespace must contain the application mgr."""
-
-   if ns is None:
-      import __main__
-      ns = __main__.__dict__
-
-   _printFromLookup( ns, 'algorithms', 'Algorithm', 'TopAlg', GaudiPython.Bindings.iAlgorithm )
-
-
-def dumpServices( ns = None ):
-   """Dump all service properties. The namespace must contain the application mgr."""
-
-   if ns is None:
-      import __main__
-      ns = __main__.__dict__
-
-   _printFromLookup( ns, 'services', 'Service', [ 'StoreGateSvc' ], GaudiPython.Bindings.iService )

Control/AthenaCommon/python/PropertiesManip.py

-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
-
-# File: AthenaCommon/python/PropertiesManip.py
-# Author: Wim Lavrijsen (WLavrijsen@lbl.gov)
-
-"""Convenience functions to manipulate Gaudi object properties."""
-
-
-### data ---------------------------------------------------------------------
-__version__ = '1.0.0'
-__author__  = 'Wim Lavrijsen (WLavrijsen@lbl.gov)'
-
-__all__ = [ 'appendItemsToList', 'removeItemsFromList', 'removeFromList' ]
-
-
-### item appending for a list property --------------------------------------
-def appendItemsToList( owner, property, items ):
-   """Append the items to owner.property."""
-
-   if type(property) != str:
-      property = property.name()
-
-   if type(items) != list:
-      items = [ items ]
-
-   exec( 'owner.%s += items' % property )
-
-
-def removeItemsFromList( owner, property, items ):
-   """Remove all occurrences of the items from owner.property."""
-
-   if type(property) != str:
-      property = property.name()
-
-   if type(items) != list and type(items) != tuple:
-      items = [ items ]
-
-   current = getattr( owner, property )  # noqa: F841
-   exec( 'owner.%s = [ e for e in current if not e in items ]' % property )
-
-
-def removeFromList( owner, property, item ):
-   """Remove the first occurrence of item from owner.property."""
-
-   if type(property) != str:
-      property = property.name()
-
-   current = getattr( owner, property )
-   current.remove( item )
-   exec( 'owner.%s = current' % property )

Control/AthenaCommon/python/Resilience.py

-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
-
-# usage treatException(" problem with bla ")
-# catch exception if recConfFlags.AllowIgnoreConfigError otherwise rethrow
-#  from AthenaCommon.Resilience import treatException
-#  try:
-#      something=wrong
-#  except Exception:
-#      print "doSomethingIntelligent about this exception"
-#      treatException(" failed to something=wrong. Did something intelligent about it")
-
-def treatException (desc):
-  from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
-  if athenaCommonFlags.AllowIgnoreConfigError():
-      import traceback
-      traceback.print_exc()
-      print ("ERROR: "+desc)
-      return None
-  else:
-    raise
-
-# do comand with optional catch of exception
-# usage protect ("Could not load MakeAODCellsFromCluster!", include, "CaloRec/MakeAODCellsFromCluster_jobOptions.py")
-# be careful that this will not catch exception if an argument is invalid
-def protect (desc, fn, *args, **kw):
-  try:
-    return fn (*args, **kw)      
-  except Exception:
-    treatException(desc)
-    return None
-
-  
-
-# protectedInclude ("Could not load MakeAODCellsFromCluster!", "CaloRec/MakeAODCellsFromCluster_jobOptions.py")
-# exception is caught optionally
-def protectedInclude (jo):
-  from AthenaCommon.Include import include
-  return protect ("Could not load " + jo, include, jo)
-
-
-
-

Control/AthenaCommon/python/__init__.py

-# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
 
 # File: AthenaCommon/python/__init__.py
 # Author: Wim Lavrijsen (WLavrijsen@lbl.gov)
-
-__version__ = '1.7.0'
-__author__  = 'Wim Lavrijsen (WLavrijsen@lbl.gov)'
-
-__all__ = [ 'Constants', 'PhysicalConstants', 'SystemOfUnits',
-            'Include', 'Logging', 'ResourceLimits', 'ShellEscapes',
-            'DetFlags', 'GlobalFlags',
-            'PropertiesManip', 'Utils',
-            'AlgSequence', 'Configurable', 'CfgMgr' ]

Control/AthenaCommon/share/AthAppMgrUnitTests.ref

-Mon Aug  7 11:15:46 EDT 2023
-Py:Athena            INFO using release [?-22.0.0] [?] [?/?] -- built on [?]
-Py:Athena            INFO including file "AthenaCommon/Preparation.py"
-Py:Athena            INFO including file "AthenaCommon/Atlas.UnixStandardJob.py"
-Py:Athena            INFO executing ROOT6Setup
-Py:Athena            INFO including file "AthenaCommon/Execution.py"
-Py:Athena            INFO including file "AthenaCommon/AthAppMgrUnitTests.py"
-Py:Athena            INFO using release [?-22.0.0] [?] [?/?] -- built on [?]
-.Py:Athena            INFO using release [?-22.0.0] [?] [?/?] -- built on [?]
-..
-----------------------------------------------------------------------
-Ran 3 tests in 43.512s
-
-OK

Control/AthenaCommon/share/CFElementsTest.ref

-test_findAlgorithms (AthenaCommon.CFElements.TestConf2CF) ... ok
-test_findDeep (AthenaCommon.CFElements.TestConf2CF) ... ok
-test_findMissing (AthenaCommon.CFElements.TestConf2CF) ... ok
-test_findRespectingScope (AthenaCommon.CFElements.TestConf2CF) ... ok
-test_findTop (AthenaCommon.CFElements.TestConf2CF) ... ok
-test_flatCollectors (AthenaCommon.CFElements.TestConf2CF) ... ok
-test_findAlgorithms (AthenaCommon.CFElements.TestLegacyCF) ... ok
-test_findDeep (AthenaCommon.CFElements.TestLegacyCF) ... ok
-test_findMissing (AthenaCommon.CFElements.TestLegacyCF) ... ok
-test_findRespectingScope (AthenaCommon.CFElements.TestLegacyCF) ... ok
-test_findTop (AthenaCommon.CFElements.TestLegacyCF) ... ok
-test_flatCollectors (AthenaCommon.CFElements.TestLegacyCF) ... ok
-
-----------------------------------------------------------------------
-Ran 12 tests in 0.002s
-
-OK

Control/AthenaCommon/share/ConfigurableUnitTests.ref

-Mon Aug  7 11:18:54 EDT 2023
-Py:Athena            INFO using release [?-22.0.0] [?] [?/?] -- built on [?]
-Py:Athena            INFO including file "AthenaCommon/Preparation.py"
-Py:Athena            INFO including file "AthenaCommon/Atlas.UnixStandardJob.py"
-Py:Athena            INFO executing ROOT6Setup
-Py:Athena            INFO including file "AthenaCommon/Execution.py"
-Py:Athena            INFO including file "AthenaCommon/ConfigurableUnitTests.py"
-Py:ConfigurableDb    INFO Read module info for 4854 configurables from 2 genConfDb files
-Py:ConfigurableDb    INFO No duplicates have been found: that's good !
-..........
-----------------------------------------------------------------------
-Ran 10 tests in 4.460s
-
-OK

Control/AthenaCommon/share/JobOptionsUnitTests.ref

-Mon Aug  7 11:20:41 EDT 2023
-Py:Athena            INFO using release [?-22.0.0] [?] [?/?] -- built on [?]
-Py:Athena            INFO including file "AthenaCommon/Preparation.py"
-Py:Athena            INFO including file "AthenaCommon/Atlas.UnixStandardJob.py"
-Py:Athena            INFO executing ROOT6Setup
-Py:Athena            INFO including file "AthenaCommon/Execution.py"
-Py:Athena            INFO including file "AthenaCommon/JobOptionsUnitTests.py"
-# setting LC_ALL to "C"
-ApplicationMgr    SUCCESS 
-====================================================================================================================================
-                                                   Welcome to ApplicationMgr (GaudiCoreSvc v27r1p99)
-                                          running on karma on Mon Aug  7 11:20:56 2023
-====================================================================================================================================
-ApplicationMgr       INFO Application Manager Configured successfully
-....Py:ConfigurableDb    INFO Read module info for 4854 configurables from 2 genConfDb files
-Py:ConfigurableDb    INFO No duplicates have been found: that's good !
-...
-----------------------------------------------------------------------
-Ran 7 tests in 3.186s
-
-OK
-ApplicationMgr       INFO Application Manager Terminated successfully

Control/AthenaCommon/share/JobPropertiesUnitTests.ref

-Mon Aug  7 11:22:33 EDT 2023
-Py:Athena            INFO using release [?-22.0.0] [?] [?/?] -- built on [?]
-Py:Athena            INFO including file "AthenaCommon/Preparation.py"
-Py:Athena            INFO including file "AthenaCommon/Atlas.UnixStandardJob.py"
-Py:Athena            INFO executing ROOT6Setup
-Py:Athena            INFO including file "AthenaCommon/Execution.py"
-Py:Athena            INFO including file "AthenaCommon/JobPropertiesUnitTests.py"
-..Py:JobProperty :: WARNING You are attempting to access the jobproperty JobProperties.MyProperties.MyUnsetProperty which has not been set
-.
-----------------------------------------------------------------------
-Ran 3 tests in 0.000s
-
-OK

Control/AthenaCommon/share/KeyStoreUnitTests.ref

-Mon Aug  7 11:21:22 EDT 2023
-Py:Athena            INFO using release [?-22.0.0] [?] [?/?] -- built on [?]
-Py:Athena            INFO including file "AthenaCommon/Preparation.py"
-Py:Athena            INFO including file "AthenaCommon/Atlas.UnixStandardJob.py"
-Py:Athena            INFO executing ROOT6Setup
-Py:Athena            INFO including file "AthenaCommon/Execution.py"
-Py:Athena            INFO including file "AthenaCommon/KeyStoreUnitTests.py"
-...........
-----------------------------------------------------------------------
-Ran 11 tests in 0.313s
-
-OK
-outFileName: esd_3199293.dat

Control/AthenaCommon/share/MemTraceInclude.py

-from AthenaCommon.Include import Include
-class MemTraceInclude( Include ):
-   def __init__( self, org ):
-      Include.__init__( self, org._show )
-      for name, value in org.__dict__.items():
-         setattr (self, name, value)
-      import AthenaCommon.Logging as L
-      self.msg = L.logging.getLogger('Athena')
-      
-   def __call__( self, fn, *args, **kw ):
-      Include.__call__( self, fn, *args, **kw )
-
-      from sys import platform
-      if platform != 'darwin' :
-          pid = os.getpid()
-          statm = open( '/proc/%d/statm' % pid, 'r' ).readlines()[0].split()
-          stat =  open( '/proc/%d/stat' % pid, 'r' ).readlines()[0].split()
-
-          self.msg.info(
-             'VSIZE = %8.3f MB, SIZE = %8.3f MB, RSS = %8.3f MB',
-             int(stat[22])/1048576.,
-             int(statm[0])/256.,
-             int(statm[1])/256.
-             )
-      else:
-            # The following is placeholder code for the Mac to get vmem and rss in MB. This is available
-            # via the C task_info() function, for which there isn't a Python wrapper yet. The following is 
-         	# a hack using the 'ps' command which is certainly not optimal! The vsz and rss are claimed to
-         	# be returned from the 'ps' command in units of KB, but it looks like vsz is in pages rather than 
-         	# KB, so I'll scale it by that, whereas rsz does appear to be in units of KB. I don't know
-         	# what the equivalent of size is so I'm returning vsize for that.
-            from resource import getpagesize
-            from os import popen, getpid
-            pageSize = float(getpagesize())
-            vsz = float(popen('ps -p %d -o %s | tail -1' % (getpid(), "vsz")).read())
-            rss = float(popen('ps -p %d -o %s | tail -1' % (getpid(), "rss")).read())
-            vmem = vsz*pageSize / 1024.0 / 1024.0
-            rss  = rss / 1024.0
-         ####    print ('getpid(%d) pageSize(%f) vmem(%f) rss(%f)' % (getpid(), pageSize, vmem, rss))
-            self.msg.info(
-                'VSIZE = %8.3f MB, SIZE = %8.3f MB, RSS = %8.3f MB',
-                vmem,
-                vmem,
-                rss
-                )
-
-include = MemTraceInclude( include )
-
-# hack for folks who call "from AthenaCommon.Include import include" from other
-# python modules (those codes should be fixed ... )
-sys.modules[ 'AthenaCommon.Include' ].include = include

Control/AthenaCommon/share/zeroJO.py

-#Deliberately empty jobOptions file used for generating bootstrap file

Control/AthenaCommon/test/test_AthAppMgrUnitTests.sh

-#!/bin/sh
-athena.py AthenaCommon/AthAppMgrUnitTests.py

Control/AthenaCommon/test/test_ConfigurableUnitTests.sh

-#!/bin/sh
-athena.py AthenaCommon/ConfigurableUnitTests.py

Control/AthenaCommon/test/test_JobOptionsUnitTests.sh

-#!/bin/sh
-athena.py AthenaCommon/JobOptionsUnitTests.py

Control/AthenaCommon/test/test_JobPropertiesUnitTests.sh

-#!/bin/sh
-athena.py AthenaCommon/JobPropertiesUnitTests.py

Control/AthenaCommon/test/test_KeyStoreUnitTests.sh

-#!/bin/sh
-athena.py AthenaCommon/KeyStoreUnitTests.py