Control/PerformanceMonitoring/PerfMonComps/CMakeLists.txt

@@ -23,5 +23,5 @@ atlas_add_component( PerfMonComps
    AthDSoCallBacks nlohmann_json::nlohmann_json)
 
 # Install files from the package:
-atlas_install_python_modules( python/*.py )
-atlas_install_joboptions( share/*.py )
+atlas_install_python_modules( python/*.py POST_BUILD_CMD ${ATLAS_FLAKE8} )
+atlas_install_joboptions( share/*.py POST_BUILD_CMD ${ATLAS_FLAKE8} )

Control/PerformanceMonitoring/PerfMonComps/python/DomainsRegistry.py

-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
 
 # @file PerfMonComps/python/DomainsRegistry.py
 # @purpose hold a registry of alg names and their association w/ domain
@@ -158,8 +158,6 @@ class Registry(object):
         """
         if registry is None:
             registry=self._registry
-        start_alg = None
-        idx = None
         for ielmt, elmt in enumerate(registry):
             if elmt[0] == name:
                 return ielmt, elmt[1]
@@ -328,7 +326,7 @@ class Registry(object):
         if not self._dirty_db:
             return dict(self._d2a_db)
         # side-effect of calling self.algs: will build  self._d2a_db
-        a2d = self.algs
+        a2d = self.algs # noqa: F841
         return dict(self._d2a_db)
 
     @property
@@ -443,9 +441,6 @@ def _test_main():
         print("    ref: ",ref[d])
         assert algs == ref[d]
 
-    db = pdr.a2d_db()
-    db = pdr.d2a_db()
-    
     print("OK")
     return 0
 

Control/PerformanceMonitoring/PerfMonComps/python/JobOptCfg.py

-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
 
 # @file: JobOptCfg.py
 # @purpose: a customized Configurable class for the PerfMonSvc
@@ -145,11 +145,11 @@ class PerfMonSvc( _PerfMonSvc ):
                     ioLabels = [ "streamRDO","streamESD",
                                  "streamAOD","streamTAG",
                                  "inputBackNav","inputFile" ]
-                    for l in ioLabels:
+                    for z in ioLabels:
                         try:
-                            ioContainers.extend(keystore[l].list())
+                            ioContainers.extend(keystore[z].list())
                         except AttributeError:
-                            for k,v in keystore[l].items():
+                            for k,v in keystore[z].items():
                                 ioContainers += [ "%s#%s" % (k,c) for c in v ]
                     pass
                 ## collect everything

Control/PerformanceMonitoring/PerfMonComps/python/MTJobOptCfg.py

-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
  
 # Job options configuration file for PerfMonMTSvc
 
@@ -26,8 +26,6 @@ class PerfMonMTSvc ( _PerfMonMTSvc  ):
         if not isinstance(handle, PerfMonMTSvc):
             return
 
-        from AthenaCommon import CfgMgr
-
         ## Enable the auditors
         from AthenaCommon.AppMgr import theApp
         theApp.AuditAlgorithms = True

Control/PerformanceMonitoring/PerfMonComps/python/PMonSD.py

-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
-from __future__ import print_function
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
 
 __doc__   ='Module for parsing and basic analysis of Semi-Detailed PerfMon (PMonSD) output. More info at https://twiki.cern.ch/twiki/bin/viewauth/Atlas/PerfMonSD'
 __author__='Thomas Kittelmann <thomas.kittelmann@cern.ch>'
@@ -15,7 +14,8 @@ def pmonsd_version():
 def parse(infile,outfile=None):
     """Parse PMonSD output and return list of dictionaries. Optionally save output in pickle file."""
     p=__smart_parse(infile)
-    if p==None: return None
+    if p is None:
+        return None
     if outfile:
         __save_output(p,outfile,'.psd',infile)
     return p
@@ -25,14 +25,15 @@ def deparse(infile):
     identical to the ones it was parsed from"""
     out=[]
     p=__smart_parse(infile)
-    if p==None: return []
+    if p is None:
+        return []
     for e in p:
         out+=__deparse_single(e)
     return out
 
-def need_line(l):
+def need_line(z):
     """To identify lines which PMonSD needs for parsing"""
-    return l.startswith(_appname)
+    return z.startswith(_appname)
 
 def print_ascii(infile):
     """Print parsed PMonSD info to stdout"""
@@ -52,13 +53,16 @@ def _validate_identical(infile1,infile2):
     #For validation
     p1=__smart_parse(infile1)
     p2=__smart_parse(infile2)
-    if p1==None or p2==None: return False
+    if p1 is None or p2 is None:
+        return False
     return p1==p2
 
 def force_share(obj):
     """Dive into lists and dictionaries and make sure strings with similar content gets shared"""
-    if type(obj)==list: __fs_list(obj)
-    elif type(obj)==dict: __fs_dict(obj)
+    if type(obj)==list:
+        __fs_list(obj)
+    elif type(obj)==dict:
+        __fs_dict(obj)
 
 def get_shared_string(s): return __get_shared_string(s)
 
@@ -94,7 +98,7 @@ def __save_output(data,outfile,prefix,infile=None):
         fh=gzip.open(outfile,'w')
     else:
         fh=open(outfile,'w')
-    if infile!=None and outfile==infile:
+    if infile is not None and outfile==infile:
         print("%s.parse WARNING: output file %s equals input file. Won't dump."%(_appname,outfile))
     else:
         import cPickle
@@ -146,8 +150,10 @@ def __smart_parse(infile):
     else:
         #in case this is already parsed info, make sure we just return it as it is:
         if type(infile)==list:
-            if len(infile)==0: return infile
-            if type(infile[0])==dict and 'steps_comps' in infile[0].keys(): return infile
+            if len(infile)==0:
+                return infile
+            if type(infile[0])==dict and 'steps_comps' in infile[0].keys():
+                return infile
         #Hopefully this is something we can iterate through (like a list of strings or a file-handle):
         return __actual_parse(infile)
 
@@ -168,13 +174,13 @@ def __actual_parse(filehandle):
         return (float(v),int(i))
     d=new_dict()
     stepcount={}#for keeping track of in what order within each step a component is listed
-    for l in filehandle:
-        if not l.startswith(_prefix):
+    for z in filehandle:
+        if not z.startswith(_prefix):
             continue
         #ensure the first thing we pick up is the version:
-        if version==None:
-            if intro_version in l:
-                vstr=l.split(intro_version)[1].split()[0]
+        if version is None:
+            if intro_version in z:
+                vstr=z.split(intro_version)[1].split()[0]
                 full_info=vstr[-1]=='f'
                 v_major,v_minor=vstr[:-1].split('.')
                 version=(int(v_major),int(v_minor))
@@ -185,22 +191,23 @@ def __actual_parse(filehandle):
                     print("WARNING: Using PMonSD of version %f to parse output made with version %f"%(pmonsd_version(),version))
             continue
         #remove prefix:
-        l=l[len(_prefix):].strip()
-        if l.startswith('WARNING'): continue
-        if l.startswith('=='):
+        z=z[len(_prefix):].strip()
+        if z.startswith('WARNING'):
+            continue
+        if z.startswith('=='):
             #This is a comment/separator. Look for end marker:
-            if end_marker in l:
+            if end_marker in z:
                 #found. Grab parsed info and make room for more (in case of concatenated logs)
                 output+=[d]
                 d=new_dict()
                 version=None#reset
-            elif 'Full output inside:' in l:
-                filename=l.split('Full output inside:')[1].split('===')[0].strip()
+            elif 'Full output inside:' in z:
+                filename=z.split('Full output inside:')[1].split('===')[0].strip()
                 d['fulloutput_file']=filename
             continue
-        if not l.startswith('['):
+        if not z.startswith('['):
             continue#ignore column headers
-        f=l.split()
+        f=z.split()
         if f[0]=='[---]' and '=' in f[1]:
             for valfield in f[1:]:
                 n,vstr=valfield.split('=',1)
@@ -226,15 +233,17 @@ def __actual_parse(filehandle):
                 d['special']['snapshots'][comp]={'n':n,'cpu':float(f[0]),'wall':float(f[1]),
                                                  'vmem':float(f[2]),'malloc':float(f[3])}
         else:
-            if not step in d['steps_comps'].keys():
+            if step not in d['steps_comps'].keys():
                 d['steps_comps'][step]={}
                 d['steps_totals'][step]={}
                 stepcount[step]=0
             iorder=stepcount[step]
             stepcount[step]+=1
                 #workaround situation where two collapsed or total lines have same form (nentries is always different):
-            if is_collapsed and comp in d['steps_comps'][step].keys(): comp+=':n=%i'%n
-            if is_total and comp in d['steps_totals'][step].keys(): comp+=':n=%i'%n
+            if is_collapsed and comp in d['steps_comps'][step].keys():
+                comp+=':n=%i'%n
+            if is_total and comp in d['steps_totals'][step].keys():
+                comp+=':n=%i'%n
             if len(f)==6:
                 #has max@evt info
                 d['steps_comps'][step][comp]={'order':iorder,'n':n,'cpu':float(f[0]),'vmem':float(f[2]),'malloc':float(f[4])}
@@ -242,8 +251,10 @@ def __actual_parse(filehandle):
             else:
                 #doesn't have max@evt info (step!='evt' or 'evt' but collapsed or total)
                 nfo={'order':iorder,'n':n,'cpu':float(f[0]),'vmem':float(f[1]),'malloc':float(f[2])}
-                if is_total: d['steps_totals'][step][comp]=nfo
-                else: d['steps_comps'][step][comp]=nfo
+                if is_total:
+                    d['steps_totals'][step][comp]=nfo
+                else:
+                    d['steps_comps'][step][comp]=nfo
     force_share(output)#make sure we register shared strings
     return output
 
@@ -251,14 +262,18 @@ def __deparse_single(d):
     _prefix=_appname+' '
     out=[]
     assert type(d)==dict
-    def header(l,s,center=True):
-        if center: s=(' %s '%s).center(82,'=')
-        else: s=(' %s '%s).ljust(82,'=')
-        l+=[ _prefix+'==='+s+'===']
+    def header(z,s,center=True):
+        if center:
+            s=(' %s '%s).center(82,'=')
+        else:
+            s=(' %s '%s).ljust(82,'=')
+        z+=[ _prefix+'==='+s+'===']
         
     full_info=d['full_info']
-    if full_info: fullstr='f'
-    else: fullstr='c'
+    if full_info:
+        fullstr='f'
+    else:
+        fullstr='c'
     header(out,'semi-detailed perfmon info v%i.%i%s / start'%(d['version'][0],d['version'][1],fullstr))
     header(out,'Documentation: https://twiki.cern.ch/twiki/bin/viewauth/Atlas/PerfMonSD',center=False)
     header(out,'Note that documentation includes recipe for easy parsing from python.  ',center=False)
@@ -268,7 +283,7 @@ def __deparse_single(d):
     stdsteps=['ini','1st','cbk','evt','fin']
     steps=[]
     for step in d['steps_comps'].keys():
-        if not step in stdsteps and not step in steps:
+        if step not in stdsteps and step not in steps:
             steps+=[step]
     steps.sort()
     steps=stdsteps+steps
@@ -284,20 +299,25 @@ def __deparse_single(d):
         is_evt=step=='evt'
         header(out,'step %s'%step)
         entries=[]
-        if not step in d['steps_comps'].keys(): continue
+        if step not in d['steps_comps'].keys():
+            continue
         for comp,compdata in d['steps_comps'][step].items():
-            if '_comps]:n=' in comp: comp=comp.split('_comps]:n=')[0]+'_comps]'
+            if '_comps]:n=' in comp:
+                comp=comp.split('_comps]:n=')[0]+'_comps]'
             if is_evt and comp in d['evt_max_info'].keys():
                 s=format_evt_withmax%(compdata['n'],compdata['cpu'],format_max(d['evt_max_info'][comp]['cpu']),
                                                  compdata['vmem'],format_max(d['evt_max_info'][comp]['vmem']),
                                                  compdata['malloc'],format_max(d['evt_max_info'][comp]['malloc']),comp)
             else:
-                if is_evt: format=format_evt_nomax
-                else: format=format_notevt
+                if is_evt:
+                    format=format_evt_nomax
+                else:
+                    format=format_notevt
                 s=format%(compdata['n'],compdata['cpu'],compdata['vmem'],compdata['malloc'],comp)
             entries+=[(compdata['order'],comp,s)]
         for comp,compdata in d['steps_totals'][step].items():
-            if '_comps]:n=' in comp: comp=comp.split('_comps]:n=')[0]+'_comps]'
+            if '_comps]:n=' in comp:
+                comp=comp.split('_comps]:n=')[0]+'_comps]'
             format='%4i %6i %7i %7i %s'
             if is_evt:
                 format='%4i %6i            %7i            %7i             %s'
@@ -305,8 +325,10 @@ def __deparse_single(d):
             entries+=[(compdata['order'],comp,s)]
         if entries:
             entries.sort()
-            if is_evt: out+=[ _prefix+' '*len(step)+colheader_evt]
-            else: out+=[ _prefix+' '*len(step)+colheader_std]
+            if is_evt:
+                out+=[ _prefix+' '*len(step)+colheader_evt]
+            else:
+                out+=[ _prefix+' '*len(step)+colheader_std]
             for _,_,s in entries:
                 out+=[ '%s[%s] %s'%(_prefix,step,s)]
     header(out,'special info')
@@ -323,8 +345,6 @@ def __deparse_single(d):
     for leak in leaks:
         dl=d['special']['leaks'][leak]
         out+=[ '%s[---] %4i        -        - %8i %8i %s'%(_prefix,dl['n'],dl['vmem'],dl['malloc'],leak)]
-    specialvals=d['special']['values'].keys()
-    svs=[]
     order=[['vmem_peak','vmem_mean','rss_mean'],
            ['jobcfg_walltime','jobstart'],
            ['cpu_bmips','cpu_res','release'],
@@ -335,8 +355,10 @@ def __deparse_single(d):
         lineformat=[]
         for sv in lineorder:
             v=d['special']['values'][sv]
-            if type(v)==float: v_str='%i'%v
-            else: v_str=v
+            if type(v)==float:
+                v_str='%i'%v
+            else:
+                v_str=v
             lineformat+=['%s=%s'%(sv,v_str)]
         out+=['%s[---] %s'%(_prefix,' '.join(lineformat))]
     header(out,'semi-detailed perfmon info / end')
@@ -352,17 +374,18 @@ def _validate_deparsing(f):
         fh=gzip_fastopen(f)
     else:
         fh=open(f)
-    for l in fh:
-        if l.startswith(_prefix):
-            if l.startswith(_prefix+'WARNING'):
+    for z in fh:
+        if z.startswith(_prefix):
+            if z.startswith(_prefix+'WARNING'):
                 continue
-            if l.endswith('\n'): l=l[0:-1]
-            lines+=[l]
+            if z.endswith('\n'):
+                z=z[0:-1]
+            lines+=[z]
     if len(lines)==0:
         print("File does not have %s lines!"%_appname)
         return False
     d=__smart_parse(lines)
-    if d==None:
+    if d is None:
         return False
     lines2=deparse(d)
     if len(lines)!=len(lines2):
@@ -388,7 +411,8 @@ def _validate_deparsing(f):
 def __actual_diff(infile1,infile2):
     d1=__smart_parse(infile1)
     d2=__smart_parse(infile2)
-    if d1==None or d2==None: return False
+    if d1 is None or d2 is None:
+        return False
     #Gymnastics to accept separate types:
     if type(d1)==list and type(d2)==list:
         if len(d1)!=len(d2):
@@ -422,13 +446,15 @@ def __actual_diff(infile1,infile2):
         anycollapsed=False
         for comp,data in compdata.items():
             n=data['n']
-            if not n in nentries2ncomps.keys(): nentries2ncomps[n]=0
+            if n not in nentries2ncomps.keys():
+                nentries2ncomps[n]=0
             if comp.startswith('[collapsed_'):
                 anycollapsed=True
-                nc=int(comp.split('_')[1])
+                #nc=int(comp.split('_')[1])
             else:
-                nc=1
-            nentries2ncomps[n]+=1
+                pass
+                #nc=1
+            nentries2ncomps[n]+=1 # check if this should be +=nc
         return nentries2ncomps,anycollapsed
 
 
@@ -455,7 +481,7 @@ def __actual_diff(infile1,infile2):
         if not anycollapsed1 and not anycollapsed2:
             #awesome, we can check all comps completely before vs. after
             for comp,compdata in d1['steps_comps'][step].items():
-                if not comp in d2['steps_comps'][step].keys():
+                if comp not in d2['steps_comps'][step].keys():
                     print("Difference: Component %s only present in one input in step %s"%(comp,step))
                     return False
                 check+=[(comp,compdata,d2['steps_comps'][step][comp])]
@@ -477,14 +503,17 @@ def __get_shared_string(s):
     global __allstrings
     return __allstrings.setdefault(s,s)
 
-def __fs_list(l):
-    i=len(l)
+def __fs_list(z):
+    i=len(z)
     while i:
         i-=1
-        t=type(l[i])
-        if t==str: l[i]=__get_shared_string(l[i])
-        elif t==list: __fs_list(l[i])
-        elif t==dict: __fs_dict(l[i])
+        t=type(z[i])
+        if t==str:
+            z[i]=__get_shared_string(z[i])
+        elif t==list:
+            __fs_list(z[i])
+        elif t==dict:
+            __fs_dict(z[i])
 
 def __fs_dict(d):
     keys=d.keys()
@@ -492,7 +521,10 @@ def __fs_dict(d):
         o=d[k]
         del d[k]
         t=type(o)
-        if t==str: o=__get_shared_string(o)
-        elif t==list: __fs_list(o)
-        elif t==dict: __fs_dict(o)
+        if t==str:
+            o=__get_shared_string(o)
+        elif t==list:
+            __fs_list(o)
+        elif t==dict:
+            __fs_dict(o)
         d[__get_shared_string(k)]=o

Control/PerformanceMonitoring/PerfMonComps/python/PerfMonFlags.py

-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
 
 # @file: PerfMonFlags.py
 # @purpose: a container of flags for Performance Monitoring
@@ -58,8 +58,6 @@ class doPersistencyMonitoring(JobProperty):
         if not jobproperties.PerfMonFlags.doMonitoring():
             jobproperties.PerfMonFlags.doMonitoring = True
             pass
-        from AthenaCommon.AppMgr import ServiceMgr as svcMgr
-        #svcMgr.PerfMonSvc.MonLvl = -1
         return
 # 
 class doDetailedMonitoring(JobProperty):
@@ -105,8 +103,6 @@ class doFullMon(JobProperty):
         jobproperties.PerfMonFlags.doFastMon = False
         jobproperties.PerfMonFlags.doMonitoring = True
         # setup values
-        from AthenaCommon.AppMgr import ServiceMgr as svcMgr
-        #svcMgr.PerfMonSvc.MonLvl = -1
         # enable DSO monitoring
         jobproperties.PerfMonFlags.doDsoMonitoring = True
         # activate persistency monitoring too
@@ -410,7 +406,7 @@ def _decode_pmon_opts(opts):
         elif opt.startswith('+'):
             val = True
             flag_name = flag_name[1:]
-        if not flag_name in dispatch:
+        if flag_name not in dispatch:
             raise ValueError(
                 '[%s] is not a valid PerfMonFlag (allowed: %r)' %
                 (flag_name, dispatch.keys())

Control/PerformanceMonitoring/PerfMonComps/python/PerfMonSerializer.py

-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
 
 # @file PerfMonComps/python/PerfMonSerializer
 
-from __future__ import with_statement, print_function
-
 __version__ = "$Revision: 524466 $"
 __doc__ = "various utils to encode/decode perfmon (meta)data with base64"
 __author__ = "Sebastien Binet, Thomas Kittlemann"
@@ -161,23 +159,22 @@ def iextract_pmon_data(fname):
     else:
         raise ValueError("expect a xyz.pmon.gz or xyz.stream file (got [%s])"%(fname,))
     
-    from collections import defaultdict
     import numpy as np
     out = _init_pmon_data()
         
     with open(stream_fname, 'r') as f:
-        for l in f:
+        for z in f:
             data, step, idx, comp = (None, ) * 4
-            if l.startswith('#'):
+            if z.startswith('#'):
                 continue
             #print("[%s]" % l.strip())
             # handle things like:
             # /io/std::vector<unsigned int>#L1CaloUnpackingErrors ...
             # /io/std::map<std::string,std::vector<int> >#mapdata ...
-            l = l.replace('unsigned int', 'unsigned-int')\
+            z = z.replace('unsigned int', 'unsigned-int')\
                  .replace('> >', '>->')
             
-            fields = l.split()
+            fields = z.split()
             #print("##",repr(l))
             if fields[0].startswith(('/ini/','/evt/','/fin/',
                                      '/cbk/','/usr/',
@@ -345,7 +342,7 @@ def iextract_pmon_data(fname):
                 pass
             else:
                 print("warning: unhandled field [%s]" % (fields[0],))
-                print(repr(l))
+                print(repr(z))
 
             # yields what we got so far
             yield step, idx, comp, out
@@ -391,7 +388,8 @@ def encode(data, use_base64=True):
 def decode(s):
     """decode a (compressed) string into a python object
     """
-    if not s: return None
+    if not s:
+        return None
     import zlib
     import cPickle as pickle
     if s[0]=='B':
@@ -399,7 +397,6 @@ def decode(s):
         s=base64.b64decode(s[1:])
     else:
         s=s[1:]
-    d=pickle.loads(zlib.decompress(s))
     return pickle.loads(zlib.decompress(s))
 
 def build_callgraph(fname):
@@ -417,9 +414,7 @@ def build_callgraph(fname):
     current_step = 'ini'
     local_ctx = None
     
-    out = None
     for step, idx, comp, table in iextract_pmon_data(fname):
-        out = table
         if idx is None:
             if comp == 'PerfMonSliceIo':
                 # ignore this component for now...
@@ -493,7 +488,7 @@ def build_callgraph(fname):
                 # push the stack of contexes
                 parent_ctx = local_ctx
                 local_ctx = GraphNode(comp, parent=parent_ctx)
-                if not step in graph.keys():
+                if step not in graph.keys():
                     local_ctx.ctype = step
                 parent_ctx.children.append(local_ctx)
             elif idx == 1:

Control/PerformanceMonitoring/PerfMonComps/python/PyComps.py

-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
 
 # @file:    PerfMonComps/python/PyComps.py
 # @purpose: a set of python components to perform performance monitoring
 # @author:  Sebastien Binet <binet@cern.ch>
-from __future__ import print_function
 
 __doc__     = 'a set of python components to perform performance monitoring'
 __version__ = '$Revision: 298807 $'
 __author__  = 'Sebastien Binet <binet@cern.ch>'
 
-import AthenaCommon.SystemOfUnits as Units
 import AthenaPython.PyAthena as PyAthena
 from AthenaPython.PyAthena import StatusCode
 
@@ -105,8 +103,8 @@ class PyStorePayloadMon (PyAthena.Svc):
             tp_name = clid2name(p.clID())
             print(fmt, (mem_0, mem_1, mem_0 - mem_1, tp_name, p.name()), file=fd)
             pass
-        mem_store_0 = long(mem_store_0)
-        mem_store_1 = long(mem_store_1)
+        mem_store_0 = int(mem_store_0)
+        mem_store_1 = int(mem_store_1)
         
         print(fmt, (
             mem_store_0, mem_store_1, mem_store_0 - mem_store_1,
@@ -133,7 +131,7 @@ class PyStorePayloadMon (PyAthena.Svc):
         ##     mem_0, mem_1, mem_1 - mem_0, ncalls_0, ncalls_1,
         ##     p.clID(), p.name()
         ##     ))
-        return (p, long(mem_0), long(mem_1))
+        return (p, int(mem_0), int(mem_1))
     
     def finalize(self):
         self.msg.info('==> finalize...')

Control/PerformanceMonitoring/PerfMonComps/python/PyMonUtils.py

-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
 
 # @file: PyMonUtils.py
 # @author: Sebastien Binet <binet@cern.ch>
-from __future__ import print_function
 
 __author__  = "Sebastien Binet <binet@cern.ch>"
 __version__ = "$Revision: 1.3 $"
@@ -44,7 +43,7 @@ def mon_push_back (sgname='StoreGateSvc'):
         # no double counting from symlinks
         # FIXME: it is actually valid to have 2 different collections
         #        (=/= CLIDs) with the same key...
-        if wasted.has_key(k):
+        if k in wasted:
             continue
         clid = dp.clID()
         klass = "%s" % cl.typename(clid)
@@ -81,18 +80,22 @@ def mon_push_back (sgname='StoreGateSvc'):
 
 def dump_smaps (fname=None):
     import os,sys
-    if not (fname is None): o = open (fname, 'w')
-    else:                   o = sys.stdout
-    for l in open('/proc/%d/smaps'%os.getpid()):
-        print(l, file=o)
+    if not (fname is None):
+        o = open (fname, 'w')
+    else:
+        o = sys.stdout
+    for z in open('/proc/%d/smaps'%os.getpid()):
+        print(z, file=o)
     if not (fname is None):
         o.close()
     return
 
 def loaded_libs (fname=None, pid=None, show=False):
     import os,sys,re
-    if not (fname is None): o = open (fname, 'w')
-    else:                   o = sys.stdout
+    if not (fname is None):
+        o = open (fname, 'w')
+    else:
+        o = sys.stdout
     pat = re.compile(r'(?P<addr_beg>\w*?)\-(?P<addr_end>\w*?)\s'\
                      r'(?P<perm>.{4})\s(?P<offset>\w*?)\s'\
                      r'(?P<devmajor>\d{2}):(?P<devminor>\d{2})\s'\
@@ -102,13 +105,13 @@ def loaded_libs (fname=None, pid=None, show=False):
     if pid is None:
         pid = os.getpid()
     for line in open('/proc/%s/smaps'%pid):
-        l = line.strip()
-        res = re.match(pat,l)
+        z = line.strip()
+        res = re.match(pat,z)
         if res:
             g = res.group
             libname = g('libname').strip()
             libs.add(_realpath(libname))
-    libs = sorted([l for l in libs], reverse=True)
+    libs = sorted([z for z in libs], reverse=True)
     if show:
         for libname in libs:
             print(libname, file=o)
@@ -117,8 +120,6 @@ def loaded_libs (fname=None, pid=None, show=False):
 import sys
 if sys.platform == 'darwin':
     def pymon():
-        from os import getpid,sysconf
-        from sys import platform
         from resource import getrusage, RUSAGE_SELF
         cpu = getrusage(RUSAGE_SELF)
         cpu = (cpu.ru_utime+cpu.ru_stime) * 1e3 # in milliseconds
@@ -138,7 +139,6 @@ if sys.platform == 'darwin':
 else:
     def pymon():
         from os import getpid,sysconf
-        from sys import platform
         from resource import getrusage, RUSAGE_SELF
         cpu = getrusage(RUSAGE_SELF)
         cpu = (cpu.ru_utime+cpu.ru_stime) * 1e3 # in milliseconds
@@ -152,13 +152,17 @@ else:
 def lshosts_infos():
     import socket,commands
     hostname = '<unknown>'
-    try: hostname = socket.gethostname()
-    except Exception: pass
+    try:
+        hostname = socket.gethostname()
+    except Exception:
+        pass
     sc,out = commands.getstatusoutput('which lshosts')
-    if sc != 0: return ('no lshosts command',0.) # no lshosts could be found
+    if sc != 0:
+        return ('no lshosts command',0.) # no lshosts could be found
     cmd = out
     sc,out = commands.getstatusoutput("%s %s"%(cmd,hostname))
-    if sc != 0: return ('host not in db', 0.)
+    if sc != 0:
+        return ('host not in db', 0.)
     cpu_infos = {}
     try:
         title,data = out.splitlines()

Control/PerformanceMonitoring/PerfMonComps/python/PyPerfMon.py

@@ -2,7 +2,6 @@
 
 # @file: PyPerfMon.py
 # @author: Sebastien Binet <binet@cern.ch>
-from __future__ import with_statement
 
 __author__  = "Sebastien Binet <binet@cern.ch>"
 __version__ = "$Revision: 1.51 $"
@@ -10,17 +9,12 @@ __doc__     = """python module holding a python service to monitor athena perfor
 """
 
 import os,sys
-from time import time
-import resource
-from resource import getrusage as resource_getrusage
-import string
 
-import array
 import AthenaCommon.Logging as L
 
 _perfMonStates = ('ini','evt','fin')
 
-from PerfMonComps.PyMonUtils import *
+from PerfMonComps.PyMonUtils import Units, pymon
 
 from PyUtils.Decorators import memoize, forking
 
@@ -54,7 +48,6 @@ class Svc(object):
     instances = {}
 
     def __init__(self, name, properties = None):
-        import AthenaCommon.Logging as L
         ## init base class
         super(Svc,self).__init__()
         self.name  = name
@@ -94,9 +87,12 @@ class Svc(object):
                 cfg_module = 'PerfMonComps'
             elif c in cfgs:
                 cfg = cfgs[c]
-                if isinstance(cfg, ConfigurableAlgorithm): cfg_type = 'alg'
-                elif isinstance(cfg, ConfigurableAlgTool): cfg_type = 'algtool'
-                elif isinstance(cfg, ConfigurableService): cfg_type = 'svc'
+                if isinstance(cfg, ConfigurableAlgorithm):
+                    cfg_type = 'alg'
+                elif isinstance(cfg, ConfigurableAlgTool):
+                    cfg_type = 'algtool'
+                elif isinstance(cfg, ConfigurableService):
+                    cfg_type = 'svc'
                 cfg_class  = cfg.__class__.__name__
                 cfg_module = cfg.__class__.__module__
             else:
@@ -139,7 +135,7 @@ class Svc(object):
 
         ## perfmon domains
         try:
-            import DomainsRegistry as pdr
+            import PerfMonComps.DomainsRegistry as pdr
             self.meta['domains_a2d'] = pdr.a2d_db()
         except Exception:
             _msg.info('problem retrieving domains-registry...')
@@ -159,7 +155,9 @@ class Svc(object):
                 'rt':       (0.,0.),
                 }
 
-        import gc; gc.collect(); del gc
+        import gc
+        gc.collect()
+        del gc
         return
 
     def domains_db(self):
@@ -170,7 +168,6 @@ class Svc(object):
     
     @property
     def msg(self):
-        import AthenaCommon.Logging as L
         return L.logging.getLogger(self.name)
 
     def _set_stats(self, name,
@@ -227,7 +224,7 @@ class Svc(object):
             self._do_malloc_mon = False
         _msg.info('installing pmon-malloc hooks: %s', self._do_malloc_mon)
         import AthenaPython.PyAthena as PyAthena
-        lib = PyAthena.load_library('PerfMonEventDict')
+        PyAthena.load_library('PerfMonEventDict')
         memstats = PyAthena.PerfMon.MemStats
         memstats.enable(bool(self._do_malloc_mon))
         _msg.info('pmon-malloc hooks enabled: %s', bool(memstats.enabled()))
@@ -308,10 +305,10 @@ class Svc(object):
             statm = {}
             from sys import platform
             if platform != 'darwin' :
-                for l in open('/proc/self/status', 'r'):
+                for z in open('/proc/self/status', 'r'):
                     # lines are of the form:
                     # VmPeak: some value
-                    ll = list(map(str.strip, l.split(':')))
+                    ll = list(map(str.strip, z.split(':')))
                     k = ll[0]
                     v = ' '.join(ll[1:])
                     statm[k] = v
@@ -343,9 +340,9 @@ class Svc(object):
             for evtstr,fitn,fitted_slope in self._slope_data['fits']:
                 maxfitn=max(maxfitn,fitn)
             for evtstr,fitn,fitted_slope in self._slope_data['fits']:
-                _msg.info( '  evt %s fitted vmem-slope (%s points): %s'%
-                           (evtstr,str(fitn).rjust(len(str(maxfitn))),
-                            '%7.1f kb/evt'%fitted_slope if fitn>=2 else 'N/A') )
+                _msg.info( '  evt %s fitted vmem-slope (%s points): %s',
+                           evtstr,str(fitn).rjust(len(str(maxfitn))),
+                            '%7.1f kb/evt'%fitted_slope if fitn>=2 else 'N/A' )
             summary['job']['vmem_slope'] = self._slope_data
         else:
             _msg.info('vmem-leak estimation: [N/A]')
@@ -353,8 +350,10 @@ class Svc(object):
             
         ## try to recoup some memory by flushing out ROOT stuff...
         headerFile = os.path.splitext(self.outFileName)[0]+".dat"
-        if os.path.exists(headerFile):       os.remove(headerFile)
-        if os.path.exists(self.outFileName): os.remove(self.outFileName)
+        if os.path.exists(headerFile):
+            os.remove(headerFile)
+        if os.path.exists(self.outFileName):
+            os.remove(self.outFileName)
 
         ## build the callgraph...
         #import PerfMonComps.PerfMonSerializer as pmon_ser
@@ -431,12 +430,15 @@ class Svc(object):
         ## write out meta-data
         import PyUtils.dbsqlite as dbs
         meta = dbs.open(headerFile, 'n')
-        for k,v in six.iteritems (self.meta): meta[k] = v
+        for k,v in six.iteritems (self.meta):
+            meta[k] = v
         meta['version_id'] = '0.4.0' # stream-format + header file
         meta['pmon_tuple_files'] = map( os.path.basename, outFiles[1:] )
         import socket
-        try:   meta['hostname'] = socket.gethostname()
-        except Exception: meta['hostname'] = '<unknown>'
+        try:
+            meta['hostname'] = socket.gethostname()
+        except Exception:
+            meta['hostname'] = '<unknown>'
         meta.close()
 
         
@@ -447,8 +449,10 @@ class Svc(object):
         try:
             for outFile in outFiles:
                 outFileDirName = os.path.dirname(outFile)
-                try: os.chdir(outFileDirName)
-                except OSError as err: pass
+                try:
+                    os.chdir(outFileDirName)
+                except OSError:
+                    pass
                 outFile = os.path.basename(outFile)
                 _msg.info(' --> [%s] => %8.3f kb',
                           outFile,
@@ -517,10 +521,12 @@ class PoolMonTool(object):
         from AthenaCommon import CfgMgr
         from AthenaCommon.Configurable import Configurable
         for c in list(Configurable.allConfigurables.values()):
-            if not isinstance(c, CfgMgr.AthenaOutputStream): continue
+            if not isinstance(c, CfgMgr.AthenaOutputStream):
+                continue
             try:
                 outFile = c.properties()["OutputFile"]
-            except KeyError: continue
+            except KeyError:
+                continue
             if outFile.startswith("ROOTTREE:"):
                 outFile = outFile[len("ROOTTREE:"):]
             outFiles.add( outFile )
@@ -530,7 +536,6 @@ class PoolMonTool(object):
         
     @property
     def msg(self):
-        import AthenaCommon.Logging as L
         return L.logging.getLogger(self.name)
 
     def initialize(self):
@@ -620,7 +625,8 @@ class PoolMonTool(object):
                     self.msg.info( "Could not run checkFile on [%s] !!",
                                    inFileName )
                     self.msg.info( "Reason: %s", err )
-                    if 'inFile' in dir(): del inFile               
+                    if 'inFile' in dir():
+                        del inFile
                 _msg.unMute()
         if len(self.outputPoolFiles)>0:
             self.msg.info( "Content of output POOL files:" )
@@ -651,7 +657,8 @@ class PoolMonTool(object):
                     self.msg.info( "Could not run checkFile on [%s] !!",
                                    outFileName )
                     self.msg.info( "Reason: %s", err )
-                    if 'outFile' in dir(): del outFile               
+                    if 'outFile' in dir():
+                        del outFile
                 _msg.unMute()
                 
         return
@@ -678,13 +685,13 @@ class HephaestusMonTool(object):
         # during our finalize.
         self._heph_has_checkPoint = False 
         import sys
-        if not 'Hephaestus.atexit' in sys.modules.keys():
+        if 'Hephaestus.atexit' not in sys.modules.keys():
             self.msg.warning('Hephaestus was not correctly initialized !')
             self.msg.warning('Final report may be inaccurate...')
             self.msg.warning('(to fix this, run athena with --leak-check)')
 
         import dl, Hephaestus.MemoryTracker as m
-        _hephLib = dl.open (m.__file__, dl.RTLD_GLOBAL | dl.RTLD_NOW)
+        dl.open (m.__file__, dl.RTLD_GLOBAL | dl.RTLD_NOW)
         memtrack = m
 
         from os.path import splitext
@@ -742,7 +749,7 @@ class HephaestusMonTool(object):
         
         # consolidate last events with end-of-job leak report
         _clearCheckPoint = self.memtrack.CheckPoints.clearCheckPoint
-        for _ in xrange(self.lag):
+        for _ in range(self.lag):
             _clearCheckPoint( 0 )
 
         # put the per-evt leaks into a different file

Control/PerformanceMonitoring/PerfMonComps/share/FastMon.py

+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
 # @file: PerfMonComps/FastMon.py
 # @author: Sebastien Binet
 # $Id: FastMon.py,v 1.2 2007-12-03 19:07:38 binet Exp $

Control/PerformanceMonitoring/PerfMonComps/share/FullMon.py

+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
 # @file: PerfMonComps/FullMon.py
 # @author: Sebastien Binet
 # $Id$

Control/PerformanceMonitoring/PerfMonComps/share/PerfMonMTSvc_jobOptions.py

+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
 ###############################
 # Print what we're doing
 ###############################

Control/PerformanceMonitoring/PerfMonComps/share/PerfMonSvc_jobOptions.py

+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
 # @file: PerfMonComps/PerfMonSvc_jobOptions.py
 # @author: Sebastien Binet
 # $Id: PerfMonSvc_jobOptions.py,v 1.3 2007-08-01 20:58:52 binet Exp $

Control/StoreGate/src/SGImplSvc.cxx

@@ -1311,15 +1311,21 @@ SGImplSvc::removeProxy(DataProxy* proxy, const void* pTrans,
   }
 
   // remove all entries from t2p map
-  this->t2pRemove(pTrans);
-  SG::DataProxy::CLIDCont_t clids = proxy->transientID();
-  for (SG::DataProxy::CLIDCont_t::const_iterator i = clids.begin();
-       i != clids.end();
-       ++i)
+  //  --- only if the proxy actually has an object!
+  //      otherwise, we can trigger I/O.
+  //      besides being useless here, we can get deadlocks if we
+  //      call into the I/O code while holding the SG lock.
+  if (proxy->isValidObject()) {
+    this->t2pRemove(pTrans);
+    SG::DataProxy::CLIDCont_t clids = proxy->transientID();
+    for (SG::DataProxy::CLIDCont_t::const_iterator i = clids.begin();
+         i != clids.end();
+         ++i)
     {
       void* ptr = SG::DataProxy_cast (proxy, *i);
       this->t2pRemove(ptr);
     }
+  }
 
   // remove from store
   return m_pStore->removeProxy(proxy, forceRemove, true);

Event/xAOD/xAODMetaData/Root/FileMetaData_v1.cxx

 /*
-  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+  Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
 */
 
 // $Id: FileMetaData_v1.cxx 683694 2015-07-17 09:03:52Z krasznaa $
@@ -362,8 +362,6 @@ namespace xAOD {
       return true;
    }
 
-} // namespace xAOD
-
 /// Helper macro used to print MetaDataType values
 #define PRINT_TYPE( TYPE )                      \
    case xAOD::FileMetaData_v1::TYPE:            \
@@ -404,3 +402,5 @@ std::ostream& operator<< ( std::ostream& out,
    return out;
 }
 
+
+} // namespace xAOD

Event/xAOD/xAODMetaData/xAODMetaData/versions/FileMetaData_v1.h

 // Dear emacs, this is -*- c++ -*-
 
 /*
-  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+  Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
 */
 
 // $Id: FileMetaData_v1.h 685184 2015-07-23 20:25:43Z cranshaw $
@@ -106,11 +106,11 @@ namespace xAOD {
 
    }; // class FileMetaData_v1
 
-} // namespace xAOD
+   /// A convenience print operator for xAOD::FileMetaData_v1::MetaDataType
+   std::ostream& operator<< ( std::ostream& out,
+                              xAOD::FileMetaData_v1::MetaDataType type );
 
-/// A convenience print operator for xAOD::FileMetaData_v1::MetaDataType
-std::ostream& operator<< ( std::ostream& out,
-                           xAOD::FileMetaData_v1::MetaDataType type );
+} // namespace xAOD
 
 // Declare a base class for the type:
 #include "xAODCore/BaseInfo.h"

ForwardDetectors/ZDC/ZdcConditions/ZdcConditions/ZdcCablingService.h

@@ -74,7 +74,6 @@ public:
     int         m_channel_db[4][64];
     int         m_hv_db[4][64];
     int         m_ppm_db[16];
-    int         m_crate_db[8];
     int         m_crate_index[4];
     int         m_ncrate;
     //int         m_crate_lookup[2][3][2];

InnerDetector/InDetMonitoring/PixelMonitoring/src/PixelAthErrorMonAlg.cxx

@@ -137,8 +137,8 @@ StatusCode PixelAthErrorMonAlg::fillHistograms( const EventContext& ctx ) const
       is_fei4 = false;
     }
     // flagging/counting categorized errors per module.
-    bool has_err_cat[ErrorCategory::COUNT][nFEIBL2D] = {false};
-    int nerrors_cat_rodmod[ErrorCategoryRODMOD::COUNT][nFEIBL2D] = {0};
+    bool has_err_cat[ErrorCategory::COUNT][nFEIBL2D] = {{false}};
+    int nerrors_cat_rodmod[ErrorCategoryRODMOD::COUNT][nFEIBL2D] = {{0}};
 
     // count number of words w/ MCC/FE flags per module
     unsigned int num_femcc_errwords = 0;

InnerDetector/InDetRecTools/InDetRecToolInterfaces/InDetRecToolInterfaces/IInDetTestBLayerTool.h

 /*
-  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+  Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
 */
 
 #ifndef IInDetTestBLayerTool_H
 #define IInDetTestBLayerTool_H
-#include "GaudiKernel/EventContext.h"
 #include "GaudiKernel/IAlgTool.h"
-#include "GaudiKernel/ThreadLocalContext.h"
+#include "GaudiKernel/ThreadLocalContext.h" //for Gaudi::Hive::currentContext()
 #include "TrkParameters/TrackParameters.h"
-#include "TrkEventPrimitives/ResidualPull.h"
 #include <vector>
-#include <string>
 
 namespace Trk {
-class Track;
-class IResidualPullCalculator;
+  class Track;
+  class IResidualPullCalculator;
+  class ResidualPull;
 }
+
 namespace InDet {
 class TrackStateOnBLayerInfo;
 }
 
+class EventContext;
+
 namespace InDet {