WIP: Add aunused script

Tools/PyUtils/CMakeLists.txt

-# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration
 
 # Declare the package name:
 atlas_subdir( PyUtils )
@@ -19,7 +19,7 @@ atlas_install_scripts( bin/acmd.py bin/checkFile.py bin/checkPlugins.py
    bin/diffPoolFiles.py bin/dlldep.py bin/dso-stats.py bin/dump-athfile.py
    bin/dumpAthfilelite.py bin/filter-and-merge-d3pd.py bin/getMetadata.py
    bin/gprof2dot bin/issues bin/magnifyPoolFile.py bin/merge-poolfiles.py
-   bin/apydep.py bin/pool_extractFileIdentifier.py
+   bin/apydep.py bin/aunused.py bin/pool_extractFileIdentifier.py
    bin/pool_insertFileToCatalog.py bin/print_auditor_callgraph.py bin/pyroot.py
    bin/vmem-sz.py bin/meta-reader.py bin/meta-diff.py bin/tree-orderer.py
    POST_BUILD_CMD ${ATLAS_FLAKE8} )

Tools/PyUtils/bin/aunused.py

+#!/usr/bin/env python3
+# Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration
+#
+# Created: Oct 2020, Frank Winklmeier
+#
+"""
+Extract Python dependencies between packages and create DOT graph.
+Both `import` and `include` dependencies are considered.
+"""
+
+import ast
+import sys
+import os
+import argparse
+from pathlib import Path
+
+class DependencyFinder(ast.NodeVisitor):
+   """Walk an AST collecting import/include statements."""
+
+   def __init__(self, filename, pkg):
+      self.filename = filename
+      self.pkg = pkg
+      self.symbols = set()
+
+   def visit_Import(self, node):
+      """import XYZ"""
+      self.symbols.update(alias.name for alias in node.names)
+
+   def visit_ImportFrom(self, node):
+      """from XYZ import ABC"""
+      prefix = ''
+      if node.level>0:
+         parent = parent_module(self.filename)
+         if parent is None: return   # Import must be buggy
+         prefix = '.'.join((self.pkg,) + parent_module(self.filename)[-node.level:]) + '.'
+
+      self.symbols.update(prefix + (node.module or '') + '.' + alias.name for alias in node.names)
+
+   #def visit_Call(self, node):
+   #   """"include(XYZ/ABC.py)"""
+   #   if isinstance(node.func, ast.Name) and node.func.id=='include' and node.args:
+   #      if isinstance(node.args[0], ast.Str):
+   #         self.symbols.add(node.args[0].s)
+
+
+def parent_module(filename):
+   """Return parent module name for filename: .../python/A/B/c.py -> ['A','B']"""
+   p = Path(filename).parts
+   try:
+      return p[p.index('python')+1:-1]
+   except ValueError:  # no python directory
+      return None
+
+def get_python_modules():
+   """Return all available python modules as Pkg.Module"""
+
+   root = Path().joinpath(os.environ['AtlasArea'], 'InstallArea',
+                           os.environ['BINARY_TAG'], 'python')
+
+   return [str(p.relative_to(root).with_suffix('')).replace(os.sep,'.')
+           for p in root.rglob('*.py') if p.name!='__init__.py']
+
+
+def unused_modules(symbols):
+   modules = set(get_python_modules())
+   used = set()
+   for m in modules:
+      for s in symbols:
+         if s.startswith(m):
+            used.add(m)
+            continue
+
+   #for s in symbols: print(s)
+   for m in sorted(modules-used):
+      print(m)
+
+def walk_tree(path='./', print_error=False, filterFnc=None):
+   """Walk the source tree and extract python dependencies, filtered by FilterFnc"""
+
+   pkg = 'UNKNOWN'
+   all_symbols = set()
+   for root, dirs, files in os.walk(path):
+      if 'CMakeLists.txt' in files:
+         pkg = os.path.basename(root)
+
+      if (filterFnc and not filterFnc(pkg)):
+         continue
+
+      for f in filter(lambda p : os.path.splitext(p)[1]=='.py', files):
+         filename = os.path.join(root,f)
+
+         try:
+            tree = ast.parse(open(filename,'rb').read(), filename=filename)
+         except SyntaxError as e:
+            if print_error:
+               print(e, file=sys.stderr)
+            continue
+
+         finder = DependencyFinder(filename, pkg)
+         finder.visit(tree)
+
+         all_symbols.update(finder.symbols)
+
+   unused_modules(all_symbols)
+
+
+
+
+def main():
+   parser = argparse.ArgumentParser(description=__doc__)
+
+   parser.add_argument('path', metavar='DIRECTORY', nargs='?', default='./',
+                       help='root of source tree [%(default)s]')
+
+   parser.add_argument('-o', '--output', metavar='FILE', type=str,
+                       help='output file for DOT graph')
+
+   parser.add_argument('-p', '--packages', metavar='FILE', type=str,
+                       help='path to packages.txt file [from release]')
+
+   parser.add_argument('-a', '--all', action='store_true',
+                       help='include non-athena dependencies')
+
+   parser.add_argument('-v', '--verbose', action='store_true',
+                       help='print parse errors')
+
+   args = parser.parse_args()
+
+   packages = None
+   if not args.all:
+      package_file = args.packages or os.path.join(os.environ['AtlasArea'],'InstallArea',
+                                                   os.environ['BINARY_TAG'],'packages.txt')
+
+      try:
+         with open(package_file) as f:
+            packages = set(line.rstrip().split('/')[-1] for line in f if not line.startswith('#'))
+      except FileNotFoundError:
+         parser.error(f"Cannot read '{package_file}'. Specify via '-p/--packages' or run with '-a/--all'")
+
+   #print(get_python_modules())
+   # By default only show athena packages:
+   filterFnc = None if args.all else lambda p : p in packages
+
+   # Walk source tree and create DOT graph:
+   walk_tree(args.path, args.verbose, filterFnc)
+
+
+
+if __name__ == "__main__":
+   sys.exit(main())