diff --git a/Tools/PyUtils/CMakeLists.txt b/Tools/PyUtils/CMakeLists.txt
index a2035f304db501c7e69770893591277868af8e7b..21f7fc908c7cd7071ab061317dbef54c7e2f7971 100644
--- a/Tools/PyUtils/CMakeLists.txt
+++ b/Tools/PyUtils/CMakeLists.txt
@@ -19,7 +19,7 @@ atlas_install_scripts( bin/acmd.py bin/checkFile.py bin/checkPlugins.py
bin/diffPoolFiles.py bin/dlldep.py bin/dso-stats.py bin/dump-athfile.py
bin/dumpAthfilelite.py bin/filter-and-merge-d3pd.py bin/getMetadata.py
bin/gprof2dot bin/issues bin/magnifyPoolFile.py bin/merge-poolfiles.py
- bin/pool_extractFileIdentifier.py
+ bin/apydep.py bin/pool_extractFileIdentifier.py
bin/pool_insertFileToCatalog.py bin/print_auditor_callgraph.py bin/pyroot.py
bin/vmem-sz.py bin/meta-reader.py bin/meta-diff.py bin/tree-orderer.py
POST_BUILD_CMD ${ATLAS_FLAKE8} )
@@ -54,3 +54,19 @@ atlas_add_test( RootUtils
atlas_add_test( fprint_test
SCRIPT python -m PyUtils.fprint )
+
+
+# Create python package dependencies in release building mode.
+# Used as input for `acmd.py cmake depends`:
+if( NOT "${CMAKE_PROJECT_NAME}" STREQUAL "WorkDir" )
+
+ set( _pydot ${CMAKE_CURRENT_BINARY_DIR}/packages.py.dot )
+ add_custom_command( OUTPUT ${_pydot}
+ COMMAND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/atlas_build_run.sh
+ ${CMAKE_CURRENT_SOURCE_DIR}/bin/apydep.py -o ${_pydot} -p ${CMAKE_BINARY_DIR}/${ATLAS_PLATFORM}/packages.txt ${CMAKE_SOURCE_DIR}/../../ )
+
+ add_custom_target( build_pydeps ALL DEPENDS ${_pydot} )
+
+ # Install output if available:
+ install( FILES ${_pydot} DESTINATION . OPTIONAL )
+endif()
diff --git a/Tools/PyUtils/bin/apydep.py b/Tools/PyUtils/bin/apydep.py
new file mode 100755
index 0000000000000000000000000000000000000000..c8778245935ad56a8a65583382c6424651458e2c
--- /dev/null
+++ b/Tools/PyUtils/bin/apydep.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+#
+# Created: Oct 2020, Frank Winklmeier
+#
+"""
+Extract Python dependencies between packages and create DOT graph.
+Both `import` and `include` dependencies are considered.
+"""
+
+import ast
+import sys
+import os
+import argparse
+import pygraphviz
+from collections import defaultdict
+
+class DependencyFinder(ast.NodeVisitor):
+ """Walk an AST collecting import/include statements."""
+
+ def __init__(self):
+ self.imports = set()
+ self.includes = set()
+
+ def visit_Import(self, node):
+ """import XYZ"""
+ self.imports.update(alias.name.split('.',1)[0] for alias in node.names)
+
+ def visit_ImportFrom(self, node):
+ """from XYZ import ABC"""
+ if node.level==0: # ignore relative imports
+ self.imports.add(node.module.split('.',1)[0])
+
+ def visit_Call(self, node):
+ """"include(XYZ/ABC.py)"""
+ if isinstance(node.func, ast.Name) and node.func.id=='include' and node.args:
+ if isinstance(node.args[0], ast.Str):
+ self.includes.add(node.args[0].s.split('/',1)[0])
+
+
+def get_dependencies(filename, print_error=False):
+ """Get all the imports/includes in a file."""
+
+ try:
+ tree = ast.parse(open(filename,'rb').read(), filename=filename)
+ except SyntaxError as e:
+ if print_error:
+ print(e, file=sys.stderr)
+ return DependencyFinder()
+
+ finder = DependencyFinder()
+ finder.visit(tree)
+
+ return finder
+
+
+def walk_tree(path='./', print_error=False, filterFnc=None):
+ """Walk the source tree and extract python dependencies, filtered by FilterFnc"""
+
+ pkg = 'UNKNOWN'
+ deps = defaultdict(lambda : defaultdict(set))
+ for root, dirs, files in os.walk(path):
+ if 'CMakeLists.txt' in files:
+ pkg = os.path.basename(root)
+
+ if (filterFnc and not filterFnc(pkg)):
+ continue
+
+ for f in filter(lambda p : os.path.splitext(p)[1]=='.py', files):
+ d = get_dependencies(os.path.join(root,f), print_error)
+ deps[pkg]['import'].update(d.imports)
+ deps[pkg]['include'].update(d.includes)
+
+ return deps
+
+
+def make_graph(deps, filterFnc=None):
+ """Save the dependencies as dot graph, nodes filtered by filterFnc"""
+
+ graph = pygraphviz.AGraph(name='AthPyGraph', directed=True)
+ for a in deps:
+ for t in ['import','include']:
+ graph.add_edges_from(((a,b) for b in deps[a][t]
+ if a!=b and (filterFnc is None or (filterFnc(a) and filterFnc(b)))),
+ label = t)
+ return graph
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+
+ parser.add_argument('path', metavar='DIRECTORY', nargs='?', default='./',
+ help='root of source tree [%(default)s]')
+
+ parser.add_argument('-o', '--output', metavar='FILE', type=str,
+ help='output file for DOT graph')
+
+ parser.add_argument('-p', '--packages', metavar='FILE', type=str,
+ help='path to packages.txt file [from release]')
+
+ parser.add_argument('-a', '--all', action='store_true',
+ help='include non-athena dependencies')
+
+ parser.add_argument('-v', '--verbose', action='store_true',
+ help='print parse errors')
+
+ args = parser.parse_args()
+
+ packages = None
+ if not args.all:
+ package_file = args.packages or os.path.join(os.environ['AtlasArea'],'InstallArea',
+ os.environ['BINARY_TAG'],'packages.txt')
+
+ try:
+ with open(package_file) as f:
+ packages = set(line.rstrip().split('/')[-1] for line in f if not line.startswith('#'))
+ except FileNotFoundError:
+ parser.error(f"Cannot read '{package_file}'. Specify via '-p/--packages' or run with '-a/--all'")
+
+ # By default only show athena packages:
+ filterFnc = None if args.all else lambda p : p in packages
+
+ # Walk source tree and create DOT graph:
+ g = make_graph(walk_tree(args.path, args.verbose, filterFnc), filterFnc)
+
+ if args.output:
+ g.write(args.output)
+ else:
+ print(g)
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/Tools/PyUtils/python/scripts/cmake_depends.py b/Tools/PyUtils/python/scripts/cmake_depends.py
index 0d3d628b068ea8bbb8247e91ec87f38494f50f90..df351c9b385b180f669425f115bb207620e854b4 100644
--- a/Tools/PyUtils/python/scripts/cmake_depends.py
+++ b/Tools/PyUtils/python/scripts/cmake_depends.py
@@ -93,6 +93,16 @@ def subgraph(graph, sources, reverse=False, maxdepth=None, nodegetter=lambda n :
return g
+def add_legend(graph):
+ """Add legend to graph"""
+ graph.add_subgraph(name='clusterLegend', label='Legend')
+ l = graph.subgraphs()[-1]
+ for n in 'abcd':
+ l.add_node(n, shape='point', style='invis')
+ l.add_edge('a','b', label='C++', constraint=False)
+ l.add_edge('c','d', label='Python', style='dashed', constraint=False)
+
+
class AthGraph:
"""Class to hold dependency information for release"""
@@ -104,13 +114,13 @@ class AthGraph:
# Build dictionary for node types:
legend = self.graph.get_subgraph('clusterLegend')
- self.types = { n.attr['label'] : n.attr['shape'] for n in legend.iternodes() }
+ self.types = { n.attr['label'] : n.attr['shape'] for n in legend.nodes_iter() }
# Build dictionary for node names:
- self.node = { n.attr['label'] : n.get_name() for n in self.graph.iternodes() }
+ self.node = { n.attr['label'] : n.get_name() for n in self.graph.nodes_iter() }
# Extract package dependencies:
- for e in self.graph.iteredges():
+ for e in self.graph.edges_iter():
p = e[0].attr['label']
# Decorate target with package name:
if p.startswith('Package_'):
@@ -119,7 +129,7 @@ class AthGraph:
# Assign "package" names to externals if possible:
external_nodes = filter(lambda n : 'package' not in n.attr.keys(),
- self.graph.iternodes())
+ self.graph.nodes_iter())
for n in external_nodes:
name = externals_name(n.attr['label'])
n.attr['package'] = name.split('::')[0]
@@ -162,6 +172,9 @@ class AthGraph:
type=int, default=1, const=None,
help='recursively resolve dependencies up to DEPTH (default: unlimited)')
+@acmdlib.argument('--py', action='store_true',
+ help='add Python dependencies')
+
@acmdlib.argument('--regex', action='store_true',
help='treat NAME as regular expression')
@@ -171,8 +184,14 @@ class AthGraph:
@acmdlib.argument('-d', '--dot', action='store_true',
help='print DOT graph')
+@acmdlib.argument('--noLegend', action='store_true',
+ help='do not add legend to graph')
+
+
# Debugging/expert options:
@acmdlib.argument('--cmakedot', help=argparse.SUPPRESS)
+@acmdlib.argument('--pydot', help=argparse.SUPPRESS)
+
def main(args):
"""Inspect cmake build dependencies"""
@@ -185,6 +204,18 @@ def main(args):
except KeyError:
main.parser.error("Cannot find 'packages.dot'. Setup a release or use --cmakedot.")
+ # Find packages.py.dot:
+ pygraph = None
+ if args.py:
+ if args.target:
+ main.parser.error("Python dependencies not possible in target mode.")
+
+ args.pydot = args.pydot or args.cmakedot.replace('.dot','.py.dot')
+ try:
+ pygraph = pygraphviz.AGraph(args.pydot)
+ except Exception:
+ main.parser.error(f"Cannot read '{args.pydot}'. Setup a release or use --pydot.")
+
# Read packages.txt if needed:
package_paths = {}
if args.long:
@@ -195,8 +226,9 @@ def main(args):
main.parser.error("Cannot read 'packages.txt'. Setup a release or run without -l/--long.")
# In package mode we have one extra level due to the Package_ target:
+ depth = args.recursive
if not args.target and not args.clients and args.recursive is not None:
- args.recursive += 1
+ depth += 1
# Read dependencies:
d = AthGraph(args.cmakedot, package_paths)
@@ -208,14 +240,14 @@ def main(args):
a = 'label' if args.target else 'package'
return node.attr[a]
- graph = pygraphviz.AGraph(name='AthGraph', directed=True)
+ graph = pygraphviz.AGraph(name='AthGraph', directed=True, strict=False)
for p in args.names:
target = p.split('/')[-1] # in case of full package path
# With regex, find all matching targets:
if args.regex:
r = re.compile(target)
- targets = [getnode(n) for n in d.graph.iternodes() if r.match(n.attr['label'])]
+ targets = [getnode(n) for n in d.graph.nodes_iter() if r.match(n.attr['label'])]
targets = list(filter(lambda t : t is not None, targets))
else:
targets = [target]
@@ -242,12 +274,31 @@ def main(args):
# Extract the dependency subgraph:
g = subgraph(d.graph, sources, reverse=args.clients,
- maxdepth=args.recursive, nodegetter=getnode)
+ maxdepth=depth, nodegetter=getnode)
graph.add_subgraph(name=target)
graph.get_subgraph(target).add_edges_from(g.edges())
- # Print result:
+ # Add python dependencies:
+ if args.py:
+ # Here the nodes are the actual package names:
+ pysources = [pygraph.get_node(t) for t in targets if pygraph.has_node(t)]
+ g = subgraph(pygraph, pysources, reverse=args.clients,
+ maxdepth=args.recursive, nodegetter=lambda n : n.name)
+
+ graph.get_subgraph(target).add_edges_from(g.edges(), style='dashed')
+
+ # Change style of nodes that have only Python dependencies:
+ g = graph.get_subgraph(target)
+ for n in g.nodes_iter():
+ if all(e.attr['style']=='dashed' for e in g.edges_iter(n)):
+ n.attr['style'] = 'dashed'
+
+
+ # Output final graph:
+ if not args.noLegend:
+ add_legend(graph)
+
if args.dot:
print(graph)
else: