Merge branch 'apearce-flow-graphs' into 'master'

Generate data and control flow graphs by default

Closes #62, #18, and #48

See merge request lhcb/Moore!225

Hlt/Hlt1Conf/options/hlt1_example.py

@@ -31,4 +31,3 @@ with FTRawBankDecoder.bind(DecodingVersion=ftdec_v), \
 
 setupInputFromTestFileDB('MiniBrunel_2018_MinBias_FTv4_DIGI')
 env.configure()
-# env.plotDataFlow()

Hlt/Hlt2Conf/options/hlt2_example.py

@@ -70,4 +70,3 @@ with FTRawBankDecoder.bind(DecodingVersion=ftdec_v), \
 env.register_public_tool(stateProvider_with_simplified_geom())
 
 env.configure()
-# env.plotDataFlow()

PyConf/options/example.py

@@ -64,4 +64,3 @@ line1 = CompositeNode("line1", children=[float1, float2])
 e = EverythingHandler(threadPoolSize=4, nEventSlots=5, evtMax=50)
 e.addNode(line1)
 e.configure()
-e.plotDataFlow("example_dataflow.gv")

PyConf/python/PyConf/components.py

@@ -28,11 +28,12 @@ import inspect
 import re
 import json
 
+import pydot
+
 from GaudiKernel.ConfigurableMeta import ConfigurableMeta
 
 from . import ConfigurationError, configurable
 from .dataflow import DataHandle, configurable_outputs, configurable_inputs, dataflow_config, contains_datahandle
-from .utilities import graphviz_module
 
 __all__ = [
     'Algorithm',
@@ -543,46 +544,63 @@ class Algorithm(object):
         return config
 
     def _graph(self, graph):
-        """Fill the graphviz.Digraph argument with our dataflow."""
-        graphviz = graphviz_module()
-        if graphviz is None:
-            return
+        """Add our dataflow as a `pydot.Subgraph` to graph.
+
+        Parameters
+        ----------
+        graph -- pydot.Graph
+        """
 
         # TODO deduplicate graphing?
         # in all diamond structures, the subgraph is built multiple times
 
+        def format_prop(name, value, max_len=100):
+            assert max_len > 15, 'max_len should be at least 15'
+            value = str(value)
+            if len(value) > max_len:
+                value = (value[:max_len // 2] + '[...]' +
+                         value[max_len // 2 + 5:max_len])
+            return '{} = {}'.format(name, value)
+
         #inner part ########
         own_name = html_escape(self.fullname)
-        g = graphviz.Digraph(name='cluster_' + own_name)
+        sg = pydot.Subgraph(graph_name='cluster_' + own_name)
+        sg.set_label('')
+        sg.set_bgcolor('palegreen1')
+
         props = self._properties
         # Include output locations when they define algorithm behaviour
         if self._outputs_define_identity:
             output_props = {k: v.location for k, v in self._outputs.items()}
             props = dict(props.items() + output_props.items())
         props_str = '<BR/>'.join(
-            html_escape('{} = {}'.format(k, v)) for k, v in props.items())
-        label = ('<{}<BR/><FONT POINT-SIZE="8">{}</FONT>>'.format(
-            own_name, props_str or 'defaults-only'))
-        g.attr(label='', fillcolor='palegreen1', style='filled')
-        g.node(own_name, label=label, shape='plaintext')
+            html_escape(format_prop(k, v)) for k, v in props.items())
+        label = ('<<B>{}</B><BR/>{}>'.format(own_name, props_str
+                                             or 'defaults-only'))
+        gnode = pydot.Node(own_name, label=label, shape='plaintext')
+        sg.add_node(gnode)
 
         #IO for the inner part
         for name in self.inputs:
             input_id = html_escape('{}_in_{}'.format(self.fullname, name))
-            g.node(
+            node = pydot.Node(
                 input_id,
                 label=html_escape(name),
                 fillcolor='deepskyblue1',
                 style='filled')
-            g.edge(input_id, own_name, style='invis', minlen='0')
+            edge = pydot.Edge(gnode, node, style='invis', minlen='0')
+            sg.add_node(node)
+            sg.add_edge(edge)
         for name in self.outputs:
             output_id = html_escape('{}_out_{}'.format(self.fullname, name))
-            g.node(
+            node = pydot.Node(
                 output_id,
                 label=html_escape(name),
                 fillcolor='coral1',
                 style='filled')
-            g.edge(own_name, output_id, style='invis', minlen='0')
+            edge = pydot.Edge(gnode, node, style='invis', minlen='0')
+            sg.add_node(node)
+            sg.add_edge(edge)
 
         # tool inputs
         def _inputs_from_tool(tool):
@@ -598,44 +616,39 @@ class Algorithm(object):
         for toolname, inputs in tool_inputs.items():
             for name in inputs:
                 input_id = html_escape('{}_in_{}'.format(toolname, name))
-                label = ('<{}<BR/><FONT POINT-SIZE="8">from {}</FONT>>'.format(
-                    name, toolname))
-                g.node(
+                label = ('<<B>{}</B><BR/>from {}>'.format(name, toolname))
+                node = pydot.Node(
                     input_id,
                     label=label,
                     fillcolor='deepskyblue1',
                     style='filled')
-                g.edge(input_id, own_name, style='invis', minlen='0')
+                edge = pydot.Edge(style='invis', minlen='0')
+                sg.add_edge(gnode, node)
+
+        graph.add_subgraph(sg)
 
         # external links #######
-        graph.subgraph(g)
         for key, handle in self.inputs.items():
-            graph.edge(
+            edge = pydot.Edge(
                 html_escape('{}_out_{}'.format(handle.producer.fullname,
                                                handle.key)),
                 html_escape('{}_in_{}'.format(self.fullname, key)))
+            graph.add_edge(edge)
             handle.producer._graph(graph)
 
         for toolname, inputs in tool_inputs.items():
             for name, handle in inputs.items():
-                graph.edge(
+                edge = pydot.Edge(
                     html_escape('{}_out_{}'.format(handle.producer.fullname,
                                                    handle.key)),
                     html_escape('{}_in_{}'.format(toolname, name)))
+                graph.add_edge(edge)
 
-    def plot_dataflow(self, filename):
-        """Save the dataflow defined by this Algorithm as a graph."""
-        graphviz = graphviz_module()
-        if graphviz is None:
-            return
-
-        top = graphviz.Digraph(self.fullname, strict=True)
-        top.attr('node', shape='box')
+    def plot_dataflow(self):
+        """Return a `pydot.Dot` of the dataflow defined by this Algorithm."""
+        top = pydot.Dot(graph_name=self.fullname, strict=True)
+        top.set_node_defaults(shape='box')
         self._graph(top)
-        try:
-            top.view(filename=filename)
-        except OSError:
-            pass
         return top
 
     def __setitem__(self, k, v):

PyConf/python/PyConf/control_flow.py

@@ -9,9 +9,15 @@
 # or submit itself to any jurisdiction.                                       #
 ###############################################################################
 from __future__ import absolute_import, division, print_function
+try:
+    from html import escape as html_escape
+except ImportError:
+    from cgi import escape as html_escape
 
 from enum import Enum
+import pydot
 
+from .components import Algorithm
 from .dataflow import DataHandle
 
 __all__ = [
@@ -70,3 +76,56 @@ class CompositeNode(object):
     def represent(self):
         return (self.name, self.combineLogic.value,
                 [c.fullname for c in self.children], self.forceOrder)
+
+    def _graph(self, graph):
+        own_name = html_escape(self.name)
+        sg = pydot.Subgraph(graph_name='cluster_' + own_name)
+
+        label = ('<<B>{}</B><BR/>{}, {}>'.format(
+            own_name,
+            str(self.combineLogic).replace('NodeLogic.', ''),
+            'ordered' if self.forceOrder else 'unordered'))
+        sg.set_label(label)
+        sg.set_edge_defaults(dir='forward' if self.forceOrder else 'none')
+
+        prev_node = None
+        for child in self.children:
+            if isinstance(child, Algorithm):
+                # Must name nodes uniquely within a node, otherwise they will
+                # only be drawn one (which makes sense for the *data* flow!)
+                node = pydot.Node(
+                    html_escape('{}_{}'.format(self.name, child.fullname)),
+                    label=child.fullname)
+                sg.add_node(node)
+            else:
+                node = child._graph(sg)
+
+            if prev_node is not None:
+                # When drawing edges to/from subgraphs, the target node must be
+                # a node inside the subgraph, which we take as the first.
+                # However we want the arrow to start/from from the edge of the
+                # subgraph, so must set the ltail/lhead attribute appropriately
+                if isinstance(prev_node, pydot.Subgraph):
+                    tail_node = prev_node.get_nodes()[0]
+                    ltail = prev_node.get_name()
+                else:
+                    tail_node = prev_node
+                    ltail = None
+                if isinstance(node, pydot.Subgraph):
+                    head_node = node.get_nodes()[0]
+                    lhead = node.get_name()
+                else:
+                    head_node = node
+                    lhead = None
+                edge = pydot.Edge(tail_node, head_node)
+                if ltail is not None:
+                    edge.set_ltail(ltail)
+                if lhead is not None:
+                    edge.set_lhead(lhead)
+                sg.add_edge(edge)
+
+            prev_node = node
+
+        graph.add_subgraph(sg)
+
+        return sg

PyConf/python/PyConf/environment.py

@@ -9,9 +9,11 @@
 # or submit itself to any jurisdiction.                                       #
 ###############################################################################
 from __future__ import absolute_import, division, print_function
-
+import datetime
 import logging
 
+import pydot
+
 from Configurables import (
     CallgrindProfile,
     DeterministicPrescaler,
@@ -30,7 +32,6 @@ from . import ConfigurationError
 from .components import Algorithm, setup_component, is_algorithm, is_tool
 from .control_flow import CompositeNode, NodeLogic
 from .dataflow import dataflow_config
-from .utilities import graphviz_module
 
 __all__ = [
     'EverythingHandler',
@@ -190,6 +191,14 @@ def setupOutput(filename, filetype, TESName='HiveWhiteBoard'):
     return writer
 
 
+def _gaudi_datetime_format(dt):
+    """Return the datetime object formatted as Gaudi does it.
+
+    As seen in the application "Welcome" banner.
+    """
+    return dt.strftime('%a %h %d %H:%M:%S %Y')
+
+
 class PythonLoggingConf(ConfigurableUser):
     """Takes care of configuring the python logging verbosity."""
     # Make sure we're applied before anything else by listing
@@ -220,7 +229,7 @@ class EverythingHandler(object):
       set of trigger lines)
     - Translates the data and control flow configuration into job options
       via `configure`
-    - can plot your data flow, if you have graphviz installed
+    - can plot your data flow
 
     The current implementation is quite specific to the needs of an HLT-like
     application.
@@ -356,6 +365,75 @@ class EverythingHandler(object):
         self._lines.append(line)
         self.addNode(line)
 
+    def plot_data_flow(self, filename='data_flow', extensions=('gv', )):
+        """Save a visualisation of the current data flow.
+
+        Parameters
+        ----------
+        filename : str
+            Basename of the file to create.
+        extensions : list
+            List of file extensions to create. One file is created per
+            extensions. Possible values include `'gv'` for saving the raw
+            graphviz representation, and `'png'` and `'pdf'` for saving
+            graphics.
+
+            **Note**: The `dot` binary must be present on the system for saving
+            files with graphical extensions. The raw `.gv` format can be
+            convert be hand like::
+
+                dot -Tpdf data_flow.gv > data_flow.pdf
+        """
+        now = _gaudi_datetime_format(datetime.datetime.now())
+        label = 'Data flow generated at {}'.format(now)
+        top = pydot.Dot(graph_name='Data flow', label=label, strict=True)
+        top.set_node_defaults(shape='box')
+        for alg in self._algs:
+            alg._graph(top)
+        for ext in extensions:
+            format = 'raw' if ext == 'gv' else ext
+            top.write('{}.{}'.format(filename, ext), format=format)
+
+    def plot_control_flow(self, filename='control_flow', extensions=('gv', )):
+        """Save a visualisation of the current control flow.
+
+        Parameters
+        ----------
+        filename : str
+            Basename of the file to create.
+        extensions : list
+            List of file extensions to create. One file is created per
+            extensions. Possible values include `'gv'` for saving the raw
+            graphviz representation, and `'png'` and `'pdf'` for saving
+            graphics.
+
+            **Note**: The `dot` binary must be present on the system for saving
+            files with graphical extensions. The raw `.gv` format can be
+            convert be hand like::
+
+                dot -Tpdf data_flow.gv > data_flow.pdf
+        """
+        # Find the node at the top of the control flow
+        # This is the node which is not a child of any other node
+        child_nodes = set(
+            child for node in self._nodes for child in node.children)
+        top_node = None
+        for node in self._nodes:
+            if node not in child_nodes:
+                top_node = node
+        if top_node is None:
+            raise ConfigurationError('Not not find top control flow node')
+
+        now = _gaudi_datetime_format(datetime.datetime.now())
+        label = 'Control flow generated at {}'.format(now)
+        graph = pydot.Dot(
+            graph_name='control_flow', label=label, strict=True, compound=True)
+        top_node._graph(graph)
+
+        for ext in extensions:
+            format = 'raw' if ext == 'gv' else ext
+            graph.write('{}.{}'.format(filename, ext), format=format)
+
     def configure(self):
         """Instantiate all underlying Configurables.
 
@@ -394,16 +472,5 @@ class EverythingHandler(object):
             node.represent() for node in self._nodes
         ]
 
-    def plotDataFlow(self, filename):
-        graphviz = graphviz_module()
-        if graphviz is None:
-            return
-
-        top = graphviz.Digraph('HLT', strict=True)
-        top.attr('node', shape='box')
-        for alg in self._algs:
-            alg._graph(top)
-        try:
-            top.view(filename=filename)
-        except OSError:
-            pass
+        self.plot_data_flow()
+        self.plot_control_flow()

PyConf/python/PyConf/utilities.py

@@ -10,13 +10,3 @@
 ###############################################################################
 class ConfigurationError(Exception):
     pass
-
-
-def graphviz_module():
-    """Return the graphviz module is import'able, else print a warning."""
-    try:
-        import graphviz
-    except ImportError:
-        print('Cannot import graphviz; dataflow graphing disabled')
-        graphviz = None
-    return graphviz