Commit c6e3be02 authored by Alex Pearce's avatar Alex Pearce
Browse files

Always create a raw graphviz file of the data flow.

Uses pydot to create the graph, as graphviz is not available in the LCG
distribution.

Closes #18, and closes #48 by extension as the HLT1/2 tests will run the
graphing methods.
parent 118fc9b0
......@@ -31,4 +31,3 @@ with FTRawBankDecoder.bind(DecodingVersion=ftdec_v), \
setupInputFromTestFileDB('MiniBrunel_2018_MinBias_FTv4_DIGI')
env.configure()
# env.plotDataFlow()
......@@ -70,4 +70,3 @@ with FTRawBankDecoder.bind(DecodingVersion=ftdec_v), \
env.register_public_tool(stateProvider_with_simplified_geom())
env.configure()
# env.plotDataFlow()
......@@ -64,4 +64,3 @@ line1 = CompositeNode("line1", children=[float1, float2])
e = EverythingHandler(threadPoolSize=4, nEventSlots=5, evtMax=50)
e.addNode(line1)
e.configure()
e.plotDataFlow("example_dataflow.gv")
......@@ -28,11 +28,12 @@ import inspect
import re
import json
import pydot
from GaudiKernel.ConfigurableMeta import ConfigurableMeta
from . import ConfigurationError, configurable
from .dataflow import DataHandle, configurable_outputs, configurable_inputs, dataflow_config, contains_datahandle
from .utilities import graphviz_module
__all__ = [
'Algorithm',
......@@ -543,17 +544,21 @@ class Algorithm(object):
return config
def _graph(self, graph):
"""Fill the graphviz.Digraph argument with our dataflow."""
graphviz = graphviz_module()
if graphviz is None:
return
"""Add our dataflow as a `pydot.Subgraph` to graph.
Parameters
----------
graph -- pydot.Graph
"""
# TODO deduplicate graphing?
# in all diamond structures, the subgraph is built multiple times
#inner part ########
own_name = html_escape(self.fullname)
g = graphviz.Digraph(name='cluster_' + own_name)
sg = pydot.Subgraph(graph_name='cluster_' + own_name)
sg.set_label('')
sg.set_bgcolor('palegreen1')
props = self._properties
# Include output locations when they define algorithm behaviour
if self._outputs_define_identity:
......@@ -563,26 +568,30 @@ class Algorithm(object):
html_escape('{} = {}'.format(k, v)) for k, v in props.items())
label = ('<{}<BR/><FONT POINT-SIZE="8">{}</FONT>>'.format(
own_name, props_str or 'defaults-only'))
g.attr(label='', fillcolor='palegreen1', style='filled')
g.node(own_name, label=label, shape='plaintext')
gnode = pydot.Node(own_name, label=label, shape='plaintext')
sg.add_node(gnode)
#IO for the inner part
for name in self.inputs:
input_id = html_escape('{}_in_{}'.format(self.fullname, name))
g.node(
node = pydot.Node(
input_id,
label=html_escape(name),
fillcolor='deepskyblue1',
style='filled')
g.edge(input_id, own_name, style='invis', minlen='0')
edge = pydot.Edge(gnode, node, style='invis', minlen='0')
sg.add_node(node)
sg.add_edge(edge)
for name in self.outputs:
output_id = html_escape('{}_out_{}'.format(self.fullname, name))
g.node(
node = pydot.Node(
output_id,
label=html_escape(name),
fillcolor='coral1',
style='filled')
g.edge(own_name, output_id, style='invis', minlen='0')
edge = pydot.Edge(gnode, node, style='invis', minlen='0')
sg.add_node(node)
sg.add_edge(edge)
# tool inputs
def _inputs_from_tool(tool):
......@@ -600,42 +609,36 @@ class Algorithm(object):
input_id = html_escape('{}_in_{}'.format(toolname, name))
label = ('<{}<BR/><FONT POINT-SIZE="8">from {}</FONT>>'.format(
name, toolname))
g.node(
node = pydot.Node(
input_id,
label=label,
fillcolor='deepskyblue1',
style='filled')
g.edge(input_id, own_name, style='invis', minlen='0')
edge = pydot.Edge(style='invis', minlen='0')
sg.add_edge(gnode, node)
graph.add_subgraph(sg)
# external links #######
graph.subgraph(g)
for key, handle in self.inputs.items():
graph.edge(
html_escape('{}_out_{}'.format(handle.producer.fullname,
handle.key)),
html_escape('{}_in_{}'.format(self.fullname, key)))
edge = pydot.Edge(html_escape('{}_out_{}'.format(handle.producer.fullname, handle.key)),
html_escape('{}_in_{}'.format(self.fullname, key)))
graph.add_edge(edge)
handle.producer._graph(graph)
for toolname, inputs in tool_inputs.items():
for name, handle in inputs.items():
graph.edge(
edge = pydot.Edge(
html_escape('{}_out_{}'.format(handle.producer.fullname,
handle.key)),
html_escape('{}_in_{}'.format(toolname, name)))
graph.add_edge(edge)
def plot_dataflow(self, filename):
"""Save the dataflow defined by this Algorithm as a graph."""
graphviz = graphviz_module()
if graphviz is None:
return
top = graphviz.Digraph(self.fullname, strict=True)
top.attr('node', shape='box')
def plot_dataflow(self):
"""Return a `pydot.Dot` of the dataflow defined by this Algorithm."""
top = pydot.Dot(graph_name=self.fullname, strict=True)
top.set_node_defaults(shape='box')
self._graph(top)
try:
top.view(filename=filename)
except OSError:
pass
return top
def __setitem__(self, k, v):
......
......@@ -12,6 +12,8 @@ from __future__ import absolute_import, division, print_function
import logging
import pydot
from Configurables import (
CallgrindProfile,
DeterministicPrescaler,
......@@ -30,7 +32,6 @@ from . import ConfigurationError
from .components import Algorithm, setup_component, is_algorithm, is_tool
from .control_flow import CompositeNode, NodeLogic
from .dataflow import dataflow_config
from .utilities import graphviz_module
__all__ = [
'EverythingHandler',
......@@ -205,7 +206,7 @@ class EverythingHandler(object):
set of trigger lines)
- Translates the data and control flow configuration into job options
via `configure`
- can plot your data flow, if you have graphviz installed
- can plot your data flow
The current implementation is quite specific to the needs of an HLT-like
application.
......@@ -340,6 +341,33 @@ class EverythingHandler(object):
self._lines.append(line)
self.addNode(line)
def plot_data_flow(self, filename='data_flow', extensions=['dot']):
"""Save a visualisation of the current data flow.
Parameters
----------
filename : str
Basename of the file to create.
extensions : list
List of file extensions to create. One file is created per
extensions. Possible values include `'dot'` for saving the raw
graphviz representation, and `'png'` and `'pdf'` for saving
graphics.
**Note**: The `dot` binary must be present on the system for saving
files with graphical extensions. The raw `.dot` format can be
convert be hand like::
dot -Tpdf data_flow.dot > data_flow.pdf
"""
top = pydot.Dot(graph_name='HLT', strict=True)
top.set_node_defaults(shape='box')
for alg in self._algs:
alg._graph(top)
for ext in extensions:
format = 'raw' if ext == 'dot' else ext
top.write('{}.{}'.format(filename, ext), format=format)
def configure(self):
"""Instantiate all underlying Configurables.
......@@ -378,16 +406,4 @@ class EverythingHandler(object):
node.represent() for node in self._nodes
]
def plotDataFlow(self, filename):
graphviz = graphviz_module()
if graphviz is None:
return
top = graphviz.Digraph('HLT', strict=True)
top.attr('node', shape='box')
for alg in self._algs:
alg._graph(top)
try:
top.view(filename=filename)
except OSError:
pass
self.plot_data_flow()
......@@ -10,13 +10,3 @@
###############################################################################
class ConfigurationError(Exception):
pass
def graphviz_module():
"""Return the graphviz module is import'able, else print a warning."""
try:
import graphviz
except ImportError:
print('Cannot import graphviz; dataflow graphing disabled')
graphviz = None
return graphviz
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment