Commit 465ac849 authored by Rosen Matev's avatar Rosen Matev 🌲
Browse files

Merge branch 'apearce-flow-graphs' into 'master'

Generate data and control flow graphs by default

Closes #62, #18, and #48

See merge request lhcb/Moore!225
parents 597d1145 19c26f51
......@@ -31,4 +31,3 @@ with FTRawBankDecoder.bind(DecodingVersion=ftdec_v), \
setupInputFromTestFileDB('MiniBrunel_2018_MinBias_FTv4_DIGI')
env.configure()
# env.plotDataFlow()
......@@ -70,4 +70,3 @@ with FTRawBankDecoder.bind(DecodingVersion=ftdec_v), \
env.register_public_tool(stateProvider_with_simplified_geom())
env.configure()
# env.plotDataFlow()
......@@ -64,4 +64,3 @@ line1 = CompositeNode("line1", children=[float1, float2])
e = EverythingHandler(threadPoolSize=4, nEventSlots=5, evtMax=50)
e.addNode(line1)
e.configure()
e.plotDataFlow("example_dataflow.gv")
......@@ -28,11 +28,12 @@ import inspect
import re
import json
import pydot
from GaudiKernel.ConfigurableMeta import ConfigurableMeta
from . import ConfigurationError, configurable
from .dataflow import DataHandle, configurable_outputs, configurable_inputs, dataflow_config, contains_datahandle
from .utilities import graphviz_module
__all__ = [
'Algorithm',
......@@ -543,46 +544,63 @@ class Algorithm(object):
return config
def _graph(self, graph):
"""Fill the graphviz.Digraph argument with our dataflow."""
graphviz = graphviz_module()
if graphviz is None:
return
"""Add our dataflow as a `pydot.Subgraph` to graph.
Parameters
----------
graph -- pydot.Graph
"""
# TODO deduplicate graphing?
# in all diamond structures, the subgraph is built multiple times
def format_prop(name, value, max_len=100):
assert max_len > 15, 'max_len should be at least 15'
value = str(value)
if len(value) > max_len:
value = (value[:max_len // 2] + '[...]' +
value[max_len // 2 + 5:max_len])
return '{} = {}'.format(name, value)
#inner part ########
own_name = html_escape(self.fullname)
g = graphviz.Digraph(name='cluster_' + own_name)
sg = pydot.Subgraph(graph_name='cluster_' + own_name)
sg.set_label('')
sg.set_bgcolor('palegreen1')
props = self._properties
# Include output locations when they define algorithm behaviour
if self._outputs_define_identity:
output_props = {k: v.location for k, v in self._outputs.items()}
props = dict(props.items() + output_props.items())
props_str = '<BR/>'.join(
html_escape('{} = {}'.format(k, v)) for k, v in props.items())
label = ('<{}<BR/><FONT POINT-SIZE="8">{}</FONT>>'.format(
own_name, props_str or 'defaults-only'))
g.attr(label='', fillcolor='palegreen1', style='filled')
g.node(own_name, label=label, shape='plaintext')
html_escape(format_prop(k, v)) for k, v in props.items())
label = ('<<B>{}</B><BR/>{}>'.format(own_name, props_str
or 'defaults-only'))
gnode = pydot.Node(own_name, label=label, shape='plaintext')
sg.add_node(gnode)
#IO for the inner part
for name in self.inputs:
input_id = html_escape('{}_in_{}'.format(self.fullname, name))
g.node(
node = pydot.Node(
input_id,
label=html_escape(name),
fillcolor='deepskyblue1',
style='filled')
g.edge(input_id, own_name, style='invis', minlen='0')
edge = pydot.Edge(gnode, node, style='invis', minlen='0')
sg.add_node(node)
sg.add_edge(edge)
for name in self.outputs:
output_id = html_escape('{}_out_{}'.format(self.fullname, name))
g.node(
node = pydot.Node(
output_id,
label=html_escape(name),
fillcolor='coral1',
style='filled')
g.edge(own_name, output_id, style='invis', minlen='0')
edge = pydot.Edge(gnode, node, style='invis', minlen='0')
sg.add_node(node)
sg.add_edge(edge)
# tool inputs
def _inputs_from_tool(tool):
......@@ -598,44 +616,39 @@ class Algorithm(object):
for toolname, inputs in tool_inputs.items():
for name in inputs:
input_id = html_escape('{}_in_{}'.format(toolname, name))
label = ('<{}<BR/><FONT POINT-SIZE="8">from {}</FONT>>'.format(
name, toolname))
g.node(
label = ('<<B>{}</B><BR/>from {}>'.format(name, toolname))
node = pydot.Node(
input_id,
label=label,
fillcolor='deepskyblue1',
style='filled')
g.edge(input_id, own_name, style='invis', minlen='0')
edge = pydot.Edge(style='invis', minlen='0')
sg.add_edge(gnode, node)
graph.add_subgraph(sg)
# external links #######
graph.subgraph(g)
for key, handle in self.inputs.items():
graph.edge(
edge = pydot.Edge(
html_escape('{}_out_{}'.format(handle.producer.fullname,
handle.key)),
html_escape('{}_in_{}'.format(self.fullname, key)))
graph.add_edge(edge)
handle.producer._graph(graph)
for toolname, inputs in tool_inputs.items():
for name, handle in inputs.items():
graph.edge(
edge = pydot.Edge(
html_escape('{}_out_{}'.format(handle.producer.fullname,
handle.key)),
html_escape('{}_in_{}'.format(toolname, name)))
graph.add_edge(edge)
def plot_dataflow(self, filename):
"""Save the dataflow defined by this Algorithm as a graph."""
graphviz = graphviz_module()
if graphviz is None:
return
top = graphviz.Digraph(self.fullname, strict=True)
top.attr('node', shape='box')
def plot_dataflow(self):
"""Return a `pydot.Dot` of the dataflow defined by this Algorithm."""
top = pydot.Dot(graph_name=self.fullname, strict=True)
top.set_node_defaults(shape='box')
self._graph(top)
try:
top.view(filename=filename)
except OSError:
pass
return top
def __setitem__(self, k, v):
......
......@@ -9,9 +9,15 @@
# or submit itself to any jurisdiction. #
###############################################################################
from __future__ import absolute_import, division, print_function
try:
from html import escape as html_escape
except ImportError:
from cgi import escape as html_escape
from enum import Enum
import pydot
from .components import Algorithm
from .dataflow import DataHandle
__all__ = [
......@@ -70,3 +76,56 @@ class CompositeNode(object):
def represent(self):
return (self.name, self.combineLogic.value,
[c.fullname for c in self.children], self.forceOrder)
def _graph(self, graph):
own_name = html_escape(self.name)
sg = pydot.Subgraph(graph_name='cluster_' + own_name)
label = ('<<B>{}</B><BR/>{}, {}>'.format(
own_name,
str(self.combineLogic).replace('NodeLogic.', ''),
'ordered' if self.forceOrder else 'unordered'))
sg.set_label(label)
sg.set_edge_defaults(dir='forward' if self.forceOrder else 'none')
prev_node = None
for child in self.children:
if isinstance(child, Algorithm):
# Must name nodes uniquely within a node, otherwise they will
# only be drawn one (which makes sense for the *data* flow!)
node = pydot.Node(
html_escape('{}_{}'.format(self.name, child.fullname)),
label=child.fullname)
sg.add_node(node)
else:
node = child._graph(sg)
if prev_node is not None:
# When drawing edges to/from subgraphs, the target node must be
# a node inside the subgraph, which we take as the first.
# However we want the arrow to start/from from the edge of the
# subgraph, so must set the ltail/lhead attribute appropriately
if isinstance(prev_node, pydot.Subgraph):
tail_node = prev_node.get_nodes()[0]
ltail = prev_node.get_name()
else:
tail_node = prev_node
ltail = None
if isinstance(node, pydot.Subgraph):
head_node = node.get_nodes()[0]
lhead = node.get_name()
else:
head_node = node
lhead = None
edge = pydot.Edge(tail_node, head_node)
if ltail is not None:
edge.set_ltail(ltail)
if lhead is not None:
edge.set_lhead(lhead)
sg.add_edge(edge)
prev_node = node
graph.add_subgraph(sg)
return sg
......@@ -9,9 +9,11 @@
# or submit itself to any jurisdiction. #
###############################################################################
from __future__ import absolute_import, division, print_function
import datetime
import logging
import pydot
from Configurables import (
CallgrindProfile,
DeterministicPrescaler,
......@@ -30,7 +32,6 @@ from . import ConfigurationError
from .components import Algorithm, setup_component, is_algorithm, is_tool
from .control_flow import CompositeNode, NodeLogic
from .dataflow import dataflow_config
from .utilities import graphviz_module
__all__ = [
'EverythingHandler',
......@@ -190,6 +191,14 @@ def setupOutput(filename, filetype, TESName='HiveWhiteBoard'):
return writer
def _gaudi_datetime_format(dt):
"""Return the datetime object formatted as Gaudi does it.
As seen in the application "Welcome" banner.
"""
return dt.strftime('%a %h %d %H:%M:%S %Y')
class PythonLoggingConf(ConfigurableUser):
"""Takes care of configuring the python logging verbosity."""
# Make sure we're applied before anything else by listing
......@@ -220,7 +229,7 @@ class EverythingHandler(object):
set of trigger lines)
- Translates the data and control flow configuration into job options
via `configure`
- can plot your data flow, if you have graphviz installed
- can plot your data flow
The current implementation is quite specific to the needs of an HLT-like
application.
......@@ -356,6 +365,75 @@ class EverythingHandler(object):
self._lines.append(line)
self.addNode(line)
def plot_data_flow(self, filename='data_flow', extensions=('gv', )):
"""Save a visualisation of the current data flow.
Parameters
----------
filename : str
Basename of the file to create.
extensions : list
List of file extensions to create. One file is created per
extensions. Possible values include `'gv'` for saving the raw
graphviz representation, and `'png'` and `'pdf'` for saving
graphics.
**Note**: The `dot` binary must be present on the system for saving
files with graphical extensions. The raw `.gv` format can be
convert be hand like::
dot -Tpdf data_flow.gv > data_flow.pdf
"""
now = _gaudi_datetime_format(datetime.datetime.now())
label = 'Data flow generated at {}'.format(now)
top = pydot.Dot(graph_name='Data flow', label=label, strict=True)
top.set_node_defaults(shape='box')
for alg in self._algs:
alg._graph(top)
for ext in extensions:
format = 'raw' if ext == 'gv' else ext
top.write('{}.{}'.format(filename, ext), format=format)
def plot_control_flow(self, filename='control_flow', extensions=('gv', )):
"""Save a visualisation of the current control flow.
Parameters
----------
filename : str
Basename of the file to create.
extensions : list
List of file extensions to create. One file is created per
extensions. Possible values include `'gv'` for saving the raw
graphviz representation, and `'png'` and `'pdf'` for saving
graphics.
**Note**: The `dot` binary must be present on the system for saving
files with graphical extensions. The raw `.gv` format can be
convert be hand like::
dot -Tpdf data_flow.gv > data_flow.pdf
"""
# Find the node at the top of the control flow
# This is the node which is not a child of any other node
child_nodes = set(
child for node in self._nodes for child in node.children)
top_node = None
for node in self._nodes:
if node not in child_nodes:
top_node = node
if top_node is None:
raise ConfigurationError('Not not find top control flow node')
now = _gaudi_datetime_format(datetime.datetime.now())
label = 'Control flow generated at {}'.format(now)
graph = pydot.Dot(
graph_name='control_flow', label=label, strict=True, compound=True)
top_node._graph(graph)
for ext in extensions:
format = 'raw' if ext == 'gv' else ext
graph.write('{}.{}'.format(filename, ext), format=format)
def configure(self):
"""Instantiate all underlying Configurables.
......@@ -394,16 +472,5 @@ class EverythingHandler(object):
node.represent() for node in self._nodes
]
def plotDataFlow(self, filename):
graphviz = graphviz_module()
if graphviz is None:
return
top = graphviz.Digraph('HLT', strict=True)
top.attr('node', shape='box')
for alg in self._algs:
alg._graph(top)
try:
top.view(filename=filename)
except OSError:
pass
self.plot_data_flow()
self.plot_control_flow()
......@@ -10,13 +10,3 @@
###############################################################################
class ConfigurationError(Exception):
pass
def graphviz_module():
"""Return the graphviz module is import'able, else print a warning."""
try:
import graphviz
except ImportError:
print('Cannot import graphviz; dataflow graphing disabled')
graphviz = None
return graphviz
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment