From f26f95c44f8c773bd2db1a6f346527ae4c22bc33 Mon Sep 17 00:00:00 2001
From: Maurizio Martinelli <maurizio.martinelli@cern.ch>
Date: Wed, 1 Feb 2023 12:57:00 +0100
Subject: [PATCH] Automatic documentation of DaVinciExamples and
DaVinciTutorials
---
.../python/DaVinciExamples/debugging.py | 3 +
.../option_davinci_v2_composites.py | 5 +
.../option_davinci_tupling_from_hlt2.py | 8 +
.../DaVinciTutorials/tutorial0_basic_DVjob.py | 3 +
Phys/DaVinci/python/DaVinci/LbExec.py | 6 +-
doc/Makefile | 10 +-
doc/examples/examples_index.rst | 5 -
doc/guide/developing.rst | 26 +++
doc/{tutorials => guide}/running.rst | 2 +-
doc/index.rst | 1 +
doc/make_auto_docs.py | 191 ++++++++++++++++++
doc/tutorials/tutorials_index.rst | 6 -
12 files changed, 247 insertions(+), 19 deletions(-)
delete mode 100644 doc/examples/examples_index.rst
rename doc/{tutorials => guide}/running.rst (99%)
create mode 100644 doc/make_auto_docs.py
delete mode 100644 doc/tutorials/tutorials_index.rst
diff --git a/DaVinciExamples/python/DaVinciExamples/debugging.py b/DaVinciExamples/python/DaVinciExamples/debugging.py
index 8ba031463..3372998d9 100644
--- a/DaVinciExamples/python/DaVinciExamples/debugging.py
+++ b/DaVinciExamples/python/DaVinciExamples/debugging.py
@@ -10,6 +10,9 @@
###############################################################################
"""
Example of a DaVinci job printing decay trees via `PrintDecayTree`.
+
+rst_title: Printing Decay Trees
+rst_desc: This example shows how to print the decay tree of a candidate.
"""
from PyConf.application import configure, configure_input
from PyConf.application import make_odin
diff --git a/DaVinciExamples/python/DaVinciExamples/option_davinci_v2_composites.py b/DaVinciExamples/python/DaVinciExamples/option_davinci_v2_composites.py
index 2d779c64d..ebc3a980c 100644
--- a/DaVinciExamples/python/DaVinciExamples/option_davinci_v2_composites.py
+++ b/DaVinciExamples/python/DaVinciExamples/option_davinci_v2_composites.py
@@ -8,6 +8,11 @@
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
+"""
+Example of a DaVinci job using v2 functor on composite particles.
+
+rst_title: V2 Composites
+"""
from PyConf.Algorithms import ChargedBasicsProducer, UniqueIDGeneratorAlg
from PyConf.Algorithms import ThOrCombiner__2ChargedBasics
from PyConf.Algorithms import ThOrCombiner__CompositesChargedBasics
diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_hlt2.py b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_hlt2.py
index 8312777fc..4827ac0f1 100644
--- a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_hlt2.py
+++ b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_hlt2.py
@@ -10,6 +10,14 @@
###############################################################################
"""
Read an HLT2 file and create an ntuple with the new DaVinci configuration.
+
+rst_title: Tupling from HLT2
+rst_description: This example shows how to read an HLT2 output file and create a n-tuple.
+It defines various functors operating on the head and daughters of the decay channel
+and creates a FunTuple.
+
+rst_running: lb-run DaVinci/vXXrY lbexec DaVinciExamples.tupling.option_davinci_tupling_from_hlt2
+rst_yaml: ../DaVinciExamples/example_data/FEST_November_2021_dst_newPacking.yaml
"""
from PyConf.reading import get_particles, get_pvs
import Functors as F
diff --git a/DaVinciTutorials/python/DaVinciTutorials/tutorial0_basic_DVjob.py b/DaVinciTutorials/python/DaVinciTutorials/tutorial0_basic_DVjob.py
index 2f7792f50..038d39a6e 100644
--- a/DaVinciTutorials/python/DaVinciTutorials/tutorial0_basic_DVjob.py
+++ b/DaVinciTutorials/python/DaVinciTutorials/tutorial0_basic_DVjob.py
@@ -8,6 +8,9 @@
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
+'''
+rst_title: Basic DaVinci Job
+'''
from DaVinci import Options, make_config
from DaVinci.algorithms import add_filter
from PyConf.reading import get_particles
diff --git a/Phys/DaVinci/python/DaVinci/LbExec.py b/Phys/DaVinci/python/DaVinci/LbExec.py
index 0fff237c1..473273db8 100644
--- a/Phys/DaVinci/python/DaVinci/LbExec.py
+++ b/Phys/DaVinci/python/DaVinci/LbExec.py
@@ -32,15 +32,13 @@ class Options(DefaultOptions):
- input_process (str): Input process type, see `GaudiConf.LbExec.options.InputProcessTypes` values.
The optional parameters that need to be set are :
- - stream (str): Stream name. Default is "default"
- Note: for `input_process=Hlt2` the stream must be strictly empty. The default value is overwritten in this case.
+ - stream (str): Stream name. Default is "default" (Note: for `input_process=Hlt2` the stream must be strictly empty. The default value is overwritten in this case.)
- lumi (bool): Flag to store luminosity information. Default is False.
- evt_pre_filters (dict[str,str]): Event pre-filter code. Default is None.
- annsvc_config (str): Path to the configuration file from sprucing or Hlt2. Default is None.
- write_fsr (bool): Flag to write full stream record. Default is True.
- merge_genfsr (bool): Flag to merge the full stream record. Default is False.
- - metainfo_additional_tags: (list): Additional central tags for `PyConf.filecontent_metadata.metainfo_repos`.
- Default is [].
+ - metainfo_additional_tags: (list): Additional central tags for `PyConf.filecontent_metadata.metainfo_repos`. Default is [].
"""
input_process: InputProcessTypes
stream: Optional[str] = 'default'
diff --git a/doc/Makefile b/doc/Makefile
index 17f01049e..d81a7dc4e 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -41,7 +41,7 @@ $(SPHINXBUILD): requirements.txt Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
#html pdf: Makefile graphs functor_docs $(SPHINXBUILD)
-html pdf: Makefile graphs $(SPHINXBUILD)
+html pdf: Makefile graphs examples_and_tutorials_doc $(SPHINXBUILD)
$(RUNENV) $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
#$(RUNENV) $(SPHINXVERSION) build -r mamartin-docs . _build/html
#-M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O);
@@ -58,13 +58,17 @@ linkcheck: Makefile graphs $(SPHINXBUILD)
clean: Makefile $(SPHINXBUILD)
rm -rf graphviz
+ rm -rf autoapi
$(RUNENV) $(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
purge:
- rm -rf $(SPHINXPREFIX) $(BUILDDIR) graphviz
+ rm -rf $(SPHINXPREFIX) $(BUILDDIR) graphviz autoapi
graphs: $(DATA_GRAPH_SOURCES) $(CONTROL_GRAPH_SOURCES)
+examples_and_tutorials_doc:
+ python make_auto_docs.py
+
# # Generate graphs with names based on the options file that creates them
# graphviz/%_data_flow.gv graphviz/%_control_flow.gv: scripts/%.py
# mkdir -p graphviz
@@ -78,4 +82,4 @@ graphs: $(DATA_GRAPH_SOURCES) $(CONTROL_GRAPH_SOURCES)
# python make_functor_docs.py "$(GIT_COMMIT)" "master" > "$@"
#.PHONY: help html pdf clean graphs functor_docs purge
-.PHONY: help html pdf clean graphs purge
+.PHONY: help html pdf clean graphs examples_and_tutorials_doc purge
diff --git a/doc/examples/examples_index.rst b/doc/examples/examples_index.rst
deleted file mode 100644
index 01a91bc76..000000000
--- a/doc/examples/examples_index.rst
+++ /dev/null
@@ -1,5 +0,0 @@
-Examples
-========
-
-.. toctree::
-
diff --git a/doc/guide/developing.rst b/doc/guide/developing.rst
index a6cd5e0d5..ebd074822 100644
--- a/doc/guide/developing.rst
+++ b/doc/guide/developing.rst
@@ -1,6 +1,26 @@
Developing DaVinci
==================
+There are various ways to work on `DaVinci`.
+If you plan to make continous development work, the simplest is probably by building on top of the whole stack using `lb-stack-setup <https://gitlab.cern.ch/rmatev/lb-stack-setup>`_.
+
+.. code-block:: bash
+
+ source /cvmfs/lhcb.cern.ch/lib/LbEnv
+ curl https://gitlab.cern.ch/rmatev/lb-stack-setup/raw/master/setup.py | python3 - stack
+ make DaVinci
+
+
+This will require approximately 7GB of disk, but will guarantee to be up-to-date with all the dependencies.
+
+Alternatively, especially for small changes, a development environment can be built with the commands
+
+.. code-block:: bash
+
+ lb-dev --name DaVinciDev DaVinci/latest
+ cd DaVinciDev
+ git lb-use DaVinci
+
Coding conventions
------------------
@@ -9,6 +29,12 @@ Coding conventions
Tests and continuous integration
--------------------------------
+Whenever a new commit is made to the DaVinci repository, a `CI pipeline <https://docs.gitlab.com/ee/ci/>`_ runs that performs some style and syntax checks:
+
+ - The LHCb copyright statement should be present at the top of every source file.
+ - The LHCb Python formatting rules must be adhered to.
+ - The Python code must have valid syntax and not raise any `flake8 error codes <https://flake8.pycqa.org/en/2.5.5/warnings.html>`_.
+
Debugging
---------
diff --git a/doc/tutorials/running.rst b/doc/guide/running.rst
similarity index 99%
rename from doc/tutorials/running.rst
rename to doc/guide/running.rst
index c711fd3fb..8513f64bd 100644
--- a/doc/tutorials/running.rst
+++ b/doc/guide/running.rst
@@ -50,7 +50,7 @@ Also make a file named ``options.yaml`` containing:
.. code-block:: yaml
-
+
input_files:
- root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp3/tests/hlt2_passthrough_thor_lines.dst
annsvc_config: root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp3/tests/hlt2_passthrough_thor_lines.tck.json
diff --git a/doc/index.rst b/doc/index.rst
index 50aadd4bb..fcbed5fd8 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -28,6 +28,7 @@ and Turbo output.
configuration/davinci_configuration
guide/funtuple
+ guide/running
guide/developing
guide/documentation
diff --git a/doc/make_auto_docs.py b/doc/make_auto_docs.py
new file mode 100644
index 000000000..6ff856f31
--- /dev/null
+++ b/doc/make_auto_docs.py
@@ -0,0 +1,191 @@
+###############################################################################
+# (c) Copyright 2023 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+'''
+This script is used to automatically create the documentation pages for the scripts in
+DaVinciExamples and DaVinciTutorials and show them in the section Examples and Tutorials
+of the DaVinci documentation site.
+
+The script so far creates the documentation pages based on information added on top of
+the original python files.
+The information is formatted in the form of a comment
+
+"""
+<minimal description of the python script>
+
+rst_title: <documentation page title>
+rst_description: <documentation page description>
+rst_running: <script to run the example/tutorial>
+rst_yaml: <address of the yaml file relative to the 'doc' folder in DaVinci>
+"""
+
+Further sections can be added if needed.
+The information above is decoded in the function GetPageInfo of make_auto_docs.py and used in the functions WritePage to create the page.
+The index for DaVinciExamples or DaVinciTutorials and their respective sub-folders is written by the function WriteIndex that is called recursively when a sub-folder is found.
+'''
+import os, warnings
+
+
+def GetPageInfo(script):
+ """A function to extract the information to write the documentation page from the script
+
+ Args:
+ script (str): the file name of the script
+
+ Raises:
+ Warning: invalid documentation keys
+
+ Returns:
+ dict: a dictionary with information to write the documentation page
+ """
+ info = {'title': None, 'description': None, 'running': None, 'yaml': None}
+ f = open(script, 'r')
+ active_comment = False
+ active_key = None
+ line_start_code = 0
+ for l in f:
+ line_start_code += 1
+ if active_comment and ('"""' in l or "'''" in l):
+ break # avoid looping on the whole file, stop search after the first comment
+ if '"""' in l or "'''" in l: active_comment = not active_comment
+ if 'rst_' not in l and active_key is None or l == '\n':
+ continue # ignore lines without sphinx info
+ if 'rst_' in l:
+ active_key = l[l.rfind('rst_') + 4:l.find(':')] # extract the key
+ if active_key not in info.keys():
+ warnings.warn(
+ f'The key {active_key} in {script} is not valid!')
+ info[active_key] = l.split(':')[1].lstrip() # save information
+ else:
+ info[active_key] += l
+ for k in info.keys():
+ if k != 'description' and info[k] is not None:
+ info[k] = info[k].replace('\n', '')
+ for k in ['yaml']:
+ if info[k] is not None and not os.path.exists(info[k]):
+ warnings.warn(f'The {k} path is not valid: {info[k]}')
+ info['line_start_code'] = line_start_code
+ return info
+
+
+def WritePage(script, info, directory='.'):
+ """A function to write the documentation page
+
+ Args:
+ script (str): the file name of the script
+ info (dict): a dictionary with information to write the documentation page
+ """
+ # Title
+ out = info['title'] + '\n'
+ out += '=' * len(info['title']) + '\n\n'
+ # Description
+ if info['description'] is not None:
+ out += info['description'] + '\n\n'
+ # Include script
+ out += f'.. literalinclude:: /{script}\n'
+ out += ' :language: python\n'
+ if info['line_start_code'] != 0:
+ out += f' :lines: {info["line_start_code"]+1}-\n'
+ out += ' :class: toggle\n\n'
+ # How-to run
+ if info['running'] is not None:
+ out += 'To run the example:\n\n'
+ out += '.. code-block:: sh\n\n'
+ out += f' {info["running"]}\n\n'
+ # Yaml Options
+ if info['yaml'] is not None:
+ out += 'For reference, these are the options of this example\n\n'
+ out += f'.. literalinclude:: /{info["yaml"]}\n'
+ out += ' :language: yaml\n\n'
+ outname = f'{directory}/{script[script.rfind("/"):script.rfind(".py")]}.rst'
+ with open(outname, 'w') as f:
+ f.write(out)
+ print(f'Wrote file {outname}')
+ return
+
+
+def WriteIndex(directory, doc_directory, title, description=''):
+ """Write the index page for the relevant scripts folder
+
+ Args:
+ directory (str): the folder where the scripts lay
+ doc_directory (str): the name of the folder in the sphinx documentation
+ title (str): the title of the page
+ description (str, optional): a description of the content of the folder. Defaults to ''.
+ """
+ # Create documents' directory if not existing
+ if not os.path.exists(doc_directory):
+ os.makedirs(doc_directory)
+ # Title
+ out = f'{title}\n'
+ out += '=' * len(title) + '\n\n'
+ # Description
+ if description != '':
+ out += description + '\n\n'
+ # Search for scripts in the directory
+ valid_scripts = []
+ valid_folders = []
+ missingDoc = []
+ for f in os.listdir(directory):
+ if not f.endswith('.py'):
+ if os.path.isdir(f'{directory}/{f}'):
+ if not os.path.exists(f'{doc_directory}/{f}'):
+ os.makedirs(f'{doc_directory}/{f}')
+ missingDoc += WriteIndex(
+ f'{directory}/{f}', f'{doc_directory}/{f}', f.capitalize())
+ valid_folders += [f'{doc_directory}/{f}']
+ continue
+ if f == '__init__.py': continue
+ valid_scripts += [f'{directory}/{f}']
+ # toctree
+ if len(valid_scripts):
+ wroteTocTree = False
+ if len(valid_folders) and not wroteTocTree:
+ out += '.. toctree::\n\n'
+ wroteTocTree = True
+ for vf in valid_folders:
+ # remove the first folder of the path since the file lives there
+ out += f' {vf[vf.find("/")+1:]}/{vf[vf.rfind("/")+1:]}_index\n'
+ for sc in valid_scripts:
+ info = GetPageInfo(sc)
+ if info['title'] is None:
+ missingDoc += [sc]
+ warnings.warn(
+ f'File {sc} is missing the necessary documentation!')
+ continue
+ if not wroteTocTree:
+ out += '.. toctree::\n\n'
+ wroteTocTree = True
+ WritePage(sc, info, doc_directory)
+ out += f' {sc[sc.rfind("/")+1:].replace(".py","")}\n'
+ # Write filename
+ outname = f'{doc_directory}/{doc_directory[doc_directory.rfind("/")+1:]}_index.rst'
+ with open(outname, 'w') as f:
+ f.write(out)
+ print(f'Wrote file {outname}')
+ return missingDoc
+
+
+def main():
+ missingDoc = []
+ missingDoc += WriteIndex('../DaVinciExamples/python/DaVinciExamples',
+ 'examples', 'DaVinci Examples', '')
+ missingDoc += WriteIndex('../DaVinciTutorials/python/DaVinciTutorials/',
+ 'tutorials', 'DaVinci Tutorials', '')
+ # if len(missingDoc):
+ # raise ValueError(
+ # '\033[91mThe following files are missing documentation:\n' +
+ # '\n'.join(missingDoc) + '\033[0m')
+ return
+
+
+#-------------------------------
+if __name__ == "__main__":
+ main()
diff --git a/doc/tutorials/tutorials_index.rst b/doc/tutorials/tutorials_index.rst
deleted file mode 100644
index e11337e67..000000000
--- a/doc/tutorials/tutorials_index.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-Tutorials
-=========
-
-.. toctree::
-
- running
\ No newline at end of file
--
GitLab