From 9085e163b61deda0c04227054766cea3cf40dc48 Mon Sep 17 00:00:00 2001
From: Christoph Hasse <christoph.hasse@cern.ch>
Date: Thu, 17 Feb 2022 12:54:06 +0100
Subject: [PATCH 1/3] feat(functor): add functor data dependencies to
ExtraInputs property of owning algorithm
---
PyConf/python/PyConf/components.py | 32 ++++++++++++++++++++++++++++++
1 file changed, 32 insertions(+)
diff --git a/PyConf/python/PyConf/components.py b/PyConf/python/PyConf/components.py
index af32d030c33..3109d4a658f 100644
--- a/PyConf/python/PyConf/components.py
+++ b/PyConf/python/PyConf/components.py
@@ -232,6 +232,18 @@ def _ids_from_list(handles):
return tuple(h.id for h in handles)
+def _gather_extra_inputs(io_dict):
+ """Returns a list of extra_inputs contained in io_dict"""
+ d = []
+ for k, v in io_dict.items():
+ if k.endswith(_DATA_DEPENDENCY_KEY_SUFFIX):
+ if isinstance(v, list):
+ d += [vv.location for vv in v]
+ else:
+ d.append(v.location)
+ return d
+
+
def _gather_locations(io_dict):
"""Return the dictionary with all values mapped to their `.location` property.
@@ -822,6 +834,16 @@ class Algorithm(object):
input_dict = _gather_locations(self.inputs)
output_dict = _gather_locations(self.outputs)
+ # get the functor data dependencies to add them to ExtraInputs
+ extra_inputs = _gather_extra_inputs(self.inputs)
+ if extra_inputs:
+ # ExtraInputs is also a property so we need to be careful to not
+ # overwrite it if its there, but only extend it.
+ cfg['ExtraInputs'] = cfg.get('ExtraInputs', []) + extra_inputs
+ # I don't think duplicates would matter but it's easy to declutter
+ # here so let's quickly do it
+ cfg['ExtraInputs'] = list(set(cfg['ExtraInputs']))
+
if self._input_transform:
input_dict = self._input_transform(**input_dict)
@@ -1170,6 +1192,16 @@ class Tool(object):
cfg = config[(self.type, name)] = self.properties.copy()
input_dict = _gather_locations(self.inputs)
tools_dict = _gather_tool_names(self.tools)
+ # get the functor data dependencies to add them to ExtraInputs
+ extra_inputs = _gather_extra_inputs(self.inputs)
+ if extra_inputs:
+ # ExtraInputs is also a property so we need to be careful to not
+ # overwrite it if its there, but only extend it.
+ cfg['ExtraInputs'] = cfg.get('ExtraInputs', []) + extra_inputs
+ # I don't think duplicates would matter but it's easy to declutter
+ # here so let's quickly do it
+ cfg['ExtraInputs'] = list(set(cfg['ExtraInputs']))
+
if conflict := _dict_add(cfg, input_dict):
raise ConfigurationError(f"input properties try to overwrite "
f"already set properties: {conflict}")
--
GitLab
From 34dfaf51f6dd41fff6c654b8dbd5e3609c142821 Mon Sep 17 00:00:00 2001
From: Christoph Hasse <christoph.hasse@cern.ch>
Date: Thu, 3 Mar 2022 16:23:39 +0100
Subject: [PATCH 2/3] add ignore of timing print outs to LHCbExclusions.py
---
GaudiConf/python/GaudiConf/QMTest/LHCbExclusions.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/GaudiConf/python/GaudiConf/QMTest/LHCbExclusions.py b/GaudiConf/python/GaudiConf/QMTest/LHCbExclusions.py
index 699719c5e9b..19f2df2fd97 100755
--- a/GaudiConf/python/GaudiConf/QMTest/LHCbExclusions.py
+++ b/GaudiConf/python/GaudiConf/QMTest/LHCbExclusions.py
@@ -149,7 +149,12 @@ preprocessor = (
r'.*SUCCESS (Exceptions/Errors/Warnings/Infos Statistics :| #WARNINGS =| #ERRORS =|List of booked \dD histograms in directory).*'
) +
# TODO To be removed when GaudiAlgorithm is gone and thus this output line is no more present
- SortGroupOfLines(r'CompareTracksTr.*DEBUG Changing \S* to \S*'))
+ SortGroupOfLines(r'CompareTracksTr.*DEBUG Changing \S* to \S*') +
+ RegexpReplacer( # normalize any timing prints
+ when="INFO",
+ orig=r'\d+\.*\d*(\s+(ns|us|ms|seconds|s|minutes|min)(\s+|\.|$))',
+ repl=r'n\1',
+ ))
try:
from DDDB.Configuration import GIT_CONDDBS
--
GitLab
From 119b3708b8bdfff71311648ee0cc2b9ac6161ab1 Mon Sep 17 00:00:00 2001
From: Rosen Matev <rosen.matev@cern.ch>
Date: Thu, 10 Mar 2022 12:16:04 +0100
Subject: [PATCH 3/3] Prefer "sorted(set(list))" to make output deterministic
---
PyConf/python/PyConf/components.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/PyConf/python/PyConf/components.py b/PyConf/python/PyConf/components.py
index 3109d4a658f..46a4826e4fd 100644
--- a/PyConf/python/PyConf/components.py
+++ b/PyConf/python/PyConf/components.py
@@ -842,7 +842,7 @@ class Algorithm(object):
cfg['ExtraInputs'] = cfg.get('ExtraInputs', []) + extra_inputs
# I don't think duplicates would matter but it's easy to declutter
# here so let's quickly do it
- cfg['ExtraInputs'] = list(set(cfg['ExtraInputs']))
+ cfg['ExtraInputs'] = sorted(set(cfg['ExtraInputs']))
if self._input_transform:
input_dict = self._input_transform(**input_dict)
@@ -1200,7 +1200,7 @@ class Tool(object):
cfg['ExtraInputs'] = cfg.get('ExtraInputs', []) + extra_inputs
# I don't think duplicates would matter but it's easy to declutter
# here so let's quickly do it
- cfg['ExtraInputs'] = list(set(cfg['ExtraInputs']))
+ cfg['ExtraInputs'] = sorted(set(cfg['ExtraInputs']))
if conflict := _dict_add(cfg, input_dict):
raise ConfigurationError(f"input properties try to overwrite "
--
GitLab