From 2779ca966f638fe28d5e030d587cbe66040659d2 Mon Sep 17 00:00:00 2001
From: erodrigu <eduardo.rodrigues@cern.ch>
Date: Mon, 13 Mar 2023 12:28:43 +0100
Subject: [PATCH 01/13] Add options to test of DaVinci issue 100
---
...inci-issue-100_multiple_bkgcat_mc-truth.py | 95 +++++++++++++++++++
...ci-issue-100_multiple_bkgcat_mc-truth.yaml | 17 ++++
2 files changed, 112 insertions(+)
create mode 100644 DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
create mode 100644 DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml
diff --git a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
new file mode 100644
index 000000000..436bdbd40
--- /dev/null
+++ b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
@@ -0,0 +1,95 @@
+###############################################################################
+# (c) Copyright 2023 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+"""
+rst_title: Test for DaVinci issue 100
+rst_description: Test for DaVinci issue 100 on an incorrect behaviour
+of the `MCTruthAndBkgCat` helper in Analysis's `Phys/DaVinciMCTools` package,
+traced back to a bug in one of the tools internally called.
+The test ensures that several instances of `MCTruthAndBkgCat`
+can be used in the same script to produce several ntuples.
+rst_running: lbexec DaVinciTests.mc.option_davinci-issue-100_multiple_bkgcat_mc-truth:main
+"$DAVINCITESTSROOT/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml"
+rst_yaml: ../DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml
+"""
+from PyConf.reading import get_particles
+import Functors as F
+from FunTuple import FunctorCollection, FunTuple_Particles as Funtuple
+from FunTuple.functorcollections import SelectionInfo
+from DaVinciMCTools import MCTruthAndBkgCat
+from DaVinci.algorithms import add_filter
+from DaVinci import Options, make_config
+
+jpsi_branches = {
+ "J_psi_1S": "J/psi(1S) -> mu- mu+",
+ "muminus": "J/psi(1S) -> ^mu- mu+",
+ "muplus": "J/psi(1S) -> mu- ^mu+",
+}
+
+hlt2_tag_lines = [
+ "Hlt2TrackEff_DiMuon_VeloMuon_mup_Tag",
+ "Hlt2TrackEff_DiMuon_VeloMuon_mum_Tag",
+ "Hlt2TrackEff_DiMuon_SeedMuon_mup_Tag",
+ "Hlt2TrackEff_DiMuon_SeedMuon_mum_Tag",
+]
+
+hlt2_match_lines = [
+ "Hlt2TrackEff_DiMuon_VeloMuon_mup_Match",
+ "Hlt2TrackEff_DiMuon_VeloMuon_mum_Match",
+ "Hlt2TrackEff_DiMuon_SeedMuon_mup_Match",
+ "Hlt2TrackEff_DiMuon_SeedMuon_mum_Match",
+]
+
+
+def alg_config(options: Options):
+ evt_variables = SelectionInfo(
+ selection_type="Hlt2", trigger_lines=hlt2_tag_lines + hlt2_match_lines
+ )
+
+ Tuples = {}
+ for line in hlt2_tag_lines:
+ jpsi_data = get_particles(f"/Event/HLT2/{line}/Particles")
+ MC_TRUTH = MCTruthAndBkgCat(jpsi_data)
+ MCMOTHER_ID = lambda n: F.VALUE_OR(0) @ MC_TRUTH(F.MC_MOTHER(n, F.PARTICLE_ID))
+ MCMOTHER_KEY = lambda n: F.VALUE_OR(-1) @ MC_TRUTH(F.MC_MOTHER(n, F.OBJECT_KEY))
+
+ trueid_bkgcat_info = FunctorCollection(
+ {
+ "TRUEID": F.VALUE_OR(0) @ MC_TRUTH(F.PARTICLE_ID),
+ "TRUEKEY": F.VALUE_OR(-1) @ MC_TRUTH(F.OBJECT_KEY),
+ "TRUEPT": MC_TRUTH(F.PT),
+ "TRUEM": MC_TRUTH(F.MASS),
+ "MC_MOTHER_ID": MCMOTHER_ID(1),
+ "MC_MOTHER_KEY": MCMOTHER_KEY(1),
+ "MC_GD_MOTHER_ID": MCMOTHER_ID(2),
+ "MC_GD_MOTHER_KEY": MCMOTHER_KEY(2),
+ "MC_GD_GD_MOTHER_ID": MCMOTHER_ID(3),
+ "MC_GD_GD_MOTHER_KEY": MCMOTHER_KEY(3),
+ "BKGCAT": MC_TRUTH.BkgCat,
+ }
+ )
+
+ variables = {"ALL": trueid_bkgcat_info}
+
+ jpsi_tuple = Funtuple(
+ name="Tuple_" + line,
+ tuple_name="DecayTree",
+ fields=jpsi_branches,
+ variables=variables,
+ event_variables=evt_variables,
+ inputs=jpsi_data,
+ )
+
+ decision = line + "Decision"
+ jpsi_filter = add_filter(f"Filter_{line}", f"HLT_PASS('{decision}')")
+
+ Tuples[line] = [jpsi_filter, jpsi_tuple]
+
+ return make_config(options, Tuples)
diff --git a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml
new file mode 100644
index 000000000..35333125f
--- /dev/null
+++ b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml
@@ -0,0 +1,17 @@
+input_files:
+ - root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp3/tests/test_davinci-issue-100_multiple_bkgcat_mc-truth.dst
+input_manifest_file: '/eos/lhcb/wg/dpa/wp3/tests/test_davinci-issue-100_multiple_bkgcat_mc-truth.tck.json'
+
+data_type: 'Upgrade'
+input_type: ROOT
+simulation: True
+input_process: 'Hlt2'
+
+conddb_tag: sim-20210617-vc-md100
+dddb_tag: dddb-20210617
+
+evt_max: -1
+input_raw_format: 0.3
+lumi: false
+
+ntuple_file: davinci-issue-100_multiple_bkgcat_mc-truth_ntuple.root
--
GitLab
From 61f73f21e91308557fca939afdb1c0d838928d2e Mon Sep 17 00:00:00 2001
From: erodrigu <eduardo.rodrigues@cern.ch>
Date: Mon, 13 Mar 2023 12:29:04 +0100
Subject: [PATCH 02/13] Test only for non DD4hep
---
DaVinciTests/CMakeLists.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/DaVinciTests/CMakeLists.txt b/DaVinciTests/CMakeLists.txt
index ee4df4c9f..17aa76d6a 100644
--- a/DaVinciTests/CMakeLists.txt
+++ b/DaVinciTests/CMakeLists.txt
@@ -22,6 +22,7 @@ if(BUILD_TESTING AND USE_DD4HEP)
set_property(
TEST
DaVinciTests.mc.test_davinci-issue-97_bkgcat_mc-truth
+ DaVinciTests.mc.test_davinci-issue-100_multiple_bkgcat_mc-truth
PROPERTY
DISABLED TRUE
)
--
GitLab
From c8f1a536dfa73f427051b03b28641cf7f7ee0658 Mon Sep 17 00:00:00 2001
From: erodrigu <eduardo.rodrigues@cern.ch>
Date: Mon, 13 Mar 2023 12:31:15 +0100
Subject: [PATCH 03/13] Add basics of QMTest
---
...nci-issue-100_multiple_bkgcat_mc-truth.qmt | 36 +++++++++++++++++++
1 file changed, 36 insertions(+)
create mode 100644 DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt
diff --git a/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt b/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt
new file mode 100644
index 000000000..39118e79e
--- /dev/null
+++ b/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt
@@ -0,0 +1,36 @@
+<?xml version="1.0" ?>
+<!--
+###############################################################################
+# (c) Copyright 2023 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+-->
+<!DOCTYPE extension PUBLIC '-//QM/2.3/Extension//EN' 'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
+<extension class="GaudiTest.GaudiExeTest" kind="test">
+ <argument name="program"><text>lbexec</text></argument>
+ <argument name="args"><set>
+ <text>DaVinciTests.mc.option_davinci-issue-100_multiple_bkgcat_mc-truth:main</text>
+ </set></argument>
+ <argument
+name="options_yaml_fn"><text>$DAVINCITESTSROOT/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml</text></argument>
+ <argument name="extra_options_yaml"><text>
+ ntuple_file: davinci-issue-100_multiple_bkgcat_mc-truth_ntuple.root
+ histo_file: davinci-issue-100_multiple_bkgcat_mc-truth_histos.root
+ evt_max: 10
+ </text></argument>
+ <argument name="reference"><text>../refs/test_davinci-issue-100_multiple_bkgcat_mc-truth.ref</text></argument>
+ <argument name="error_reference"><text>../refs/empty.ref</text></argument>
+ <argument name="validator"><text>
+from DaVinciTests.QMTest.DaVinciExclusions import preprocessor, counter_preprocessor
+validateWithReference(preproc = preprocessor, counter_preproc = counter_preprocessor)
+
+countErrorLines({"FATAL": 0, "ERROR": 0}, stdout=stdout)
+
+ </text></argument>
+</extension>
--
GitLab
From a1f7f6cece3493ef1d35ac1f62bac0d46d6cabf4 Mon Sep 17 00:00:00 2001
From: Gitlab CI <noreply@cern.ch>
Date: Mon, 13 Mar 2023 11:32:26 +0000
Subject: [PATCH 04/13] Fixed formatting
patch generated by https://gitlab.cern.ch/lhcb/DaVinci/-/jobs/28143051
---
...inci-issue-100_multiple_bkgcat_mc-truth.py | 42 +++++++++++--------
1 file changed, 25 insertions(+), 17 deletions(-)
diff --git a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
index 436bdbd40..d28e82b26 100644
--- a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
+++ b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
@@ -50,8 +50,7 @@ hlt2_match_lines = [
def alg_config(options: Options):
evt_variables = SelectionInfo(
- selection_type="Hlt2", trigger_lines=hlt2_tag_lines + hlt2_match_lines
- )
+ selection_type="Hlt2", trigger_lines=hlt2_tag_lines + hlt2_match_lines)
Tuples = {}
for line in hlt2_tag_lines:
@@ -60,21 +59,30 @@ def alg_config(options: Options):
MCMOTHER_ID = lambda n: F.VALUE_OR(0) @ MC_TRUTH(F.MC_MOTHER(n, F.PARTICLE_ID))
MCMOTHER_KEY = lambda n: F.VALUE_OR(-1) @ MC_TRUTH(F.MC_MOTHER(n, F.OBJECT_KEY))
- trueid_bkgcat_info = FunctorCollection(
- {
- "TRUEID": F.VALUE_OR(0) @ MC_TRUTH(F.PARTICLE_ID),
- "TRUEKEY": F.VALUE_OR(-1) @ MC_TRUTH(F.OBJECT_KEY),
- "TRUEPT": MC_TRUTH(F.PT),
- "TRUEM": MC_TRUTH(F.MASS),
- "MC_MOTHER_ID": MCMOTHER_ID(1),
- "MC_MOTHER_KEY": MCMOTHER_KEY(1),
- "MC_GD_MOTHER_ID": MCMOTHER_ID(2),
- "MC_GD_MOTHER_KEY": MCMOTHER_KEY(2),
- "MC_GD_GD_MOTHER_ID": MCMOTHER_ID(3),
- "MC_GD_GD_MOTHER_KEY": MCMOTHER_KEY(3),
- "BKGCAT": MC_TRUTH.BkgCat,
- }
- )
+ trueid_bkgcat_info = FunctorCollection({
+ "TRUEID":
+ F.VALUE_OR(0) @ MC_TRUTH(F.PARTICLE_ID),
+ "TRUEKEY":
+ F.VALUE_OR(-1) @ MC_TRUTH(F.OBJECT_KEY),
+ "TRUEPT":
+ MC_TRUTH(F.PT),
+ "TRUEM":
+ MC_TRUTH(F.MASS),
+ "MC_MOTHER_ID":
+ MCMOTHER_ID(1),
+ "MC_MOTHER_KEY":
+ MCMOTHER_KEY(1),
+ "MC_GD_MOTHER_ID":
+ MCMOTHER_ID(2),
+ "MC_GD_MOTHER_KEY":
+ MCMOTHER_KEY(2),
+ "MC_GD_GD_MOTHER_ID":
+ MCMOTHER_ID(3),
+ "MC_GD_GD_MOTHER_KEY":
+ MCMOTHER_KEY(3),
+ "BKGCAT":
+ MC_TRUTH.BkgCat,
+ })
variables = {"ALL": trueid_bkgcat_info}
--
GitLab
From 7f9c87bc1c3794d11976a149f2fc7e1e6dcac523 Mon Sep 17 00:00:00 2001
From: erodrigu <eduardo.rodrigues@cern.ch>
Date: Mon, 13 Mar 2023 12:35:26 +0100
Subject: [PATCH 05/13] Add basic ref file
---
...est_davinci-issue-100_multiple_bkgcat_mc-truth.ref.detdesc | 4 ++++
1 file changed, 4 insertions(+)
create mode 100644 DaVinciTests/tests/refs/test_davinci-issue-100_multiple_bkgcat_mc-truth.ref.detdesc
diff --git a/DaVinciTests/tests/refs/test_davinci-issue-100_multiple_bkgcat_mc-truth.ref.detdesc b/DaVinciTests/tests/refs/test_davinci-issue-100_multiple_bkgcat_mc-truth.ref.detdesc
new file mode 100644
index 000000000..06d20fe3d
--- /dev/null
+++ b/DaVinciTests/tests/refs/test_davinci-issue-100_multiple_bkgcat_mc-truth.ref.detdesc
@@ -0,0 +1,4 @@
+ApplicationMgr SUCCESS
+====================================================================================================================================
+====================================================================================================================================
+ApplicationMgr INFO Application Manager Configured successfully
--
GitLab
From 7bf72c2c11cf05344c331fdc2fce90b0d14ca08b Mon Sep 17 00:00:00 2001
From: erodrigu <eduardo.rodrigues@cern.ch>
Date: Mon, 13 Mar 2023 16:40:13 +0100
Subject: [PATCH 06/13] Finalise YAML file
---
...option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml
index 35333125f..75fa6df30 100644
--- a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml
+++ b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml
@@ -1,11 +1,11 @@
input_files:
- root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp3/tests/test_davinci-issue-100_multiple_bkgcat_mc-truth.dst
-input_manifest_file: '/eos/lhcb/wg/dpa/wp3/tests/test_davinci-issue-100_multiple_bkgcat_mc-truth.tck.json'
+input_manifest_file: /eos/lhcb/wg/dpa/wp3/tests/test_davinci-issue-100_multiple_bkgcat_mc-truth.tck.json
-data_type: 'Upgrade'
+data_type: Upgrade
input_type: ROOT
simulation: True
-input_process: 'Hlt2'
+input_process: Hlt2
conddb_tag: sim-20210617-vc-md100
dddb_tag: dddb-20210617
@@ -13,5 +13,3 @@ dddb_tag: dddb-20210617
evt_max: -1
input_raw_format: 0.3
lumi: false
-
-ntuple_file: davinci-issue-100_multiple_bkgcat_mc-truth_ntuple.root
--
GitLab
From b7f3946b01365e55366ef021a2bf46c62c1ca3db Mon Sep 17 00:00:00 2001
From: erodrigu <eduardo.rodrigues@cern.ch>
Date: Mon, 13 Mar 2023 16:40:42 +0100
Subject: [PATCH 07/13] Add ntuple content checks
---
...nci-issue-100_multiple_bkgcat_mc-truth.qmt | 58 ++++++++++++++++++-
1 file changed, 56 insertions(+), 2 deletions(-)
diff --git a/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt b/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt
index 39118e79e..bde017696 100644
--- a/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt
+++ b/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt
@@ -15,14 +15,13 @@
<extension class="GaudiTest.GaudiExeTest" kind="test">
<argument name="program"><text>lbexec</text></argument>
<argument name="args"><set>
- <text>DaVinciTests.mc.option_davinci-issue-100_multiple_bkgcat_mc-truth:main</text>
+ <text>DaVinciTests.mc.option_davinci-issue-100_multiple_bkgcat_mc-truth:alg_config</text>
</set></argument>
<argument
name="options_yaml_fn"><text>$DAVINCITESTSROOT/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml</text></argument>
<argument name="extra_options_yaml"><text>
ntuple_file: davinci-issue-100_multiple_bkgcat_mc-truth_ntuple.root
histo_file: davinci-issue-100_multiple_bkgcat_mc-truth_histos.root
- evt_max: 10
</text></argument>
<argument name="reference"><text>../refs/test_davinci-issue-100_multiple_bkgcat_mc-truth.ref</text></argument>
<argument name="error_reference"><text>../refs/empty.ref</text></argument>
@@ -32,5 +31,60 @@ validateWithReference(preproc = preprocessor, counter_preproc = counter_preproce
countErrorLines({"FATAL": 0, "ERROR": 0}, stdout=stdout)
+import os
+from DaVinciTests.QMTest.check_helpers import get_pandas_dataframe, list_fields_with_nan
+
+filename = "./davinci-issue-100_multiple_bkgcat_mc-truth_ntuple.root"
+
+samples = ["Tuple_Hlt2TrackEff_DiMuon_SeedMuon_mup_Tag",
+ "Tuple_Hlt2TrackEff_DiMuon_SeedMuon_mum_Tag",
+ "Tuple_Hlt2TrackEff_DiMuon_VeloMuon_mup_Tag",
+ "Tuple_Hlt2TrackEff_DiMuon_VeloMuon_mum_Tag"
+ ]
+
+ntuples = dict(zip(samples, [f"{directory}/DecayTree" for directory in samples]))
+
+df_shapes = dict(zip(samples, [(113, 44), (133, 44), (87, 44), (72, 44)]))
+
+l_branches_with_nans =\
+ dict(zip(samples, [["J_psi_1S_TRUEM", "J_psi_1S_TRUEPT", "muplus_TRUEM", "muplus_TRUEPT"]] + 3*[["J_psi_1S_TRUEM", "J_psi_1S_TRUEPT"]]))
+
+def run_basic_checks(sample):
+ df = get_pandas_dataframe(filename, ntuples[sample])
+
+ # Check ntuple structure
+ if df.empty:
+ causes.append(f"File {filename}: ntuple {ntuples[sample]} does not contain any branches")
+ if df.shape != df_shapes[sample]:
+ causes.append(f"Ntuple {ntuples[sample]} not with expected number of entries and/or branches")
+
+ # Check there are no NaN values in the ntuple except where expected
+ l_test = list_fields_with_nan(filename, ntuples[sample])
+ if sorted(l_test) != sorted(l_branches_with_nans[sample]):
+ causes.append(f"Ntuple {ntuples[sample]}: unexpected list of branches with NaN values")
+
+for sample in samples:
+ run_basic_checks(sample)
+
+# ==> Extra checks on sample "Tuple_Hlt2TrackEff_DiMuon_SeedMuon_mup_Tag"
+# Checks PIDs are correctly assigned
+if not ( ( (df["J_psi_1S_TRUEID"]==443).value_counts()[False] == 4 )
+ and ( (df["J_psi_1S_MC_MOTHER_ID"].abs()==531).value_counts()[False] == 4 )
+ and ( (df["muplus_TRUEID"].abs()==13).value_counts()[False] == 1 ) # all true mu+ except 1
+ and ( (df["muminus_TRUEID"].abs() == 13).all().all() ) # all true mu-
+ and ( (df["muplus_MC_MOTHER_ID"]==443).sum() == 110 )
+ and ( (df["muminus_MC_MOTHER_ID"]==443).sum() == 112 )
+ ):
+ causes.append("Ntuple contains unexpected MC ID values")
+
+# Check background categories
+if not ( ( (df["J_psi_1S_BKGCAT"]==0).sum() == 98 )
+ and ( (df["muplus_BKGCAT"]==-1).all().all() ) # all entries are -1 for final-state particles
+ and ( (df["muminus_BKGCAT"]==-1).all().all() ) # all entries are -1 for final-state particles
+ ):
+ causes.append("Ntuple contains unexpected BKGCAT values")
+
+print('Test successfully completed!')
+os.system(f"rm {filename}")
</text></argument>
</extension>
--
GitLab
From 06be0efe17a1a841b84e6b29af1eafa73e999e8c Mon Sep 17 00:00:00 2001
From: erodrigu <eduardo.rodrigues@cern.ch>
Date: Mon, 13 Mar 2023 16:48:53 +0100
Subject: [PATCH 08/13] Fix typo
---
.../mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
index d28e82b26..a386780fd 100644
--- a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
+++ b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
@@ -15,8 +15,7 @@ of the `MCTruthAndBkgCat` helper in Analysis's `Phys/DaVinciMCTools` package,
traced back to a bug in one of the tools internally called.
The test ensures that several instances of `MCTruthAndBkgCat`
can be used in the same script to produce several ntuples.
-rst_running: lbexec DaVinciTests.mc.option_davinci-issue-100_multiple_bkgcat_mc-truth:main
-"$DAVINCITESTSROOT/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml"
+rst_running: lbexec DaVinciTests.mc.option_davinci-issue-100_multiple_bkgcat_mc-truth:alg_config "$DAVINCITESTSROOT/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml"
rst_yaml: ../DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml
"""
from PyConf.reading import get_particles
--
GitLab
From 793fe695ead421f15de8ee125ec2fb507c99ba2f Mon Sep 17 00:00:00 2001
From: erodrigu <eduardo.rodrigues@cern.ch>
Date: Thu, 30 Mar 2023 18:01:08 +0200
Subject: [PATCH 09/13] Finalise ntuple content checks
---
...nci-issue-100_multiple_bkgcat_mc-truth.qmt | 76 ++++++++++++++++---
1 file changed, 66 insertions(+), 10 deletions(-)
diff --git a/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt b/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt
index bde017696..c3c1b8275 100644
--- a/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt
+++ b/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt
@@ -32,7 +32,6 @@ validateWithReference(preproc = preprocessor, counter_preproc = counter_preproce
countErrorLines({"FATAL": 0, "ERROR": 0}, stdout=stdout)
import os
-from DaVinciTests.QMTest.check_helpers import get_pandas_dataframe, list_fields_with_nan
filename = "./davinci-issue-100_multiple_bkgcat_mc-truth_ntuple.root"
@@ -49,7 +48,9 @@ df_shapes = dict(zip(samples, [(113, 44), (133, 44), (87, 44), (72, 44)]))
l_branches_with_nans =\
dict(zip(samples, [["J_psi_1S_TRUEM", "J_psi_1S_TRUEPT", "muplus_TRUEM", "muplus_TRUEPT"]] + 3*[["J_psi_1S_TRUEM", "J_psi_1S_TRUEPT"]]))
-def run_basic_checks(sample):
+for sample in samples:
+ from DaVinciTests.QMTest.check_helpers import get_pandas_dataframe, list_fields_with_nan
+
df = get_pandas_dataframe(filename, ntuples[sample])
# Check ntuple structure
@@ -63,26 +64,81 @@ def run_basic_checks(sample):
if sorted(l_test) != sorted(l_branches_with_nans[sample]):
causes.append(f"Ntuple {ntuples[sample]}: unexpected list of branches with NaN values")
-for sample in samples:
- run_basic_checks(sample)
-
-# ==> Extra checks on sample "Tuple_Hlt2TrackEff_DiMuon_SeedMuon_mup_Tag"
+# ==> Extra checks on
+sample = "Tuple_Hlt2TrackEff_DiMuon_SeedMuon_mup_Tag"
+df = get_pandas_dataframe(filename, ntuples[sample])
# Checks PIDs are correctly assigned
if not ( ( (df["J_psi_1S_TRUEID"]==443).value_counts()[False] == 4 )
and ( (df["J_psi_1S_MC_MOTHER_ID"].abs()==531).value_counts()[False] == 4 )
and ( (df["muplus_TRUEID"].abs()==13).value_counts()[False] == 1 ) # all true mu+ except 1
- and ( (df["muminus_TRUEID"].abs() == 13).all().all() ) # all true mu-
+ and ( (df["muminus_TRUEID"].abs() == 13).all() ) # all true mu-
and ( (df["muplus_MC_MOTHER_ID"]==443).sum() == 110 )
and ( (df["muminus_MC_MOTHER_ID"]==443).sum() == 112 )
):
- causes.append("Ntuple contains unexpected MC ID values")
-
+ causes.append(f"Ntuple {ntuples[sample]} contains unexpected MC ID values")
# Check background categories
if not ( ( (df["J_psi_1S_BKGCAT"]==0).sum() == 98 )
+ and ( (df["muplus_BKGCAT"]==-1).all() ) # all entries are -1 for final-state particles
+ and ( (df["muminus_BKGCAT"]==-1).all() ) # all entries are -1 for final-state particles
+ ):
+ causes.append(f"Ntuple {ntuples[sample]} contains unexpected BKGCAT values")
+
+# ==> Extra checks on
+sample = "Tuple_Hlt2TrackEff_DiMuon_SeedMuon_mum_Tag"
+df = get_pandas_dataframe(filename, ntuples[sample])
+# Checks PIDs are correctly assigned
+if not ( ( (df["J_psi_1S_TRUEID"]==443).value_counts()[False] == 9 )
+ and ( (df["J_psi_1S_MC_MOTHER_ID"].abs()==531).value_counts()[False] == 9 )
+ and ( (df["muplus_TRUEID"].abs()==13).all() ) # all true mu+
+ and ( (df["muminus_TRUEID"].abs() == 13).value_counts()[False] == 6 )
+ and ( (df["muplus_MC_MOTHER_ID"]==443).sum() == 131 )
+ and ( (df["muminus_MC_MOTHER_ID"]==443).sum() == 125 )
+ ):
+ causes.append(f"Ntuple {ntuples[sample]} contains unexpected MC ID values")
+# Check background categories
+if not ( ( (df["J_psi_1S_BKGCAT"]==0).sum() == 120 )
+ and ( (df["muplus_BKGCAT"]==-1).all() ) # all entries are -1 for final-state particles
+ and ( (df["muminus_BKGCAT"]==-1).all() ) # all entries are -1 for final-state particles
+ ):
+ causes.append(f"Ntuple {ntuples[sample]} contains unexpected BKGCAT values")
+
+# ==> Extra checks on
+sample = "Tuple_Hlt2TrackEff_DiMuon_VeloMuon_mup_Tag"
+df = get_pandas_dataframe(filename, ntuples[sample])
+# Checks PIDs are correctly assigned
+if not ( ( (df["J_psi_1S_TRUEID"]==443).value_counts()[False] == 5 )
+ and ( (df["J_psi_1S_MC_MOTHER_ID"].abs()==531).value_counts()[False] == 5 )
+ and ( (df["muplus_TRUEID"].abs()==13).value_counts()[False] == 2 ) # all true mu+ except 2
+ and ( (df["muminus_TRUEID"].abs() == 13).value_counts()[False] == 2 ) # all true mu- except 2
+ and ( (df["muplus_MC_MOTHER_ID"]==443).sum() == 84 )
+ and ( (df["muminus_MC_MOTHER_ID"]==443).sum() == 84 )
+ ):
+ causes.append(f"Ntuple {ntuples[sample]} contains unexpected MC ID values")
+# Check background categories
+if not ( ( (df["J_psi_1S_BKGCAT"]==0).sum() == 79 )
+ and ( (df["muplus_BKGCAT"]==-1).all() ) # all entries are -1 for final-state particles
+ and ( (df["muminus_BKGCAT"]==-1).all() ) # all entries are -1 for final-state particles
+ ):
+ causes.append(f"Ntuple {ntuples[sample]} contains unexpected BKGCAT values")
+
+# ==> Extra checks on
+sample = "Tuple_Hlt2TrackEff_DiMuon_VeloMuon_mum_Tag"
+df = get_pandas_dataframe(filename, ntuples[sample])
+# Checks PIDs are correctly assigned
+if not ( ( (df["J_psi_1S_TRUEID"]==443).value_counts()[False] == 1 )
+ and ( (df["J_psi_1S_MC_MOTHER_ID"].abs()==531).value_counts()[False] == 1 )
+ and ( (df["muplus_TRUEID"].abs()==13).all() ) # all true mu+
+ and ( (df["muminus_TRUEID"].abs() == 13).value_counts()[False] ) # all true mu- except 1
+ and ( (df["muplus_MC_MOTHER_ID"]==443).all() )
+ and ( (df["muminus_MC_MOTHER_ID"]==443).sum() == 71 )
+ ):
+ causes.append(f"Ntuple {ntuples[sample]} contains unexpected MC ID values")
+# Check background categories
+if not ( ( (df["J_psi_1S_BKGCAT"]==0).sum() == 66 )
and ( (df["muplus_BKGCAT"]==-1).all().all() ) # all entries are -1 for final-state particles
and ( (df["muminus_BKGCAT"]==-1).all().all() ) # all entries are -1 for final-state particles
):
- causes.append("Ntuple contains unexpected BKGCAT values")
+ causes.append(f"Ntuple {ntuples[sample]} contains unexpected BKGCAT values")
print('Test successfully completed!')
os.system(f"rm {filename}")
--
GitLab
From 383af9d80692584700d9be693d88be7b74f63471 Mon Sep 17 00:00:00 2001
From: erodrigu <eduardo.rodrigues@cern.ch>
Date: Thu, 30 Mar 2023 20:29:40 +0200
Subject: [PATCH 10/13] Adapt to changes in instantiation of MCTruthAndBkgCat
---
.../mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
index a386780fd..aa75ca795 100644
--- a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
+++ b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
@@ -54,7 +54,7 @@ def alg_config(options: Options):
Tuples = {}
for line in hlt2_tag_lines:
jpsi_data = get_particles(f"/Event/HLT2/{line}/Particles")
- MC_TRUTH = MCTruthAndBkgCat(jpsi_data)
+ MC_TRUTH = MCTruthAndBkgCat(jpsi_data, name=f"MCTruthAndBkgCat_{line[-16:]}")
MCMOTHER_ID = lambda n: F.VALUE_OR(0) @ MC_TRUTH(F.MC_MOTHER(n, F.PARTICLE_ID))
MCMOTHER_KEY = lambda n: F.VALUE_OR(-1) @ MC_TRUTH(F.MC_MOTHER(n, F.OBJECT_KEY))
--
GitLab
From fc02a87ef1a2d7e9859765fa3c8cf596b6ead540 Mon Sep 17 00:00:00 2001
From: erodrigu <eduardo.rodrigues@cern.ch>
Date: Thu, 30 Mar 2023 20:34:54 +0200
Subject: [PATCH 11/13] Do not use a ref file
---
.../test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt | 5 -----
...st_davinci-issue-100_multiple_bkgcat_mc-truth.ref.detdesc | 4 ----
2 files changed, 9 deletions(-)
delete mode 100644 DaVinciTests/tests/refs/test_davinci-issue-100_multiple_bkgcat_mc-truth.ref.detdesc
diff --git a/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt b/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt
index c3c1b8275..54e1fba3c 100644
--- a/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt
+++ b/DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt
@@ -23,12 +23,7 @@ name="options_yaml_fn"><text>$DAVINCITESTSROOT/python/DaVinciTests/mc/option_dav
ntuple_file: davinci-issue-100_multiple_bkgcat_mc-truth_ntuple.root
histo_file: davinci-issue-100_multiple_bkgcat_mc-truth_histos.root
</text></argument>
- <argument name="reference"><text>../refs/test_davinci-issue-100_multiple_bkgcat_mc-truth.ref</text></argument>
- <argument name="error_reference"><text>../refs/empty.ref</text></argument>
<argument name="validator"><text>
-from DaVinciTests.QMTest.DaVinciExclusions import preprocessor, counter_preprocessor
-validateWithReference(preproc = preprocessor, counter_preproc = counter_preprocessor)
-
countErrorLines({"FATAL": 0, "ERROR": 0}, stdout=stdout)
import os
diff --git a/DaVinciTests/tests/refs/test_davinci-issue-100_multiple_bkgcat_mc-truth.ref.detdesc b/DaVinciTests/tests/refs/test_davinci-issue-100_multiple_bkgcat_mc-truth.ref.detdesc
deleted file mode 100644
index 06d20fe3d..000000000
--- a/DaVinciTests/tests/refs/test_davinci-issue-100_multiple_bkgcat_mc-truth.ref.detdesc
+++ /dev/null
@@ -1,4 +0,0 @@
-ApplicationMgr SUCCESS
-====================================================================================================================================
-====================================================================================================================================
-ApplicationMgr INFO Application Manager Configured successfully
--
GitLab
From 007d6b0299ae4faab236a362707b117fa988a188 Mon Sep 17 00:00:00 2001
From: Gitlab CI <noreply@cern.ch>
Date: Thu, 30 Mar 2023 18:36:03 +0000
Subject: [PATCH 12/13] Fixed formatting
patch generated by https://gitlab.cern.ch/lhcb/DaVinci/-/jobs/28576024
---
.../mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
index aa75ca795..86a6a7497 100644
--- a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
+++ b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py
@@ -54,7 +54,8 @@ def alg_config(options: Options):
Tuples = {}
for line in hlt2_tag_lines:
jpsi_data = get_particles(f"/Event/HLT2/{line}/Particles")
- MC_TRUTH = MCTruthAndBkgCat(jpsi_data, name=f"MCTruthAndBkgCat_{line[-16:]}")
+ MC_TRUTH = MCTruthAndBkgCat(
+ jpsi_data, name=f"MCTruthAndBkgCat_{line[-16:]}")
MCMOTHER_ID = lambda n: F.VALUE_OR(0) @ MC_TRUTH(F.MC_MOTHER(n, F.PARTICLE_ID))
MCMOTHER_KEY = lambda n: F.VALUE_OR(-1) @ MC_TRUTH(F.MC_MOTHER(n, F.OBJECT_KEY))
--
GitLab
From 6be1c150df2069689d10c19971610d84517c43c1 Mon Sep 17 00:00:00 2001
From: erodrigu <eduardo.rodrigues@cern.ch>
Date: Fri, 31 Mar 2023 09:40:41 +0200
Subject: [PATCH 13/13] Add root://eoslhcb.cern.ch/ prefix for
input_manifest_file
---
.../mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml
index 75fa6df30..61dd98e97 100644
--- a/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml
+++ b/DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml
@@ -1,6 +1,6 @@
input_files:
- root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp3/tests/test_davinci-issue-100_multiple_bkgcat_mc-truth.dst
-input_manifest_file: /eos/lhcb/wg/dpa/wp3/tests/test_davinci-issue-100_multiple_bkgcat_mc-truth.tck.json
+input_manifest_file: root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp3/tests/test_davinci-issue-100_multiple_bkgcat_mc-truth.tck.json
data_type: Upgrade
input_type: ROOT
--
GitLab