Merge branch 'erodrigu-test-for-issue-100' into 'master'

Add a test for issue 100 in DaVinciTests

Closes #100

See merge request !865

DaVinciTests/CMakeLists.txt

@@ -22,6 +22,7 @@ if(BUILD_TESTING AND USE_DD4HEP)
     set_property(
         TEST
             DaVinciTests.mc.test_davinci-issue-97_bkgcat_mc-truth
+            DaVinciTests.mc.test_davinci-issue-100_multiple_bkgcat_mc-truth
         PROPERTY
             DISABLED TRUE
     )

DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.py

+###############################################################################
+# (c) Copyright 2023 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+"""
+rst_title: Test for DaVinci issue 100
+rst_description: Test for DaVinci issue 100 on an incorrect behaviour
+of the `MCTruthAndBkgCat` helper in Analysis's `Phys/DaVinciMCTools` package,
+traced back to a bug in one of the tools internally called.
+The test ensures that several instances of `MCTruthAndBkgCat`
+can be used in the same script to produce several ntuples.
+rst_running: lbexec DaVinciTests.mc.option_davinci-issue-100_multiple_bkgcat_mc-truth:alg_config "$DAVINCITESTSROOT/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml"
+rst_yaml: ../DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml
+"""
+from PyConf.reading import get_particles
+import Functors as F
+from FunTuple import FunctorCollection, FunTuple_Particles as Funtuple
+from FunTuple.functorcollections import SelectionInfo
+from DaVinciMCTools import MCTruthAndBkgCat
+from DaVinci.algorithms import add_filter
+from DaVinci import Options, make_config
+
+jpsi_branches = {
+    "J_psi_1S": "J/psi(1S) -> mu- mu+",
+    "muminus": "J/psi(1S) -> ^mu- mu+",
+    "muplus": "J/psi(1S) -> mu- ^mu+",
+}
+
+hlt2_tag_lines = [
+    "Hlt2TrackEff_DiMuon_VeloMuon_mup_Tag",
+    "Hlt2TrackEff_DiMuon_VeloMuon_mum_Tag",
+    "Hlt2TrackEff_DiMuon_SeedMuon_mup_Tag",
+    "Hlt2TrackEff_DiMuon_SeedMuon_mum_Tag",
+]
+
+hlt2_match_lines = [
+    "Hlt2TrackEff_DiMuon_VeloMuon_mup_Match",
+    "Hlt2TrackEff_DiMuon_VeloMuon_mum_Match",
+    "Hlt2TrackEff_DiMuon_SeedMuon_mup_Match",
+    "Hlt2TrackEff_DiMuon_SeedMuon_mum_Match",
+]
+
+
+def alg_config(options: Options):
+    evt_variables = SelectionInfo(
+        selection_type="Hlt2", trigger_lines=hlt2_tag_lines + hlt2_match_lines)
+
+    Tuples = {}
+    for line in hlt2_tag_lines:
+        jpsi_data = get_particles(f"/Event/HLT2/{line}/Particles")
+        MC_TRUTH = MCTruthAndBkgCat(
+            jpsi_data, name=f"MCTruthAndBkgCat_{line[-16:]}")
+        MCMOTHER_ID = lambda n: F.VALUE_OR(0) @ MC_TRUTH(F.MC_MOTHER(n, F.PARTICLE_ID))
+        MCMOTHER_KEY = lambda n: F.VALUE_OR(-1) @ MC_TRUTH(F.MC_MOTHER(n, F.OBJECT_KEY))
+
+        trueid_bkgcat_info = FunctorCollection({
+            "TRUEID":
+            F.VALUE_OR(0) @ MC_TRUTH(F.PARTICLE_ID),
+            "TRUEKEY":
+            F.VALUE_OR(-1) @ MC_TRUTH(F.OBJECT_KEY),
+            "TRUEPT":
+            MC_TRUTH(F.PT),
+            "TRUEM":
+            MC_TRUTH(F.MASS),
+            "MC_MOTHER_ID":
+            MCMOTHER_ID(1),
+            "MC_MOTHER_KEY":
+            MCMOTHER_KEY(1),
+            "MC_GD_MOTHER_ID":
+            MCMOTHER_ID(2),
+            "MC_GD_MOTHER_KEY":
+            MCMOTHER_KEY(2),
+            "MC_GD_GD_MOTHER_ID":
+            MCMOTHER_ID(3),
+            "MC_GD_GD_MOTHER_KEY":
+            MCMOTHER_KEY(3),
+            "BKGCAT":
+            MC_TRUTH.BkgCat,
+        })
+
+        variables = {"ALL": trueid_bkgcat_info}
+
+        jpsi_tuple = Funtuple(
+            name="Tuple_" + line,
+            tuple_name="DecayTree",
+            fields=jpsi_branches,
+            variables=variables,
+            event_variables=evt_variables,
+            inputs=jpsi_data,
+        )
+
+        decision = line + "Decision"
+        jpsi_filter = add_filter(f"Filter_{line}", f"HLT_PASS('{decision}')")
+
+        Tuples[line] = [jpsi_filter, jpsi_tuple]
+
+    return make_config(options, Tuples)

DaVinciTests/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml

+input_files:
+  - root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp3/tests/test_davinci-issue-100_multiple_bkgcat_mc-truth.dst
+input_manifest_file: root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp3/tests/test_davinci-issue-100_multiple_bkgcat_mc-truth.tck.json
+
+data_type: Upgrade
+input_type: ROOT
+simulation: True
+input_process: Hlt2
+
+conddb_tag: sim-20210617-vc-md100
+dddb_tag: dddb-20210617
+
+evt_max: -1
+input_raw_format: 0.3
+lumi: false

DaVinciTests/tests/qmtest/mc.qms/test_davinci-issue-100_multiple_bkgcat_mc-truth.qmt

+<?xml version="1.0" ?>
+<!--
+###############################################################################
+# (c) Copyright 2023 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+-->
+<!DOCTYPE extension  PUBLIC '-//QM/2.3/Extension//EN'  'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
+<extension class="GaudiTest.GaudiExeTest" kind="test">
+  <argument name="program"><text>lbexec</text></argument>
+  <argument name="args"><set>
+    <text>DaVinciTests.mc.option_davinci-issue-100_multiple_bkgcat_mc-truth:alg_config</text>
+  </set></argument>
+  <argument 
+name="options_yaml_fn"><text>$DAVINCITESTSROOT/python/DaVinciTests/mc/option_davinci-issue-100_multiple_bkgcat_mc-truth.yaml</text></argument>
+  <argument name="extra_options_yaml"><text>
+    ntuple_file: davinci-issue-100_multiple_bkgcat_mc-truth_ntuple.root
+    histo_file: davinci-issue-100_multiple_bkgcat_mc-truth_histos.root
+  </text></argument>
+  <argument name="validator"><text>
+countErrorLines({"FATAL": 0, "ERROR": 0}, stdout=stdout)
+
+import os
+
+filename = "./davinci-issue-100_multiple_bkgcat_mc-truth_ntuple.root"
+
+samples = ["Tuple_Hlt2TrackEff_DiMuon_SeedMuon_mup_Tag",
+           "Tuple_Hlt2TrackEff_DiMuon_SeedMuon_mum_Tag",
+           "Tuple_Hlt2TrackEff_DiMuon_VeloMuon_mup_Tag",
+           "Tuple_Hlt2TrackEff_DiMuon_VeloMuon_mum_Tag"
+          ]
+
+ntuples = dict(zip(samples, [f"{directory}/DecayTree" for directory in samples]))
+
+df_shapes = dict(zip(samples, [(113, 44), (133, 44), (87, 44), (72, 44)]))
+
+l_branches_with_nans =\
+    dict(zip(samples, [["J_psi_1S_TRUEM", "J_psi_1S_TRUEPT", "muplus_TRUEM", "muplus_TRUEPT"]] + 3*[["J_psi_1S_TRUEM", "J_psi_1S_TRUEPT"]]))
+
+for sample in samples:
+    from DaVinciTests.QMTest.check_helpers import get_pandas_dataframe, list_fields_with_nan
+
+    df = get_pandas_dataframe(filename, ntuples[sample])
+
+    # Check ntuple structure
+    if df.empty:
+        causes.append(f"File {filename}: ntuple {ntuples[sample]} does not contain any branches")
+    if df.shape != df_shapes[sample]:
+        causes.append(f"Ntuple {ntuples[sample]} not with expected number of entries and/or branches")
+
+    # Check there are no NaN values in the ntuple except where expected
+    l_test = list_fields_with_nan(filename, ntuples[sample])
+    if sorted(l_test) != sorted(l_branches_with_nans[sample]):
+        causes.append(f"Ntuple {ntuples[sample]}: unexpected list of branches with NaN values")
+
+# ==> Extra checks on
+sample = "Tuple_Hlt2TrackEff_DiMuon_SeedMuon_mup_Tag"
+df = get_pandas_dataframe(filename, ntuples[sample])
+# Checks PIDs are correctly assigned
+if not ( ( (df["J_psi_1S_TRUEID"]==443).value_counts()[False] == 4 )
+         and ( (df["J_psi_1S_MC_MOTHER_ID"].abs()==531).value_counts()[False] == 4 )
+         and ( (df["muplus_TRUEID"].abs()==13).value_counts()[False] == 1 )  # all true mu+ except 1
+         and ( (df["muminus_TRUEID"].abs() == 13).all() )  # all true mu- 
+         and ( (df["muplus_MC_MOTHER_ID"]==443).sum() == 110 )
+         and ( (df["muminus_MC_MOTHER_ID"]==443).sum() == 112 )
+       ):
+    causes.append(f"Ntuple {ntuples[sample]} contains unexpected MC ID values")
+# Check background categories
+if not ( ( (df["J_psi_1S_BKGCAT"]==0).sum() == 98 )
+         and ( (df["muplus_BKGCAT"]==-1).all() )  # all entries are -1 for final-state particles
+         and ( (df["muminus_BKGCAT"]==-1).all() )  # all entries are -1 for final-state particles
+       ):
+    causes.append(f"Ntuple {ntuples[sample]} contains unexpected BKGCAT values")
+
+# ==> Extra checks on
+sample = "Tuple_Hlt2TrackEff_DiMuon_SeedMuon_mum_Tag"
+df = get_pandas_dataframe(filename, ntuples[sample])
+# Checks PIDs are correctly assigned
+if not ( ( (df["J_psi_1S_TRUEID"]==443).value_counts()[False] == 9 )
+         and ( (df["J_psi_1S_MC_MOTHER_ID"].abs()==531).value_counts()[False] == 9 )
+         and ( (df["muplus_TRUEID"].abs()==13).all() )  # all true mu+
+         and ( (df["muminus_TRUEID"].abs() == 13).value_counts()[False] == 6 )
+         and ( (df["muplus_MC_MOTHER_ID"]==443).sum() == 131 )
+         and ( (df["muminus_MC_MOTHER_ID"]==443).sum() == 125 )
+       ):
+    causes.append(f"Ntuple {ntuples[sample]} contains unexpected MC ID values")
+# Check background categories
+if not ( ( (df["J_psi_1S_BKGCAT"]==0).sum() == 120 )
+         and ( (df["muplus_BKGCAT"]==-1).all() )  # all entries are -1 for final-state particles
+         and ( (df["muminus_BKGCAT"]==-1).all() )  # all entries are -1 for final-state particles
+       ):
+    causes.append(f"Ntuple {ntuples[sample]} contains unexpected BKGCAT values")
+
+# ==> Extra checks on
+sample = "Tuple_Hlt2TrackEff_DiMuon_VeloMuon_mup_Tag"
+df = get_pandas_dataframe(filename, ntuples[sample])
+# Checks PIDs are correctly assigned
+if not ( ( (df["J_psi_1S_TRUEID"]==443).value_counts()[False] == 5 )
+         and ( (df["J_psi_1S_MC_MOTHER_ID"].abs()==531).value_counts()[False] == 5 )
+         and ( (df["muplus_TRUEID"].abs()==13).value_counts()[False] == 2 )  # all true mu+ except 2
+         and ( (df["muminus_TRUEID"].abs() == 13).value_counts()[False] == 2 )  # all true mu- except 2
+         and ( (df["muplus_MC_MOTHER_ID"]==443).sum() == 84 )
+         and ( (df["muminus_MC_MOTHER_ID"]==443).sum() == 84 )
+       ):
+    causes.append(f"Ntuple {ntuples[sample]} contains unexpected MC ID values")
+# Check background categories
+if not ( ( (df["J_psi_1S_BKGCAT"]==0).sum() == 79 )
+         and ( (df["muplus_BKGCAT"]==-1).all() )  # all entries are -1 for final-state particles
+         and ( (df["muminus_BKGCAT"]==-1).all() )  # all entries are -1 for final-state particles
+       ):
+    causes.append(f"Ntuple {ntuples[sample]} contains unexpected BKGCAT values")
+
+# ==> Extra checks on
+sample = "Tuple_Hlt2TrackEff_DiMuon_VeloMuon_mum_Tag"
+df = get_pandas_dataframe(filename, ntuples[sample])
+# Checks PIDs are correctly assigned
+if not ( ( (df["J_psi_1S_TRUEID"]==443).value_counts()[False] == 1 )
+         and ( (df["J_psi_1S_MC_MOTHER_ID"].abs()==531).value_counts()[False] == 1 )
+         and ( (df["muplus_TRUEID"].abs()==13).all() )  # all true mu+
+         and ( (df["muminus_TRUEID"].abs() == 13).value_counts()[False] )  # all true mu- except 1
+         and ( (df["muplus_MC_MOTHER_ID"]==443).all() )
+         and ( (df["muminus_MC_MOTHER_ID"]==443).sum() == 71 )
+       ):
+    causes.append(f"Ntuple {ntuples[sample]} contains unexpected MC ID values")
+# Check background categories
+if not ( ( (df["J_psi_1S_BKGCAT"]==0).sum() == 66 )
+         and ( (df["muplus_BKGCAT"]==-1).all().all() )  # all entries are -1 for final-state particles
+         and ( (df["muminus_BKGCAT"]==-1).all().all() )  # all entries are -1 for final-state particles
+       ):
+    causes.append(f"Ntuple {ntuples[sample]} contains unexpected BKGCAT values")
+
+print('Test successfully completed!')
+os.system(f"rm {filename}")
+  </text></argument>
+</extension>