Merge branch 'master' into qee_upgrade

CMakeLists.txt

@@ -27,6 +27,8 @@ include(MooreAnalysisDependencies)
 lhcb_add_subdirectories(
   HltEfficiencyChecker
   HltIntegrationTests
+  Phys/DecayTreeTupleBase
+  Phys/DecayTreeTuple
 )
 
 

HltEfficiencyChecker/options/hlt1_and_hlt2_eff_example.py

+###############################################################################
+# (c) Copyright 2019-2020 CERN for the benefit of the LHCb Collaboration      #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+from Moore import options
+from HltEfficiencyChecker.config import run_chained_hlt_with_tuples
+from RecoConf.hlt1_allen import sequence as allen_sequence
+from RecoConf.reconstruction_objects import reconstruction
+from Hlt2Conf.lines.topological_b import all_lines
+
+decay = ("[${B_s0}B_s0 => "
+         "( phi(1020) => ${Kplus0}K+ ${Kminus0}K-) "
+         "( phi(1020) => ${Kplus1}K+ ${Kminus1}K-) ]CC")
+
+
+def make_hlt2_lines():
+    return [builder() for builder in all_lines.values()]
+
+
+options.lines_maker = make_hlt2_lines
+options.set_input_from_testfiledb('Upgrade_BsPhiPhi_MD_FTv4_DIGI')
+options.evt_max = 50
+options.set_conds_from_testfiledb('Upgrade_BsPhiPhi_MD_FTv4_DIGI')
+
+options.ntuple_file = "hlt1_and_hlt2_eff_ntuple.root"
+
+# needed to run over FTv2 data
+from RecoConf.hlt1_tracking import default_ft_decoding_version
+default_ft_decoding_version.global_bind(value=4)
+
+from RecoConf.global_tools import stateProvider_with_simplified_geom
+with allen_sequence.bind(sequence="hlt1_pp_default"), reconstruction.bind(
+        from_file=False):
+    run_chained_hlt_with_tuples(
+        options, decay, public_tools=[stateProvider_with_simplified_geom()])

HltEfficiencyChecker/options/hlt1_and_hlt2_eff_example.yaml

+# HltEfficiencyChecker "wizard" example running HLT1 and HLT2
+annotated_decay_descriptor:
+    "[${B_s0}B_s0 => ( phi(1020) => ${Kplus0}K+ ${Kminus0}K-) ( phi(1020) => ${Kplus1}K+ ${Kminus1}K-) ]CC"
+ntuple_path: &NTUPLE hlt1_and_hlt2_eff_ntuple.root
+job:
+    trigger_level: both
+    evt_max: 50
+    testfiledb_key: Upgrade_BsPhiPhi_MD_FTv4_DIGI
+    ft_decoding_version: 4
+    hlt2_lines_from: Hlt2Conf.lines.topological_b
+    allen_sequence: hlt1_pp_default
+    run_reconstruction: true
+    options:
+        - $HLTEFFICIENCYCHECKERROOT/options/options_template.py.jinja  # first rendered with jinja2
+analysis:
+    script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py
+    args:
+        input: *NTUPLE
+        reconstructible_children: Kplus0,Kminus0,Kplus1,Kminus1
+        legend_header: "B^{0}_{s} #rightarrow #phi#phi"
+        make_plots: true
+        vars: "PT,Kplus0:PT"
+        lines: Hlt2Topo2BodyLineDecision
+        TOS_match_to: "B_s0,Kplus0"
+        custom_denoms: "Hlt1trackmvas: Hlt1TrackMVADecision || Hlt1TwoTrackMVADecision"

HltEfficiencyChecker/options/hlt1_and_hlt2_rate_example.py

+###############################################################################
+# (c) Copyright 2019-2020 CERN for the benefit of the LHCb Collaboration      #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+from Moore import options
+from HltEfficiencyChecker.config import run_chained_hlt_with_tuples
+from RecoConf.reconstruction_objects import reconstruction
+from Hlt2Conf.lines.topological_b import all_lines
+
+
+def make_lines():
+    return [builder() for builder in all_lines.values()]
+
+
+options.lines_maker = make_lines
+options.set_input_and_conds_from_testfiledb('upgrade_DC19_01_MinBiasMD')
+options.evt_max = 50
+options.ntuple_file = "hlt1_and_hlt2_rate_ntuple.root"
+
+from RecoConf.hlt1_tracking import default_ft_decoding_version
+default_ft_decoding_version.global_bind(value=6)
+
+# TODO stateProvider_with_simplified_geom must go away from option files
+from RecoConf.global_tools import stateProvider_with_simplified_geom
+with reconstruction.bind(from_file=False):
+    run_chained_hlt_with_tuples(
+        options, public_tools=[stateProvider_with_simplified_geom()])

HltEfficiencyChecker/options/hlt1_and_hlt2_rate_example.yaml

+# HltEfficiencyChecker "wizard" example for Hlt2 given Hlt1 rates
+ntuple_path: &NTUPLE hlt1_and_hlt2_rate_ntuple.root
+job:
+    trigger_level: both
+    evt_max: 50
+    testfiledb_key: upgrade_DC19_01_MinBiasMD
+    hlt2_lines_from: Hlt2Conf.lines.topological_b
+    run_reconstruction: True
+    ft_decoding_version: 6
+    options:
+        - $HLTEFFICIENCYCHECKERROOT/options/options_template.py.jinja  # first rendered with jinja2
+analysis:
+    script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py
+    args:
+        input: *NTUPLE
+        lines: Hlt2Topo2BodyLineDecision
+        filter_lines: AnyHlt1Line
+        json: Hlt2_rates_filtered_on_Hlt1TrackMVALines.json

HltEfficiencyChecker/options/hlt1_eff_default.yaml

@@ -28,7 +28,6 @@ analysis:
     script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py
     args:
         input: *NTUPLE
-        level: Hlt1
         reconstructible_children: Kplus0,Kminus0,Kplus1,Kminus1
         # The parent is automatically deduced from the annotated decay descriptor,
         # which is passed in a file such as eff_ntuple.root.json, but can also be given:

HltEfficiencyChecker/options/hlt1_eff_example_moore.yaml

@@ -28,7 +28,6 @@ analysis:
     script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py
     args:
         input: *NTUPLE
-        level: Hlt1
         reconstructible_children: muplus,muminus,Kplus,Kminus
         # The parent is automatically deduced from the annotated decay descriptor,
         # which is passed in a file such as eff_ntuple.root.json, but can also be given:

HltEfficiencyChecker/options/hlt1_eff_no_gec.yaml

@@ -27,7 +27,6 @@ analysis:
     script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py
     args:
         input: *NTUPLE
-        level: Hlt1
         reconstructible_children: Kplus0,Kminus0,Kplus1,Kminus1
         # The parent is automatically deduced from the annotated decay descriptor,
         # which is passed in a file such as eff_ntuple.root.json, but can also be given:

HltEfficiencyChecker/options/hlt1_rate_example.yaml

@@ -24,7 +24,6 @@ analysis:
     script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py
     args:
         input: *NTUPLE
-        level: Hlt1
         plot_rates: true
         plot_total_rate: true
         plot_group_rates: true

HltEfficiencyChecker/options/hlt1_rate_example_moore.yaml

@@ -26,7 +26,6 @@ analysis:
     script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py
     args:
         input: *NTUPLE
-        level: Hlt1
         plot_rates: true
         plot_total_rate: true
         plot_group_rates: true

HltEfficiencyChecker/options/hlt2_eff_example.py

@@ -11,7 +11,6 @@
 from Moore import options
 from HltEfficiencyChecker.config import run_moore_with_tuples
 from RecoConf.reconstruction_objects import reconstruction
-from RecoConf.protoparticles import make_charged_protoparticles
 
 decay = (
     "${Bs}[B_s0 => ( J/psi(1S) => ${mup}mu+ ${mum}mu- ) ( phi(1020) => ${Kp}K+ ${Km}K- )]CC"
@@ -49,8 +48,7 @@ default_ft_decoding_version.global_bind(value=2)
 
 # TODO stateProvider_with_simplified_geom must go away from option files
 from RecoConf.global_tools import stateProvider_with_simplified_geom
-with reconstruction.bind(from_file=False), make_charged_protoparticles.bind(
-        enable_muon_id=True):
+with reconstruction.bind(from_file=False):
     run_moore_with_tuples(
         options,
         False,

HltEfficiencyChecker/options/hlt2_eff_example.yaml

@@ -34,7 +34,6 @@ analysis:
     script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py
     args:
         input: *NTUPLE
-        level: Hlt2
         reconstructible_children: mup,mum,Kp,Km
         # The parent is automatically deduced from the annotated decay descriptor,
         # which is passed in a file such as eff_ntuple.root.json, but can also be given:

HltEfficiencyChecker/options/hlt2_eff_example_reco_from_file.yaml

@@ -33,7 +33,6 @@ analysis:
     script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py
     args:
         input: *NTUPLE
-        level: Hlt2
         reconstructible_children: mup,mum,Kp,Km
         # The parent is automatically deduced from the annotated decay descriptor,
         # which is passed in a file such as eff_ntuple.root.json, but can also be given:

HltEfficiencyChecker/options/hlt2_rate_example.py

@@ -11,12 +11,10 @@
 from Moore import options
 from HltEfficiencyChecker.config import run_moore_with_tuples
 from RecoConf.reconstruction_objects import reconstruction
-from RecoConf.protoparticles import make_charged_protoparticles
 
-options.set_input_from_testfiledb('upgrade_minbias_hlt1_filtered')
+options.set_input_and_conds_from_testfiledb('upgrade_minbias_hlt1_filtered')
 options.input_raw_format = 4.3
 options.evt_max = 100
-options.set_conds_from_testfiledb('upgrade_minbias_hlt1_filtered')
 
 options.ntuple_file = "rate_ntuple.root"
 
@@ -25,7 +23,6 @@ default_ft_decoding_version.global_bind(value=2)
 
 # TODO stateProvider_with_simplified_geom must go away from option files
 from RecoConf.global_tools import stateProvider_with_simplified_geom
-with reconstruction.bind(from_file=False),\
-     make_charged_protoparticles.bind(enable_muon_id=True):
+with reconstruction.bind(from_file=False):
     run_moore_with_tuples(
         options, False, public_tools=[stateProvider_with_simplified_geom()])

HltEfficiencyChecker/options/hlt2_rate_example.yaml

@@ -28,5 +28,5 @@ analysis:
     script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py
     args:
         input: *NTUPLE
-        level: Hlt2
+        using_hlt1_filtered_MC: true
         json: Hlt2_rates.json

HltEfficiencyChecker/options/hlt2_tos_eff_example.py

@@ -11,7 +11,6 @@
 from Moore import options
 from HltEfficiencyChecker.config import run_moore_with_tuples
 from RecoConf.reconstruction_objects import reconstruction
-from RecoConf.protoparticles import make_charged_protoparticles
 from Hlt2Conf.lines.Bs2JpsiPhi import all_lines
 
 decay = ("[${B_s0}B_s0 => "
@@ -55,8 +54,7 @@ from RecoConf.hlt1_tracking import default_ft_decoding_version
 default_ft_decoding_version.global_bind(value=2)
 
 from RecoConf.global_tools import stateProvider_with_simplified_geom
-with reconstruction.bind(from_file=False), make_charged_protoparticles.bind(
-        enable_muon_id=True):
+with reconstruction.bind(from_file=False):
     run_moore_with_tuples(
         options,
         False,

HltEfficiencyChecker/options/options_template.py.jinja

 from Moore import options
 from HltEfficiencyChecker.config import run_moore_with_tuples
 from HltEfficiencyChecker.config import run_allen_in_moore_with_tuples
+from HltEfficiencyChecker.config import run_chained_hlt_with_tuples
 from RecoConf.reconstruction_objects import reconstruction
-from RecoConf.protoparticles import make_charged_protoparticles
 from RecoConf.hlt1_allen import sequence as allen_sequence
 
 {% if annotated_decay_descriptor is defined %}
@@ -11,14 +11,14 @@ decay = "{{ annotated_decay_descriptor }}"
 decay = None
 {% endif %}
 options.ntuple_file = "{{ ntuple_path }}"
-level = {{ job.trigger_level }}
-assert(level == 1 or level == 2)
+level = "{{ job.trigger_level }}"
+assert(level == "1" or level == "2" or level == "both")
 
-if level == 1:
+if level == "1" or level == "both":
     use_allen = True
 {% if job.use_moore_as_hlt1 is defined %}
-if level == 1:
-    use_allen &= not {{ job.use_moore_as_hlt1 }}
+assert level == "1",  "Doesn\'t make sense to run HLT2 with the \"use_moore_as_hlt1\" flag, and only Allen HLT1 should be used for chained HLT1+HLT2."
+use_allen &= not {{ job.use_moore_as_hlt1 }}
 {% endif %}
 
 {% if job.testfiledb_key is defined %}
@@ -30,9 +30,18 @@ options.set_conds_from_testfiledb("{{ job.testfiledb_key }}")
 {% endif %}
 
 {% if job.lines_from is defined %}
+assert level == "2" or level == "both", "\"lines_from\" key only to be used when running HLT2"
 assert("Hlt2Conf" in "{{ job.lines_from }}"), "You can only specify a Hlt2 line at this time from Hlt2Conf. Aborting..."
 from {{ job.lines_from }} import all_lines
+{% endif %}
+
+{% if job.hlt2_lines_from is defined %}
+assert level == "2" or level == "both", "\"hlt2_lines_from\" key only to be used when running HLT2"
+assert("Hlt2Conf" in "{{ job.hlt2_lines_from }}"), "You can only specify a Hlt2 line at this time from Hlt2Conf. Aborting..."
+from {{ job.hlt2_lines_from }} import all_lines
+{% endif %}
 
+{% if job.hlt2_lines_from is defined or job.lines_from is defined %}
 def make_lines():
     return [builder() for builder in all_lines.values()]
 
@@ -47,9 +56,8 @@ options.{{ key }} = {{ "{!r}".format(job[key]) }}
 {% endif %}
 {% endfor %}
 
-# Uncomment the following to increase the output verbosity
 {% if job.debug_mode is defined %}
-if {{ job. debug_mode }}:
+if {{ job.debug_mode }}:
     from Gaudi.Configuration import DEBUG
     options.output_level = DEBUG
 {% endif %}
@@ -67,16 +75,15 @@ run_reconstruction = {{ job.run_reconstruction }}
 run_reconstruction = False
 {% endif %}
 
-if level == 2:
+if level == "2":
     if run_reconstruction:
-        with reconstruction.bind(from_file={{ not job.run_reconstruction }}),\
-            make_charged_protoparticles.bind(enable_muon_id=True):
+        with reconstruction.bind(from_file={{ not job.run_reconstruction }}):
             run_moore_with_tuples(
                 options, False, decay, public_tools=[stateProvider_with_simplified_geom()])
     else:
         run_moore_with_tuples(
             options, False, decay, public_tools=[stateProvider_with_simplified_geom()])
-else:
+elif level == "1":
     if use_allen:
 {% if job.allen_sequence is defined %}
         with allen_sequence.bind(sequence={{ "{!r}".format(job.allen_sequence) }}):
@@ -86,3 +93,17 @@ else:
 {% endif %}
     else:
         run_moore_with_tuples(options, True, decay)
+else:
+    assert use_allen, "Chained HLT1 and HLT2 should only be used for Allen HLT1."
+{% if job.allen_sequence is defined %}
+    with allen_sequence.bind(sequence={{ "{!r}".format(job.allen_sequence) }}),\
+        reconstruction.bind(from_file={{ not job.run_reconstruction }}):
+
+        run_chained_hlt_with_tuples(
+            options, decay, public_tools=[stateProvider_with_simplified_geom()])
+{% else %}
+    with reconstruction.bind(from_file={{ not job.run_reconstruction }}):
+
+        run_chained_hlt_with_tuples(
+            options, decay, public_tools=[stateProvider_with_simplified_geom()])
+{% endif %}

HltEfficiencyChecker/options/run_chained_hlt_examples.sh

+#!/bin/bash
+###############################################################################
+# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+set -euxo pipefail
+
+default_dir=checker-chained-hlt-examples-$(date '+%Y%m%d-%H%M%S')
+dir=${1:-$default_dir}
+mkdir -p $dir
+cd $dir
+
+# Efficiency
+
+gaudirun.py \
+    $HLTEFFICIENCYCHECKERROOT/options/hlt1_and_hlt2_eff_example.py \
+    -o eff_job.opts.py --all-opts \
+    2>&1 | tee eff_job.log
+
+$HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py \
+    --reconstructible-children=Kplus0,Kminus0,Kplus1,Kminus1 \
+    --legend-header="B_{s} #rightarrow #phi #phi" \
+    --make-plots \
+    --TOS-match-to B_s0 \
+    hlt1_and_hlt2_eff_ntuple.root \
+    2>&1 | tee eff_results.log
+
+# Rates
+
+gaudirun.py \
+    $HLTEFFICIENCYCHECKERROOT/options/hlt1_and_hlt2_rate_example.py \
+    -o rate_job.opts.py --all-opts \
+    2>&1 | tee rate_job.log
+
+$HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py \
+    --plot-rates --plot-total-rate --plot-group-rates \
+    --json Hlt2_rates.json \
+    hlt1_and_hlt2_rate_ntuple.root\
+    2>&1 | tee rate_results.log

HltEfficiencyChecker/options/run_hlt1_allen_examples.sh

@@ -26,7 +26,6 @@ gaudirun.py \
     2>&1 | tee "eff_job_${sequence}".log
 
 $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py \
-    --level Hlt1 \
     --reconstructible-children=Kplus0,Kminus0,Kplus1,Kminus1 \
     --legend-header="B^{0}_{s} #rightarrow #phi#phi" \
     --make-plots \
@@ -42,7 +41,6 @@ gaudirun.py \
     2>&1 | tee "rate_job_${sequence}.log"
 
 $HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py \
-    --level Hlt1 \
     --plot-rates --plot-total-rate --plot-group-rates \
     --json "Hlt1_rates_${sequence}.json" \
     "rate_ntuple_${sequence}.root" \

HltEfficiencyChecker/options/run_hlt1_moore_examples.sh

@@ -26,7 +26,6 @@ gaudirun.py \
     2>&1 | tee eff_job.log
 
 $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py \
-    --level Hlt1 \
     --reconstructible-children=muplus,muminus,Kplus,Kminus \
     --legend-header="B^{0}_{s} #rightarrow #phi#phi" \
     --make-plots \
@@ -42,7 +41,6 @@ gaudirun.py \
     2>&1 | tee rate_job.log
 
 $HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py \
-    --level Hlt1 \
     --plot-rates --plot-total-rate --plot-group-rates \
     --json Hlt1_rates.json \
     rate_ntuple.root \