Commit 0e9f9eef authored by Rosen Matev's avatar Rosen Matev
Browse files

Merge branch 'chain-hlt1-and-hlt2' into 'master'

Add ability to chain HLT1 and HLT2 in 1 HltEfficiencyChecker job

Closes #12 and #24

See merge request !58
parents 5bff25c3 3f5cd6e2
Pipeline #3191234 passed with stage
in 15 seconds
###############################################################################
# (c) Copyright 2019-2020 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
from Moore import options
from HltEfficiencyChecker.config import run_chained_hlt_with_tuples
from RecoConf.hlt1_allen import sequence as allen_sequence
from RecoConf.reconstruction_objects import reconstruction
from RecoConf.protoparticles import make_charged_protoparticles
from Hlt2Conf.lines.topological_b import all_lines
decay = ("[${B_s0}B_s0 => "
"( phi(1020) => ${Kplus0}K+ ${Kminus0}K-) "
"( phi(1020) => ${Kplus1}K+ ${Kminus1}K-) ]CC")
def make_hlt2_lines():
return [builder() for builder in all_lines.values()]
options.lines_maker = make_hlt2_lines
options.set_input_from_testfiledb('Upgrade_BsPhiPhi_MD_FTv4_DIGI')
options.evt_max = 50
options.set_conds_from_testfiledb('Upgrade_BsPhiPhi_MD_FTv4_DIGI')
options.ntuple_file = "hlt1_and_hlt2_eff_ntuple.root"
# needed to run over FTv2 data
from RecoConf.hlt1_tracking import default_ft_decoding_version
default_ft_decoding_version.global_bind(value=4)
from RecoConf.global_tools import stateProvider_with_simplified_geom
with allen_sequence.bind(sequence="hlt1_pp_default"), reconstruction.bind(
from_file=False), make_charged_protoparticles.bind(
enable_muon_id=True):
run_chained_hlt_with_tuples(
options, decay, public_tools=[stateProvider_with_simplified_geom()])
# HltEfficiencyChecker "wizard" example running HLT1 and HLT2
annotated_decay_descriptor:
"[${B_s0}B_s0 => ( phi(1020) => ${Kplus0}K+ ${Kminus0}K-) ( phi(1020) => ${Kplus1}K+ ${Kminus1}K-) ]CC"
ntuple_path: &NTUPLE hlt1_and_hlt2_eff_ntuple.root
job:
trigger_level: both
evt_max: 50
testfiledb_key: Upgrade_BsPhiPhi_MD_FTv4_DIGI
ft_decoding_version: 4
hlt2_lines_from: Hlt2Conf.lines.topological_b
allen_sequence: hlt1_pp_default
run_reconstruction: true
options:
- $HLTEFFICIENCYCHECKERROOT/options/options_template.py.jinja # first rendered with jinja2
analysis:
script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py
args:
input: *NTUPLE
reconstructible_children: Kplus0,Kminus0,Kplus1,Kminus1
legend_header: "B^{0}_{s} #rightarrow #phi#phi"
make_plots: true
vars: "PT,Kplus0:PT"
lines: Hlt2Topo2BodyLineDecision
TOS_match_to: "B_s0,Kplus0"
custom_denoms: "Hlt1trackmvas: Hlt1TrackMVADecision || Hlt1TwoTrackMVADecision"
###############################################################################
# (c) Copyright 2019-2020 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
from Moore import options
from HltEfficiencyChecker.config import run_chained_hlt_with_tuples
from RecoConf.reconstruction_objects import reconstruction
from RecoConf.protoparticles import make_charged_protoparticles
from Hlt2Conf.lines.topological_b import all_lines
def make_lines():
return [builder() for builder in all_lines.values()]
options.lines_maker = make_lines
options.set_input_and_conds_from_testfiledb('upgrade_DC19_01_MinBiasMD')
options.evt_max = 50
options.ntuple_file = "hlt1_and_hlt2_rate_ntuple.root"
from RecoConf.hlt1_tracking import default_ft_decoding_version
default_ft_decoding_version.global_bind(value=6)
# TODO stateProvider_with_simplified_geom must go away from option files
from RecoConf.global_tools import stateProvider_with_simplified_geom
with reconstruction.bind(from_file=False),\
make_charged_protoparticles.bind(enable_muon_id=True):
run_chained_hlt_with_tuples(
options, public_tools=[stateProvider_with_simplified_geom()])
# HltEfficiencyChecker "wizard" example for Hlt2 given Hlt1 rates
ntuple_path: &NTUPLE hlt1_and_hlt2_rate_ntuple.root
job:
trigger_level: both
evt_max: 50
testfiledb_key: upgrade_DC19_01_MinBiasMD
hlt2_lines_from: Hlt2Conf.lines.topological_b
run_reconstruction: True
ft_decoding_version: 6
options:
- $HLTEFFICIENCYCHECKERROOT/options/options_template.py.jinja # first rendered with jinja2
analysis:
script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py
args:
input: *NTUPLE
lines: Hlt2Topo2BodyLineDecision
filter_lines: AnyHlt1Line
json: Hlt2_rates_filtered_on_Hlt1TrackMVALines.json
......@@ -28,7 +28,6 @@ analysis:
script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py
args:
input: *NTUPLE
level: Hlt1
reconstructible_children: Kplus0,Kminus0,Kplus1,Kminus1
# The parent is automatically deduced from the annotated decay descriptor,
# which is passed in a file such as eff_ntuple.root.json, but can also be given:
......
......@@ -28,7 +28,6 @@ analysis:
script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py
args:
input: *NTUPLE
level: Hlt1
reconstructible_children: muplus,muminus,Kplus,Kminus
# The parent is automatically deduced from the annotated decay descriptor,
# which is passed in a file such as eff_ntuple.root.json, but can also be given:
......
......@@ -27,7 +27,6 @@ analysis:
script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py
args:
input: *NTUPLE
level: Hlt1
reconstructible_children: Kplus0,Kminus0,Kplus1,Kminus1
# The parent is automatically deduced from the annotated decay descriptor,
# which is passed in a file such as eff_ntuple.root.json, but can also be given:
......
......@@ -24,7 +24,6 @@ analysis:
script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py
args:
input: *NTUPLE
level: Hlt1
plot_rates: true
plot_total_rate: true
plot_group_rates: true
......
......@@ -26,7 +26,6 @@ analysis:
script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py
args:
input: *NTUPLE
level: Hlt1
plot_rates: true
plot_total_rate: true
plot_group_rates: true
......
......@@ -34,7 +34,6 @@ analysis:
script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py
args:
input: *NTUPLE
level: Hlt2
reconstructible_children: mup,mum,Kp,Km
# The parent is automatically deduced from the annotated decay descriptor,
# which is passed in a file such as eff_ntuple.root.json, but can also be given:
......
......@@ -33,7 +33,6 @@ analysis:
script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py
args:
input: *NTUPLE
level: Hlt2
reconstructible_children: mup,mum,Kp,Km
# The parent is automatically deduced from the annotated decay descriptor,
# which is passed in a file such as eff_ntuple.root.json, but can also be given:
......
......@@ -13,10 +13,9 @@ from HltEfficiencyChecker.config import run_moore_with_tuples
from RecoConf.reconstruction_objects import reconstruction
from RecoConf.protoparticles import make_charged_protoparticles
options.set_input_from_testfiledb('upgrade_minbias_hlt1_filtered')
options.set_input_and_conds_from_testfiledb('upgrade_minbias_hlt1_filtered')
options.input_raw_format = 4.3
options.evt_max = 100
options.set_conds_from_testfiledb('upgrade_minbias_hlt1_filtered')
options.ntuple_file = "rate_ntuple.root"
......
......@@ -28,5 +28,5 @@ analysis:
script: $HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py
args:
input: *NTUPLE
level: Hlt2
using_hlt1_filtered_MC: true
json: Hlt2_rates.json
from Moore import options
from HltEfficiencyChecker.config import run_moore_with_tuples
from HltEfficiencyChecker.config import run_allen_in_moore_with_tuples
from HltEfficiencyChecker.config import run_chained_hlt_with_tuples
from RecoConf.reconstruction_objects import reconstruction
from RecoConf.protoparticles import make_charged_protoparticles
from RecoConf.hlt1_allen import sequence as allen_sequence
......@@ -11,14 +12,14 @@ decay = "{{ annotated_decay_descriptor }}"
decay = None
{% endif %}
options.ntuple_file = "{{ ntuple_path }}"
level = {{ job.trigger_level }}
assert(level == 1 or level == 2)
level = "{{ job.trigger_level }}"
assert(level == "1" or level == "2" or level == "both")
if level == 1:
if level == "1" or level == "both":
use_allen = True
{% if job.use_moore_as_hlt1 is defined %}
if level == 1:
use_allen &= not {{ job.use_moore_as_hlt1 }}
assert level == "1", "Doesn\'t make sense to run HLT2 with the \"use_moore_as_hlt1\" flag, and only Allen HLT1 should be used for chained HLT1+HLT2."
use_allen &= not {{ job.use_moore_as_hlt1 }}
{% endif %}
{% if job.testfiledb_key is defined %}
......@@ -30,9 +31,18 @@ options.set_conds_from_testfiledb("{{ job.testfiledb_key }}")
{% endif %}
{% if job.lines_from is defined %}
assert level == "2" or level == "both", "\"lines_from\" key only to be used when running HLT2"
assert("Hlt2Conf" in "{{ job.lines_from }}"), "You can only specify a Hlt2 line at this time from Hlt2Conf. Aborting..."
from {{ job.lines_from }} import all_lines
{% endif %}
{% if job.hlt2_lines_from is defined %}
assert level == "2" or level == "both", "\"hlt2_lines_from\" key only to be used when running HLT2"
assert("Hlt2Conf" in "{{ job.hlt2_lines_from }}"), "You can only specify a Hlt2 line at this time from Hlt2Conf. Aborting..."
from {{ job.hlt2_lines_from }} import all_lines
{% endif %}
{% if job.hlt2_lines_from is defined or job.lines_from is defined %}
def make_lines():
return [builder() for builder in all_lines.values()]
......@@ -47,9 +57,8 @@ options.{{ key }} = {{ "{!r}".format(job[key]) }}
{% endif %}
{% endfor %}
# Uncomment the following to increase the output verbosity
{% if job.debug_mode is defined %}
if {{ job. debug_mode }}:
if {{ job.debug_mode }}:
from Gaudi.Configuration import DEBUG
options.output_level = DEBUG
{% endif %}
......@@ -67,7 +76,7 @@ run_reconstruction = {{ job.run_reconstruction }}
run_reconstruction = False
{% endif %}
if level == 2:
if level == "2":
if run_reconstruction:
with reconstruction.bind(from_file={{ not job.run_reconstruction }}),\
make_charged_protoparticles.bind(enable_muon_id=True):
......@@ -76,7 +85,7 @@ if level == 2:
else:
run_moore_with_tuples(
options, False, decay, public_tools=[stateProvider_with_simplified_geom()])
else:
elif level == "1":
if use_allen:
{% if job.allen_sequence is defined %}
with allen_sequence.bind(sequence={{ "{!r}".format(job.allen_sequence) }}):
......@@ -86,3 +95,19 @@ else:
{% endif %}
else:
run_moore_with_tuples(options, True, decay)
else:
assert use_allen, "Chained HLT1 and HLT2 should only be used for Allen HLT1."
{% if job.allen_sequence is defined %}
with allen_sequence.bind(sequence={{ "{!r}".format(job.allen_sequence) }}),\
reconstruction.bind(from_file={{ not job.run_reconstruction }}),\
make_charged_protoparticles.bind(enable_muon_id=True):
run_chained_hlt_with_tuples(
options, decay, public_tools=[stateProvider_with_simplified_geom()])
{% else %}
with reconstruction.bind(from_file={{ not job.run_reconstruction }}),\
make_charged_protoparticles.bind(enable_muon_id=True):
run_chained_hlt_with_tuples(
options, decay, public_tools=[stateProvider_with_simplified_geom()])
{% endif %}
#!/bin/bash
###############################################################################
# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
set -euxo pipefail
default_dir=checker-chained-hlt-examples-$(date '+%Y%m%d-%H%M%S')
dir=${1:-$default_dir}
mkdir -p $dir
cd $dir
# Efficiency
gaudirun.py \
$HLTEFFICIENCYCHECKERROOT/options/hlt1_and_hlt2_eff_example.py \
-o eff_job.opts.py --all-opts \
2>&1 | tee eff_job.log
$HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py \
--reconstructible-children=Kplus0,Kminus0,Kplus1,Kminus1 \
--legend-header="B_{s} #rightarrow #phi #phi" \
--make-plots \
--TOS-match-to B_s0 \
hlt1_and_hlt2_eff_ntuple.root \
2>&1 | tee eff_results.log
# Rates
gaudirun.py \
$HLTEFFICIENCYCHECKERROOT/options/hlt1_and_hlt2_rate_example.py \
-o rate_job.opts.py --all-opts \
2>&1 | tee rate_job.log
$HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py \
--plot-rates --plot-total-rate --plot-group-rates \
--json Hlt2_rates.json \
hlt1_and_hlt2_rate_ntuple.root\
2>&1 | tee rate_results.log
......@@ -26,7 +26,6 @@ gaudirun.py \
2>&1 | tee "eff_job_${sequence}".log
$HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py \
--level Hlt1 \
--reconstructible-children=Kplus0,Kminus0,Kplus1,Kminus1 \
--legend-header="B^{0}_{s} #rightarrow #phi#phi" \
--make-plots \
......@@ -42,7 +41,6 @@ gaudirun.py \
2>&1 | tee "rate_job_${sequence}.log"
$HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py \
--level Hlt1 \
--plot-rates --plot-total-rate --plot-group-rates \
--json "Hlt1_rates_${sequence}.json" \
"rate_ntuple_${sequence}.root" \
......
......@@ -26,7 +26,6 @@ gaudirun.py \
2>&1 | tee eff_job.log
$HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py \
--level Hlt1 \
--reconstructible-children=muplus,muminus,Kplus,Kminus \
--legend-header="B^{0}_{s} #rightarrow #phi#phi" \
--make-plots \
......@@ -42,7 +41,6 @@ gaudirun.py \
2>&1 | tee rate_job.log
$HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py \
--level Hlt1 \
--plot-rates --plot-total-rate --plot-group-rates \
--json Hlt1_rates.json \
rate_ntuple.root \
......
......@@ -24,7 +24,6 @@ gaudirun.py \
2>&1 | tee tos_eff_job.log
$HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py \
--level Hlt1 \
--reconstructible-children=muplus,muminus,Kplus,Kminus \
--legend-header="B^{0}_{s} #rightarrow J/#psi#phi" \
--make-plots \
......
......@@ -25,7 +25,6 @@ gaudirun.py \
2>&1 | tee eff_job.log
$HLTEFFICIENCYCHECKERROOT/scripts/hlt_line_efficiencies.py \
--level Hlt2 \
--reconstructible-children=Kp,Km,mup,mum \
--legend-header="B_{s} #rightarrow J/#psi #phi" \
--make-plots \
......@@ -41,7 +40,6 @@ gaudirun.py \
2>&1 | tee rate_job.log
$HLTEFFICIENCYCHECKERROOT/scripts/hlt_calculate_rates.py \
--level Hlt2 \
--plot-rates --plot-total-rate --plot-group-rates \
--json Hlt2_rates.json \
rate_ntuple.root \
......
......@@ -312,6 +312,136 @@ def add_efficiency_tuples(moore_cf, lines, descriptor_template=None):
force_order=True)
def run_chained_hlt_with_tuples(options,
descriptor_template=None,
public_tools=[]):
assert options.ntuple_file, "You must set options.ntuple_file"
if descriptor_template:
with open(options.ntuple_file + ".metadata.json", "w") as f:
json.dump({"descriptor_template": descriptor_template}, f)
config = configure_input(options)
from Allen.config import setup_allen_non_event_data_service
# An ExtSvc is added to the ApplicationMgr to provide the Allen non-event data (geometries etc.)
setup_allen_non_event_data_service()
hlt1_node = CompositeNode(
name='Allen_running',
children=[make_allen_dec_reports()],
combine_logic=NodeLogic.NONLAZY_OR,
force_order=False)
hlt2_streams = options.lines_maker()
#If streams is a list create dict {default : `DecisionLine` objects}
if not isinstance(hlt2_streams, dict):
hlt2_streams = dict(default=hlt2_streams)
hlt2_node, barrier_algs = moore_control_flow(options, hlt2_streams, "hlt2",
True)
top_cf_node = add_chained_efficiency_tuples(
hlt1_node,
hlt2_node,
hlt2_streams["default"],
descriptor_template=descriptor_template)
config.update(
configure(
options,
top_cf_node,
public_tools=public_tools,
barrier_algorithms=barrier_algs))
return config
def add_chained_efficiency_tuples(allen_hlt1_node, moore_hlt2_node, hlt2_lines,
descriptor_template):
efficiency_mode = bool(descriptor_template)
hlt1_decision_names = list(get_allen_hlt1_decision_ids().keys())
hlt1_dec_rep_loc = make_allen_dec_reports().location
# Make sure that Hlt__RoutingBitsWriter is called for every event
# TODO add a "flagging mode" option to be used in moore_control_flow
moore_hlt2_node.combine_logic = NodeLogic.NONLAZY_AND
# TODO find a less fragile way to obtain that
hlt2_dec_reports_maker = moore_hlt2_node.children[1].children[0]
hlt2_dec_rep_loc = hlt2_dec_reports_maker.DecReportsLocation.location
hlt2_decision_names = [
name + "Decision"
for name in hlt2_dec_reports_maker.properties["Persist"]
]
if efficiency_mode:
mcttkin = MCTupleToolKinematic(
'MCDecayTreeTuple.MCTupleToolKinematic', StoreEtaInfo=True)
mcttrec = MCTupleToolReconstructed(Associate=False, FillPID=False)
sel_rep_decoder = decode_allen_sel_reports()
sel_rep_loc = sel_rep_decoder.OutputHltSelReportsLocation.location
mctttos_hlt1 = MCTupleToolTOSHLT1(
'MCDecayTreeTuple.MCTupleToolTOSHLT1',
TriggerList=hlt1_decision_names,
MC2IDLink=make_links_lhcbids_mcparticles_tracking_system().
location,
HltSelReports=sel_rep_loc)
mctttos_hlt2 = MCTupleToolTOSHLT2(
'MCDecayTreeTuple.MCTupleToolTOSHLT2',
TriggerList=hlt2_decision_names,
MC2IDLink=make_links_lhcbids_mcparticles_tracking_system().
location,
CandidateLocations=[
line.objects_to_persist[0].location for line in hlt2_lines
],
Hlt2DecReports=hlt2_dec_rep_loc)
# Want the following tools for both rates and line efficiencies
tuple_flag = 'MCDecayTreeTuple' if efficiency_mode else 'EventTuple'
ttt = TupleToolTrigger(
f'{tuple_flag}.TupleToolTrigger',
VerboseHlt1=True,
VerboseHlt2=True,
FillHlt1=True,
FillHlt2=True,
Hlt1DecReports=hlt1_dec_rep_loc,
Hlt2DecReports=hlt2_dec_rep_loc,
TriggerList=hlt1_decision_names + hlt2_decision_names)
ttei = TupleToolEventInfo(
f'{tuple_flag}.TupleToolEventInfo',
ODIN=make_odin().location,
Hlt1DecReports=hlt1_dec_rep_loc,
Hlt2DecReports=hlt2_dec_rep_loc,
)
tool_list = [ttei.getFullName(), ttt.getFullName()]
if efficiency_mode:
decay, branches, _ = parse_descriptor_template(descriptor_template)
mcdtt = _mc_decay_tree_tuple(
Decay=decay,
Branches=branches,
ToolList=tool_list + [
mcttkin.getFullName(),
mcttrec.getFullName(),
mctttos_hlt1.getFullName(),
mctttos_hlt2.getFullName()
])
nodes = [
make_links_lhcbids_mcparticles_tracking_system(),
make_mc_track_info(), sel_rep_decoder, mcdtt
]
else:
et = EventTuple(ToolList=tool_list)
nodes = [et]
return CompositeNode(
'with_efficiency_tuples',
combine_logic=NodeLogic.NONLAZY_AND,
children=[allen_hlt1_node, moore_hlt2_node] + nodes,
force_order=True)
def parse_descriptor_template(template):
"""Return decay descriptor and branches from a template.
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment