Merge branch 'fake_upstream_for_data' into 'master'

Rework creation of Upstream protoparticles and add fallback for data without UT

See merge request !2081

Hlt/Hlt2Conf/options/hlt2_pp_hlt1passthrough_data_production_options.yaml

+data_type: Upgrade
+simulation: False
+dddb_tag: upgrade/master
+conddb_tag: upgrade/master
+geometry_version: trunk
+conditions_version: master
+
+input_files:
+  - mdf:root://eoslhcb.cern.ch//eos/lhcb/wg/rta/data/hlt1_passthrough_data_2022/255620_00150010_0033.raw
+input_type: RAW
+evt_max: 100
+
+xml_summary_file: hlt2_pp_hlt1passthrough_data_summary.xml
+xml_file_catalog: hlt2_data_pool_xml_catalog.xml
+
+scheduler_legacy_mode: False
+
+output_file: hlt2_pp_hlt1passthrough_data_output.dst
+output_type: 'ROOT'

Hlt/Moore/tests/qmtest/test_lbexec_hlt2_pp_hlt1passthrough_data.qmt

+<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE extension  PUBLIC '-//QM/2.3/Extension//EN'  'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
+<!--
+    (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration
+
+    This software is distributed under the terms of the GNU General Public
+    Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
+
+    In applying this licence, CERN does not waive the privileges and immunities
+    granted to it by virtue of its status as an Intergovernmental Organization
+    or submit itself to any jurisdiction.
+-->
+<!--
+Test that HLT2 can run from LHCbDirac and that it produces the expected outputs
+-->
+<extension class="GaudiTest.GaudiExeTest" kind="test">
+<argument name="prerequisites"><set>
+  <tuple><text>test_prepare_lbexec_hlt</text><enumeral>PASS</enumeral></tuple>
+</set></argument>
+<argument name="program"><text>lbexec</text></argument>
+<argument name="args"><set>
+  <text>Moore.production:hlt2_pp_commissioning</text>
+  <text>$HLT2CONFROOT/options/hlt2_pp_hlt1passthrough_data_production_options.yaml</text>
+</set></argument>
+<argument name="use_temp_dir"><enumeral>true</enumeral></argument>
+<argument name="validator"><text>
+
+countErrorLines({"FATAL": 0, "ERROR": 10})
+
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+# Ensure the summary XML is as expected
+tree = ET.parse(Path.cwd() / "hlt2_pp_hlt1passthrough_data_summary.xml")
+out_files = tree.findall("./output/file")
+assert len(out_files) == 1, out_files
+out_file = out_files[0]
+assert int(out_file.text) &gt; 10
+assert int(out_file.text) &lt; 1234
+assert out_file.attrib["name"] == "PFN:hlt2_pp_hlt1passthrough_data_output.dst"
+
+# Ensure the pool XML was updated to contain the new output file
+tree = ET.parse(Path.cwd() / "hlt2_data_pool_xml_catalog.xml")
+catalog_output = tree.findall(f'./File/physical/pfn[@name="hlt2_pp_hlt1passthrough_data_output.dst"]')
+assert len(catalog_output) == 1, catalog_output
+
+</text></argument>
+</extension>

Hlt/RecoConf/options/hlt2_reco_upstream_particles_low_momentum.py

@@ -16,7 +16,8 @@ from PyConf.control_flow import CompositeNode, NodeLogic
 from PyConf.Algorithms import PrVeloUT
 
 from RecoConf.reconstruction_objects import upfront_reconstruction, reconstruction
-from RecoConf.hlt2_global_reco import make_light_reconstruction
+from RecoConf.hlt2_global_reco import make_light_reco_pr_kf
+from RecoConf.hlt2_global_reco import reconstruction as hlt2_reconstruction
 
 from Hlt2Conf.standard_particles import make_up_electrons_no_brem, make_up_pions, make_has_rich_up_pions, make_up_kaons, make_has_rich_up_kaons, make_up_protons, make_has_rich_up_protons, make_up_muons
 
@@ -45,9 +46,11 @@ def make_upstream_particles():
         force_order=True)
 
 
-with reconstruction.bind(from_file=False), make_light_reconstruction.bind(
-        skipUT=False), PrVeloUT.bind(
-            MinMomentum=500, MinPT=50, MinMomentumFinal=800, MinPTFinal=80):
+with hlt2_reconstruction.bind(
+        make_reconstruction=make_light_reco_pr_kf), reconstruction.bind(
+            from_file=False), PrVeloUT.bind(
+                MinMomentum=500, MinPT=50, MinMomentumFinal=800,
+                MinPTFinal=80):
     config = configure_input(options)
     top_cf_node = make_upstream_particles()
     config.update(configure(options, top_cf_node))

Hlt/RecoConf/python/RecoConf/hlt2_global_reco.py

@@ -13,6 +13,9 @@
 The 'global' reconstruction is that which produces the final output of the
 'full' HLT2 reconstruction: charged and neutral ProtoParticle containers.
 """
+import logging
+log = logging.getLogger(__name__)
+
 from .hlt1_tracking import make_reco_pvs, make_FTRawBankDecoder_clusters
 from .hlt2_tracking import make_hlt2_tracks, make_hlt2_tracks_without_UT, convert_tracks_to_v3_from_v1
 from .rich_reconstruction import make_all_rich_pids, default_rich_reco_options, make_merged_rich_pids
@@ -92,6 +95,7 @@ def make_legacy_reconstruction():
         "NeutralProtos": neutral_protos['ProtoParticleLocation'],
         "IsPhoton": neutral_protos['IsPhoton'],
         "IsNotH": neutral_protos['IsNotH'],
+        "AllTrackHandles": hlt2_tracks,
         "Tracks": best_tracks["v1"],
         "VeloTracks": velo_tracks["v1"],
         "MuonPIDTracks": merge_muontracks,
@@ -108,41 +112,6 @@ def make_legacy_reconstruction():
     }
 
 
-@configurable
-def make_light_upstream_reconstruction(fastReco=True):
-    hlt2_tracks = make_hlt2_tracks(
-        light_reco=True, fast_reco=fastReco, use_pr_kf=True)
-    upstream_tracks = hlt2_tracks["BestUpstream"]
-
-    velo_tracks = hlt2_tracks["Velo"]
-    pvs = make_reco_pvs(velo_tracks, persisted_location('PVs'))
-
-    upstream_track_types = ["Upstream"]
-
-    # RICH
-    richRecConfs = make_all_rich_pids(
-        upstream_tracks,
-        default_rich_reco_options(),
-        track_types=upstream_track_types)
-    rich_pids = richRecConfs["Upstream"]["RichPIDs"]
-
-    # Calo, enable v3 track versions
-    tracks_v3, trackrels = convert_tracks_to_v3_from_v1(
-        upstream_tracks["v1"], track_types=upstream_track_types + ["Ttrack"])
-
-    calo_pids = make_calo(tracks_v3, pvs["v3"], trackrels=trackrels)
-
-    # charged protoparticles
-    upstream_protos = make_charged_protoparticles_from(
-        tracks=upstream_tracks,
-        rich_pids=rich_pids,
-        calo_pids=calo_pids,
-        muon_pids=None,
-        track_types=upstream_track_types)
-
-    return {"ChargedUpstreamProtos": upstream_protos}
-
-
 @configurable
 def make_light_reconstruction(usePatPVFuture=False,
                               use_pr_kf=True,
@@ -250,6 +219,7 @@ def make_light_reconstruction(usePatPVFuture=False,
     # Define the output
     ####
     output = {
+        "AllTrackHandles": hlt2_tracks,
         "Tracks": best_tracks["v1"],
         "VeloTracks": velo_tracks["v1"],
         "PVs": pvs["v3"],
@@ -335,6 +305,17 @@ def make_light_reco_pr_kf_without_UT(skipRich=False,
         fastReco=False)
 
 
+@configurable
+def make_light_reco_pr_kf(skipRich=False, skipCalo=False, skipMuon=False):
+    """Shortcut for running the light reconstruction, with PrKalmanFilter, with UT
+    """
+    return make_light_reconstruction(
+        skipRich=skipRich,
+        skipCalo=skipCalo,
+        skipMuon=skipMuon,
+        fastReco=False)
+
+
 @configurable
 def make_fastest_reconstruction(skipUT=False):
     """Shortcut for running the fastest reconstruction
@@ -411,3 +392,47 @@ def make_tracks():
     """Return a DataHandle to the container of all tracks
     """
     return reconstruction()["Tracks"]
+
+
+@configurable
+def make_upstream_charged_protoparticles(make_reconstruction=reconstruction):
+    hlt2_default_reco = make_reconstruction()
+    hlt2_tracks = hlt2_default_reco["AllTrackHandles"]
+
+    if "BestUpstream" in hlt2_tracks.keys():
+        upstream_tracks = hlt2_tracks["BestUpstream"]
+    else:
+        log.warning(
+            "Upstream tracks are requested but not available in the current configuration. Please check that this is correct."
+        )
+        upstream_tracks = {
+            "v1": TracksEmptyProducer(name="FakeUpstreamTracks")
+        }
+
+    pvs = hlt2_default_reco["PVs"]
+
+    upstream_track_types = ["Upstream"]
+
+    # RICH
+    richRecConfs = make_all_rich_pids(
+        upstream_tracks,
+        default_rich_reco_options(),
+        track_types=upstream_track_types)
+    rich_pids = richRecConfs["Upstream"]["RichPIDs"]
+
+    # Calo, enable v3 track versions
+    tracks_v3, trackrels = convert_tracks_to_v3_from_v1(
+        upstream_tracks["v1"], track_types=upstream_track_types + ["Ttrack"])
+
+    # calo_pids = make_calo(tracks_v3, pvs["v3"], trackrels=trackrels)
+    calo_pids = make_calo(tracks_v3, pvs, trackrels=trackrels)
+
+    # charged protoparticles
+    upstream_protos = make_charged_protoparticles_from(
+        tracks=upstream_tracks,
+        rich_pids=rich_pids,
+        calo_pids=calo_pids,
+        muon_pids=None,
+        track_types=upstream_track_types)
+
+    return upstream_protos

Hlt/RecoConf/python/RecoConf/reconstruction_objects.py

@@ -26,6 +26,7 @@ from RecoConf.reco_objects_for_spruce import (
 )
 
 from RecoConf.hlt2_global_reco import reconstruction as reconstruction_from_reco
+from RecoConf.hlt2_global_reco import make_upstream_charged_protoparticles as make_upstream_charged_protoparticles_from_hlt2
 
 
 @configurable(cached=True)
@@ -172,5 +173,5 @@ def make_calo_mergedPi0s():
 
 
 def make_upstream_charged_protoparticles():
-    from RecoConf.hlt2_global_reco import make_light_upstream_reconstruction
-    return make_light_upstream_reconstruction()["ChargedUpstreamProtos"]
+    # Implementation for PersistReco missing
+    return make_upstream_charged_protoparticles_from_hlt2()