From 4ec0ff339985c724e48984d8b51e7b84f4213c2f Mon Sep 17 00:00:00 2001
From: Maarten Van Veghel <maarten.vanveghel@cern.ch>
Date: Tue, 20 Jun 2023 15:44:44 +0200
Subject: [PATCH] output always latest persistency version; config cleanup
---
.../options/hlt2_pp_thor_without_UT.py | 5 +
.../sprucing/spruce_example_fromfile.py | 1 -
.../python/Hlt2Conf/standard_particles.py | 8 +-
.../tests/options/mdf_input_and_conds_hlt2.py | 1 +
.../python/RecoConf/data_from_file.py | 101 +++++++++++++-----
.../python/RecoConf/hlt2_global_reco.py | 66 +++++++++---
Hlt/RecoConf/python/RecoConf/hlt2_tracking.py | 2 +-
.../python/RecoConf/muon_reconstruction.py | 15 +--
.../python/RecoConf/ttrack_selections_reco.py | 51 +++------
9 files changed, 160 insertions(+), 90 deletions(-)
diff --git a/Hlt/Hlt2Conf/options/hlt2_pp_thor_without_UT.py b/Hlt/Hlt2Conf/options/hlt2_pp_thor_without_UT.py
index 173daa9c9f9..fae02c7f36b 100644
--- a/Hlt/Hlt2Conf/options/hlt2_pp_thor_without_UT.py
+++ b/Hlt/Hlt2Conf/options/hlt2_pp_thor_without_UT.py
@@ -13,7 +13,12 @@
from Moore import options, run_moore
from RecoConf.global_tools import stateProvider_with_simplified_geom, trackMasterExtrapolator_with_simplified_geom
from RecoConf.reconstruction_objects import reconstruction
+<<<<<<< HEAD:Hlt/Hlt2Conf/options/hlt2_pp_thor_without_UT.py
from RecoConf.hlt2_global_reco import reconstruction as hlt2_reconstruction, make_light_reco_pr_kf_without_UT
+=======
+from RecoConf.hlt2_global_reco import (reconstruction as hlt2_reconstruction,
+ make_light_reco_pr_kf_without_UT)
+>>>>>>> 95e1ffced (minor fix and cleanup):Hlt/Hlt2Conf/options/hlt2_pp_thor_without_UT_RICH_Calo_Muon.py
from RecoConf.ttrack_selections_reco import make_ttrack_reco
from Hlt2Conf.lines import all_lines
from DDDB.CheckDD4Hep import UseDD4Hep
diff --git a/Hlt/Hlt2Conf/options/sprucing/spruce_example_fromfile.py b/Hlt/Hlt2Conf/options/sprucing/spruce_example_fromfile.py
index 9551172561f..659fb690716 100644
--- a/Hlt/Hlt2Conf/options/sprucing/spruce_example_fromfile.py
+++ b/Hlt/Hlt2Conf/options/sprucing/spruce_example_fromfile.py
@@ -27,7 +27,6 @@ options.input_process = 'Hlt2'
options.input_files = input_files
options.input_manifest_file = "hlt2_2or3bodytopo_fromfile.tck.json"
options.input_type = 'MDF'
-options.persistency_version = 'v0'
# When running from Upgrade MC, must use the post-juggling locations of the raw
# event
diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/standard_particles.py b/Hlt/Hlt2Conf/python/Hlt2Conf/standard_particles.py
index 964b0cdb785..f1a360c5623 100644
--- a/Hlt/Hlt2Conf/python/Hlt2Conf/standard_particles.py
+++ b/Hlt/Hlt2Conf/python/Hlt2Conf/standard_particles.py
@@ -120,15 +120,17 @@ def standard_protoparticle_filter(Code=F.ALL):
def _make_particles(species,
CheckPID=True,
make_protoparticles=_make_charged_protoparticles,
- track_type=None,
+ track_type='Charged',
get_track_selector=get_all_track_selector,
make_protoparticle_filter=standard_protoparticle_filter):
""" creates LHCb::Particles from LHCb::ProtoParticles """
tp = get_track_selector()
pp = make_protoparticle_filter()
+ pp_maker = make_protoparticles(
+ track_type=track_type
+ ) if track_type != 'Charged' else make_protoparticles()
particles = FunctionalParticleMaker(
- InputProtoParticles=make_protoparticles()
- if track_type is None else make_protoparticles(track_type=track_type),
+ InputProtoParticles=pp_maker,
ParticleID=species,
CheckPID=CheckPID,
TrackPredicate=tp,
diff --git a/Hlt/Moore/tests/options/mdf_input_and_conds_hlt2.py b/Hlt/Moore/tests/options/mdf_input_and_conds_hlt2.py
index a72d19a16b4..89f6a2ccc84 100644
--- a/Hlt/Moore/tests/options/mdf_input_and_conds_hlt2.py
+++ b/Hlt/Moore/tests/options/mdf_input_and_conds_hlt2.py
@@ -29,6 +29,7 @@ options.set_input_and_conds_from_testfiledb(
'upgrade_Sept2022_minbias_0fb_md_mdf')
options.input_files = list(set(options.input_files)) # remove dups
+options.persistency_version = 'v0'
options.evt_max = 300
options.use_iosvc = True
diff --git a/Hlt/RecoConf/python/RecoConf/data_from_file.py b/Hlt/RecoConf/python/RecoConf/data_from_file.py
index 13f7bea16e6..bcfd10b3fd2 100644
--- a/Hlt/RecoConf/python/RecoConf/data_from_file.py
+++ b/Hlt/RecoConf/python/RecoConf/data_from_file.py
@@ -45,11 +45,16 @@ from Configurables import (UnpackMuonPIDs, UnpackRichPIDs, UnpackCaloHypo,
MCMuonHitUnpacker, MCRichDigitSummaryUnpacker)
from PyConf.components import Algorithm, force_location
-from PyConf.packing import persisted_location
+from PyConf.packing import persisted_location, persistency_version
from PyConf.application import make_data_with_FetchDataFromFile
from PyConf.Tools import (ChargedProtoParticleAddRichInfo,
- ChargedProtoParticleAddMuonInfo)
-from PyConf.Algorithms import TracksSharedSplitterPerType, ProtosSharedSplitterPerTrackType, FakeRecSummaryMaker, RecV1ToPVConverter
+ ChargedProtoParticleAddMuonInfo,
+ ChargedProtoParticleAddCaloHypos)
+from PyConf.Algorithms import (TrackContainerFilteredCopy, FakeRecSummaryMaker,
+ RecV1ToPVConverter,
+ ChargedProtoParticleFilteredCopyAlg)
+
+import Functors as F
def _packed_reco_from_file():
@@ -127,8 +132,11 @@ def unpacked_mc_locations():
}
-def postprocess_unpacked_data(data):
- """ Needed for older persistency versions
+def postprocess_unpacked_data(data, persistency_version=persistency_version):
+ """ Needed to unpack older persistency versions into latest persistency style:
+ - for PVs only in v1 format
+ - for merged 'Tracks' and 'ChargedProtos' containers (to split them per track type)
+ - for non-existant RecSummary
"""
# PV v1 only persisted for now
@@ -136,24 +144,55 @@ def postprocess_unpacked_data(data):
data["PVs"] = RecV1ToPVConverter(
InputVertices=data["PVs_v1"]).OutputVertices
- # per track type splitter
- if "LongTracks" not in data.keys():
- split_tracks = TracksSharedSplitterPerType(
- name="TracksSharedSplitterPerType", InputTracks=data['Tracks'])
-
- data['LongTracks'] = split_tracks.LongTracks
- data['DownstreamTracks'] = split_tracks.DownstreamTracks
- data['UpstreamTracks'] = split_tracks.UpstreamTracks
- data['Ttracks'] = split_tracks.Ttracks
-
- split_protos = ProtosSharedSplitterPerTrackType(
- name="ProtosSharedSplitterPerTrackType",
- InputProtos=data['ChargedProtos'])
+ # per track type splitter (to convert 'v0' persistency to latest)
+ if persistency_version() == "v0":
+ split_containers = {}
+ track_predicates = {
+ 'Long': F.TRACKISLONG,
+ 'Downstream': F.TRACKISDOWNSTREAM,
+ 'Upstream': F.TRACKISUPSTREAM
+ }
+ for track_type, track_pred in track_predicates.items():
+ split_containers[track_type] = ChargedProtoParticleFilteredCopyAlg(
+ name=f'ChargedProtoParticleFilteredCopyAlg{track_type}' +
+ '_{hash}',
+ InputProtos=data['ChargedProtos'],
+ TrackPredicate=track_pred,
+ outputs={
+ 'OutputProtos':
+ persisted_location(f'{track_type}Protos'),
+ 'OutputTracks':
+ persisted_location(f'{track_type}Tracks'),
+ 'OutputRichPIDs':
+ persisted_location(f'{track_type}RichPIDs'),
+ 'OutputMuonPIDs':
+ persisted_location(f'{track_type}MuonPIDs')
+ if track_type in ['Long', 'Downstream'] else None,
+ 'OutputMuonTracks':
+ persisted_location(f'{track_type}MuonTracks')
+ if track_type in ['Long', 'Downstream'] else None,
+ })
+ data[f'{track_type}Protos'] = split_containers[
+ track_type].OutputProtos
+ data[f'{track_type}Tracks'] = split_containers[
+ track_type].OutputTracks
+ data[f'{track_type}RichPIDs'] = split_containers[
+ track_type].OutputRichPIDs
+ data[f'{track_type}MuonPIDs'] = split_containers[
+ track_type].OutputMuonPIDs
+ data[f'{track_type}MuonTracks'] = split_containers[
+ track_type].OutputMuonTracks
+ # separate handling of T-tracks
+ data['Ttracks'] = TrackContainerFilteredCopy(
+ name='TrackContainerFilteredCopyTtracks_{hash}',
+ Inputs=[data['Tracks']],
+ Selection=F.TRACKISTTRACK,
+ outputs={
+ 'Output': persisted_location('Ttracks')
+ }).Output
- data['LongProtos'] = split_protos.LongProtos
- data['DownstreamProtos'] = split_protos.DownstreamProtos
- data['UpstreamProtos'] = split_protos.UpstreamProtos
- data['TtrackProtos'] = split_protos.TtrackProtos
+ # or do a copy version and persist that?
+ # should redo AddInfo for calo, rich and muon
### Temporary: This to be compatible with data where the RecSummary does not exist.
if "RecSummary" not in data.keys():
@@ -220,6 +259,11 @@ def reco_unpackers():
richPIDs = reco_unpacker(
'PackedRichPIDs', UnpackRichPIDs, 'UnpackRichPIDs', OutputLevel=ERROR)
+ electrons = reco_unpacker('PackedCaloElectrons', UnpackCaloHypo,
+ 'UnpackCaloElectrons')
+ photons = reco_unpacker('PackedCaloPhotons', UnpackCaloHypo,
+ 'UnpackCaloPhotons')
+
# per track type splitter (if needed)
tracks = reco_unpacker('PackedTracks', UnpackTrack, 'UnpackBestTracks')
charged_protos = reco_unpacker(
@@ -229,18 +273,19 @@ def reco_unpackers():
AddInfo=[
ChargedProtoParticleAddRichInfo(InputRichPIDLocation=richPIDs),
ChargedProtoParticleAddMuonInfo(InputMuonPIDLocation=muonPIDs),
+ # NOTE missing CaloHypo links to CellID due to missing Cluster persistency is not reported
+ ChargedProtoParticleAddCaloHypos(
+ ElectronHypos=electrons,
+ PhotonHypos=photons,
+ ReportMissing=False),
])
# Ordered so that dependents are unpacked first
d = collections.OrderedDict([
('PVs', reco_unpacker('PackedPVs', UnpackRecVertex,
'UnpackRecVertices')),
- ('CaloElectrons',
- reco_unpacker('PackedCaloElectrons', UnpackCaloHypo,
- 'UnpackCaloElectrons')),
- ('CaloPhotons',
- reco_unpacker('PackedCaloPhotons', UnpackCaloHypo,
- 'UnpackCaloPhotons')),
+ ('CaloElectrons', electrons),
+ ('CaloPhotons', photons),
('CaloMergedPi0s',
reco_unpacker('PackedCaloMergedPi0s', UnpackCaloHypo,
'UnpackCaloMergedPi0s')),
diff --git a/Hlt/RecoConf/python/RecoConf/hlt2_global_reco.py b/Hlt/RecoConf/python/RecoConf/hlt2_global_reco.py
index 6135be9305f..b908a70858b 100644
--- a/Hlt/RecoConf/python/RecoConf/hlt2_global_reco.py
+++ b/Hlt/RecoConf/python/RecoConf/hlt2_global_reco.py
@@ -32,15 +32,15 @@ from PyConf.Algorithms import (
ProtoContainersSharedMerger)
-@configurable
-def make_light_reconstruction(usePatPVFuture=False,
- use_pr_kf=True,
- skipUT=False,
- skipRich=False,
- skipCalo=False,
- skipMuon=False,
- fastReco=False):
- """Return reconstruction objects of the fastest or the light reconstruction, with possibility to skip the UT
+def make_reconstruction(light_reco=True,
+ usePatPVFuture=False,
+ use_pr_kf=True,
+ skipUT=False,
+ skipRich=False,
+ skipCalo=False,
+ skipMuon=False,
+ fastReco=False):
+ """Return reconstruction objects of the legacy, fastest or the light reconstruction, with possibility to skip the UT
"""
rich_and_charged_proto_track_types = ["Long"] if skipUT else [
@@ -48,7 +48,7 @@ def make_light_reconstruction(usePatPVFuture=False,
]
hlt2_tracks_maker = make_hlt2_tracks_without_UT if skipUT else make_hlt2_tracks
hlt2_tracks = hlt2_tracks_maker(
- light_reco=True, fast_reco=fastReco, use_pr_kf=use_pr_kf)
+ light_reco=light_reco, fast_reco=fastReco, use_pr_kf=use_pr_kf)
persisted_tracks = get_persisted_tracks_per_type(hlt2_tracks, persist=True)
velo_tracks = hlt2_tracks['Velo']
@@ -76,7 +76,8 @@ def make_light_reconstruction(usePatPVFuture=False,
muon_pids, muon_tracks = make_conv_muon_pids(
muonRecConfs,
persisted_tracks,
- track_types=rich_and_charged_proto_track_types)
+ track_types=rich_and_charged_proto_track_types,
+ use_persisted_locations=True)
# merged versions
muon_tracks['Merged'] = TrackContainersSharedMerger(
name="CreateMuonPIDTrackContainer_{hash}",
@@ -134,7 +135,7 @@ def make_light_reconstruction(usePatPVFuture=False,
"AllTrackHandles": hlt2_tracks,
"Tracks": best_tracks,
"VeloTracks": velo_tracks["v1"],
- "UnfittedTtracks": persisted_tracks['UnfittedTtrack'],
+ "Ttracks": persisted_tracks['UnfittedTtrack'],
"PVs": pvs["v3"],
"PVs_v1": pvs["v1"],
"RichPIDs": rich_pids.get('Merged', None),
@@ -148,7 +149,8 @@ def make_light_reconstruction(usePatPVFuture=False,
}
for track_type, tracks in persisted_tracks.items():
- output[f'{track_type}Tracks'] = tracks
+ if track_type in charged_protos.keys():
+ output[f'{track_type}Tracks'] = tracks
for track_type, protos in charged_protos.items():
output[f'{track_type}RichPIDs'] = rich_pids.get(track_type, None)
@@ -164,6 +166,7 @@ def make_light_reconstruction(usePatPVFuture=False,
})
# for now to keep ttrack config as is alive
+ # this is for specific fitted ttracks, not the standard unfitted decloned ones
if "BestSeed" in hlt2_tracks:
output["BestSeed"] = hlt2_tracks["BestSeed"]["v1"]
@@ -201,6 +204,43 @@ def make_calo_only_reconstruction():
}
+@configurable
+def make_legacy_reconstruction(skipUT=False,
+ skipRich=False,
+ skipCalo=False,
+ skipMuon=False):
+ """Return reconstruction objects of the legacy reconstruction
+ """
+ return make_reconstruction(
+ light_reco=False,
+ use_pr_kf=False,
+ usePatPVFuture=False,
+ skipUT=skipUT,
+ skipRich=skipRich,
+ skipCalo=skipCalo,
+ skipMuon=skipMuon,
+ fastReco=False)
+
+
+@configurable
+def make_light_reconstruction(usePatPVFuture=False,
+ use_pr_kf=True,
+ skipUT=False,
+ skipRich=False,
+ skipCalo=False,
+ skipMuon=False,
+ fastReco=False):
+ return make_reconstruction(
+ light_reco=True,
+ usePatPVFuture=usePatPVFuture,
+ use_pr_kf=use_pr_kf,
+ skipUT=skipUT,
+ skipRich=skipRich,
+ skipCalo=skipCalo,
+ skipMuon=skipMuon,
+ fastReco=fastReco)
+
+
@configurable
def make_light_reco_pr_kf_without_UT(skipRich=False,
skipCalo=False,
diff --git a/Hlt/RecoConf/python/RecoConf/hlt2_tracking.py b/Hlt/RecoConf/python/RecoConf/hlt2_tracking.py
index c0c005d79b3..58a15ad7a9f 100644
--- a/Hlt/RecoConf/python/RecoConf/hlt2_tracking.py
+++ b/Hlt/RecoConf/python/RecoConf/hlt2_tracking.py
@@ -1509,7 +1509,7 @@ def get_persisted_tracks_per_type(hlt2_tracks, persist=True):
},
'UnfittedTtrack': {
'key': 'SeedDecloned',
- 'location': 'UnfittedTtracks'
+ 'location': 'Ttracks'
},
}
diff --git a/Hlt/RecoConf/python/RecoConf/muon_reconstruction.py b/Hlt/RecoConf/python/RecoConf/muon_reconstruction.py
index e39bc775539..b81f45106c1 100644
--- a/Hlt/RecoConf/python/RecoConf/muon_reconstruction.py
+++ b/Hlt/RecoConf/python/RecoConf/muon_reconstruction.py
@@ -50,10 +50,7 @@ def make_conv_muon_pids(muonPidsConfs,
best_tracks,
light_reco=True,
track_types=['Long', 'Downstream'],
- output_locations={
- 'Tracks': {},
- 'PIDs': {}
- }):
+ use_persisted_locations=False):
muon_pids_v1 = dict()
muon_tracks = dict()
for track_type in ['Long', 'Downstream']:
@@ -68,6 +65,7 @@ def make_conv_muon_pids(muonPidsConfs,
outputs={
'OutputMuonTracks':
persisted_location(f'{track_type}MuonTracks')
+ if use_persisted_locations else None
}).OutputMuonTracks
# Convert to Keyed Container for ProtoParticle
@@ -81,18 +79,23 @@ def make_conv_muon_pids(muonPidsConfs,
outputs={
'OutputMuonPIDs':
persisted_location(f'{track_type}MuonPIDs')
+ if use_persisted_locations else None
}).OutputMuonPIDs
else:
muon_tracks[track_type] = TracksEmptyProducer(
name="MuonTracksEmptyProducer_" + track_type + "_{hash}",
outputs={
- 'Output': persisted_location(f'{track_type}MuonTracks')
+ 'Output':
+ persisted_location(f'{track_type}MuonTracks')
+ if use_persisted_locations else None
}).Output
muon_pids_v1[track_type] = MuonPIDsEmptyProducer(
name="MuonPIDsEmptyProducer_" + track_type + "_{hash}",
InputMuonTracks=muon_tracks[track_type],
outputs={
- 'Output': persisted_location(f'{track_type}MuonPIDs')
+ 'Output':
+ persisted_location(f'{track_type}MuonPIDs')
+ if use_persisted_locations else None
}).Output
return muon_pids_v1, muon_tracks
diff --git a/Hlt/RecoConf/python/RecoConf/ttrack_selections_reco.py b/Hlt/RecoConf/python/RecoConf/ttrack_selections_reco.py
index 7586518638a..b6bd8f71e65 100644
--- a/Hlt/RecoConf/python/RecoConf/ttrack_selections_reco.py
+++ b/Hlt/RecoConf/python/RecoConf/ttrack_selections_reco.py
@@ -12,9 +12,7 @@
Define the necessary reconstruction chain to go from Ttracks to particles
"""
from RecoConf.protoparticles import make_charged_protoparticles
-from RecoConf.rich_reconstruction import make_all_rich_pids, default_rich_reco_options, make_merged_rich_pids
-from RecoConf.calorimeter_reconstruction import make_calo
-from RecoConf.hlt2_global_reco import convert_tracks_to_v3_from_v1
+from RecoConf.rich_reconstruction import make_rich_pids, default_rich_reco_options
from RecoConf.reconstruction_objects import reconstruction
from RecoConf.hlt2_tracking import kill_clones, make_TrackBestTrackCreator_tracks
@@ -23,7 +21,6 @@ from PyConf.Algorithms import (TrackListRefiner, TrackSelectionToContainer)
from PyConf.Tools import (ParticleVertexFitter, TrackRungeKuttaExtrapolator,
TrackStateProvider, TrackInterpolator)
from PyConf import configurable
-from PyConf.packing import persisted_location
import Functors as F
@@ -72,8 +69,8 @@ def make_good_ttracks(global_reco=reconstruction,
# this is for compatibility with reco from file and tests
# get all the tracks and filter. In the case there are no T-tracks in the container, there will be no output.
# one may also need to bind global_reco to hlt2_global_reco.reconstruction for test compatibility.
- ttracks = global_reco(
- )["UnfittedTtracks"] # TODO: can a check be added for fitted tracks? Check that there are Ttracks?
+ # TODO: can a check be added for fitted tracks? Check that there are Ttracks?
+ ttracks = reco['Ttracks' if 'Ttracks' in reco.keys() else 'Tracks']
filtered_and_reduced_tracks = make_filtered_tracks(
ttracks, filter_code)
@@ -96,27 +93,11 @@ def make_good_ttracks(global_reco=reconstruction,
return fitted_filtered_tracks["Best"]
-# Calo PIDs from T tracks
-def make_ttrack_calo_pids(ttracks, pvs):
- tracks_v3, trackrels = convert_tracks_to_v3_from_v1(
- ttracks["v1"], track_types=["Ttrack"])
- calo_pids = make_calo(
- tracks_v3,
- pvs,
- trackrels=trackrels,
- chargedpid_types={
- "calo": ["Ttrack"],
- "brem": [None]
- })
- return calo_pids
-
-
# RICH PIDs from T tracks
def make_ttrack_rich_pids(ttracks):
- richRecConfs = make_all_rich_pids(
- ttracks, default_rich_reco_options(), track_types=["Seed"])
- rich_pids = make_merged_rich_pids(richRecConfs)
- return rich_pids
+ return make_rich_pids(
+ 'Ttrack', ttracks['v1'], default_rich_reco_options(),
+ location=None)['RichPIDs']
# All the T track reco
@@ -124,24 +105,18 @@ def make_ttrack_rich_pids(ttracks):
def make_ttrack_reco(global_reco, make_ttracks, skipRich=False,
skipCalo=False):
good_ttracks = make_ttracks()
- if not skipRich:
- with persisted_location.bind(force=False):
- rich_pids = make_ttrack_rich_pids(good_ttracks)
- else:
- rich_pids = None
- if not skipCalo:
- with persisted_location.bind(force=False):
- pvs = global_reco["PVs"]
- calo_pids = make_ttrack_calo_pids(good_ttracks, pvs)
- else:
- calo_pids = None
+
+ rich_pids = make_ttrack_rich_pids(good_ttracks) if not skipRich else None
+ # TODO calo reco was configured wrong (to be added appropriately)
+ calo_pids = None
# TODO: add muon PIDs when compatible with T tracks
+ muon_pids = None
return {
"good_ttracks": good_ttracks,
"rich_pids": rich_pids,
"calo_pids": calo_pids,
- # TODO: add muon PIDs when compatible with T tracks
+ "muon_pids": muon_pids,
}
@@ -163,7 +138,7 @@ def make_ttrack_protoparticles(track_type='Ttrack',
tracks=ttrack_reco["good_ttracks"]['v1'],
rich_pids=ttrack_reco["rich_pids"],
calo_pids=ttrack_reco["calo_pids"],
- muon_pids=None, # TODO: add muon PIDs when compatible with T tracks
+ muon_pids=ttrack_reco["muon_pids"],
track_types=["Ttrack"])
--
GitLab