split track containers and persistency (versioning)

Introduces split-per-track-type persistency. Track, PID objects and ProtoParticles are all in split containers. Main reason to have more control over persistency.

Also fixes upstream proto/track persistency.

Tested with Hlt/Hlt2Conf/options/hlt2_2or3bodytopo_realtime.py and subsequent sprucing unpack step.

One of the things it does it cleans up and removes the use of merged containers that copy tracks/muonpid/richpid objects, but uses SharedObjectsContainer such that pointers to the individual per-track-type objects are used for merged containers.

Currently the tracks are copied to their persisted location, but this should be SharedObjectsContainers which should be persisted, this way the actual underlying object persistency is handled anonymously (needs !2281 (merged) and SharedObjectsContainers to be persisted)

It aims to fix #392 (closed). @sesen @graven @decianm

Also Ttrack reco should be cleaned up @isanders Maybe fixed in other MRs, see also #588

Also adds fixes for calohypo persistency in charged protoparticles (it started with this, this branch and got out of hand, as it needs these fixes to work).

In addition, uses work of @sbelin from Rec!3509 (closed), !2141 (closed) and DaVinci!897 (closed) (merged into the relevant branches)

Goes with LHCb!4127 (merged), Rec!3423 (merged), Alignment!394 (merged), DaVinci!926 (merged), MooreAnalysis!126 (merged), MooreOnline!268 (merged), lhcb-datapkg/PRConfig!327 (merged)

needs LHCb!4191 (merged) (cherry picked and committed to LHCb!4127 (merged))

Hlt/RecoConf/python/RecoConf/hlt2_global_reco.py

@@ -15,7 +15,7 @@ The 'global' reconstruction is that which produces the final output of the
 """
 from .hlt1_tracking import make_reco_pvs, make_FTRawBankDecoder_clusters
 from .hlt2_tracking import make_hlt2_tracks, make_hlt2_tracks_without_UT, convert_tracks_to_v3_from_v1
-from .rich_reconstruction import make_all_rich_pids, default_rich_reco_options, make_merged_rich_pids
+from .rich_reconstruction import make_rich_pids, make_all_rich_pids, default_rich_reco_options, make_merged_rich_pids
 from .calorimeter_reconstruction import make_calo
 from .muon_reconstruction import make_all_muon_pids, make_merged_muon_pids, make_conv_muon_pids
 
@@ -149,6 +149,23 @@ def make_light_reconstruction(usePatPVFuture=False,
         outputs={'outputLocation': persisted_location('Tracks')})
 
     best_tracks = {"v1": merge_tracks.outputLocation}
+
+    tracks_v3, trackrels = convert_tracks_to_v3_from_v1(
+        best_tracks["v1"],
+        track_types=rich_and_charged_proto_track_types + ["Ttrack"])
+
+    # split/unmerged track conversion
+    # TODO should be for all track types like this (or rather within the convert function)
+    upstream_tracks = hlt2_tracks.get(
+        'BestUpstream', {"v1": TracksEmptyProducer(name="FakeUpstreamTracks")})
+
+    upstream_v3, upstream_rels = convert_tracks_to_v3_from_v1(
+        upstream_tracks["v1"],
+        track_types=["Upstream"],
+    )
+    tracks_v3.update(upstream_v3)
+    trackrels.update(upstream_rels)
+
     # PVs
     pvs = make_reco_pvs(velo_tracks, persisted_location('PVs'))
 
@@ -156,18 +173,20 @@ def make_light_reconstruction(usePatPVFuture=False,
     if skipRich:
         rich_pids = None
     else:
-        richRecConfs = make_all_rich_pids(best_tracks,
-                                          default_rich_reco_options())
+        rich_options = default_rich_reco_options()
+        richRecConfs = make_all_rich_pids(
+            best_tracks,
+            rich_options,
+            make_pids=make_rich_pids,
+            track_types=rich_and_charged_proto_track_types)
+        # specifically for optional upstream
+        richRecConfs["Upstream"] = make_rich_pids(
+            "Upstream", upstream_tracks["v1"], rich_options)
         # Merge them for now to be backwards compatible.
         rich_pids = make_merged_rich_pids(richRecConfs,
                                           rich_and_charged_proto_track_types)
 
     # Calo, enable v3 track versions
-    tracks_v3, trackrels = convert_tracks_to_v3_from_v1(
-        best_tracks["v1"],
-        track_types=rich_and_charged_proto_track_types +
-        ["Upstream", "Ttrack"])
-
     calo_pids = make_calo(
         tracks_v3, pvs["v3"], trackrels=trackrels) if not skipCalo else None
 
@@ -208,14 +227,13 @@ def make_light_reconstruction(usePatPVFuture=False,
 
     # optional/split upstream protoparticles
     # TODO should be for all track types like this
-    upstream_tracks = hlt2_tracks.get(
-        'BestUpstream', {"v1": TracksEmptyProducer(name="FakeUpstreamTracks")})
     upstream_protos = make_charged_protoparticles_from(
         tracks=upstream_tracks,
-        rich_pids=richRecConfs["Upstream"]["RichPIDs"] if rich_pids else None,
+        rich_pids=richRecConfs['Upstream']['RichPIDs']
+        if not skipRich else None,
         calo_pids=calo_pids,
         muon_pids=None,
-        track_types=["Upstream"],
+        track_types=['Upstream'],
         location=persisted_location('UpstreamProtos'))
 
     # neutral protoparticles