Add explicit data dependency declarations for the targets of MC hit/particle linkers when reading linkers from file

Hlt/RecoConf/python/RecoConf/data_from_file.py

@@ -50,8 +50,10 @@ from PyConf.Tools import (
     ChargedProtoParticleAddCaloHypos, ChargedProtoParticleAddCombineDLLs)
 
 
-def _unpack_configurable(packed_type):
-    return {
+def _unpacker(forced_type, packed_loc, unpacked_loc, **kwargs):
+    """Return unpacker that reads from file and unpacks to a forced output location."""
+    assert 'name' not in kwargs
+    _unpack_configurable = {
         'LHCb::PackedRecVertices': UnpackRecVertex,
         'LHCb::PackedCaloHypos': UnpackCaloHypo,
         'LHCb::PackedMuonPID': UnpackMuonPIDs,
@@ -64,60 +66,9 @@ def _unpack_configurable(packed_type):
         'LHCb::PackedMCCaloHits': UnpackMCCaloHit,
         'LHCb::PackedMCRichHits': MCRichHitUnpacker,
         'LHCb::PackedMCRichDigitSummarys': MCRichDigitSummaryUnpacker,
-    }[packed_type]
-
-
-def _packed_reco_from_file():
-    # for each key, specify both the location in the TES and the type that is
-    # expected to be made available by reading it from a file
-    return {
-        'PackedPVs': ('LHCb::PackedRecVertices', '/Event/pRec/Vertex/Primary'),
-        'PackedCaloElectrons': ('LHCb::PackedCaloHypos',
-                                '/Event/pRec/Calo/Electrons'),
-        'PackedCaloPhotons': ('LHCb::PackedCaloHypos',
-                              '/Event/pRec/Calo/Photons'),
-        'PackedCaloMergedPi0s': ('LHCb::PackedCaloHypos',
-                                 '/Event/pRec/Calo/MergedPi0s'),
-        'PackedCaloSplitPhotons': ('LHCb::PackedCaloHypos',
-                                   '/Event/pRec/Calo/SplitPhotons'),
-        'PackedMuonPIDs': ('LHCb::PackedMuonPID', '/Event/pRec/Muon/MuonPID'),
-        'PackedRichPIDs': ('LHCb::PackedRichPID', '/Event/pRec/Rich/PIDs'),
-        'PackedTracks': ('LHCb::PackedTracks', '/Event/pRec/Track/Best'),
-        'PackedNeutralProtos': ('LHCb::PackedProtoParticles',
-                                '/Event/pRec/ProtoP/Neutrals'),
-        'PackedChargedProtos': ('LHCb::PackedProtoParticles',
-                                '/Event/pRec/ProtoP/Charged')
     }
-
-
-def _packed_mc_from_file():
-    # for each key, specify both the location in the TES and the type that is
-    # expected to be made available by reading it from a file
-    return {
-        'PackedMCParticles': ('LHCb::PackedMCParticles',
-                              '/Event/pSim/MCParticles'),
-        'PackedMCVertices': ('LHCb::PackedMCVertices',
-                             '/Event/pSim/MCVertices'),
-        'PackedMCVPHits': ('LHCb::PackedMCHit', '/Event/pSim/VP/Hits'),
-        'PackedMCUTHits': ('LHCb::PackedMCHit', '/Event/pSim/UT/Hits'),
-        'PackedMCFTHits': ('LHCb::PackedMCHit', '/Event/pSim/FT/Hits'),
-        'PackedMCRichHits': ('LHCb::PackedMCRichHits',
-                             '/Event/pSim/Rich/Hits'),
-        'PackedMCEcalHits': ('LHCb::PackedMCCaloHits',
-                             '/Event/pSim/Ecal/Hits'),
-        'PackedMCHcalHits': ('LHCb::PackedMCCaloHits',
-                             '/Event/pSim/Hcal/Hits'),
-        'PackedMCMuonHits': ('LHCb::PackedMCHit', '/Event/pSim/Muon/Hits'),
-        'PackedMCRichDigitSummaries': ('LHCb::PackedMCRichDigitSummarys',
-                                       '/Event/pSim/Rich/DigitSummaries'),
-    }
-
-
-def _unpacker(forced_type, packed_loc, unpacked_loc, **kwargs):
-    """Return unpacker that reads from file and unpacks to a forced output location."""
-    assert 'name' not in kwargs
     return Algorithm(
-        _unpack_configurable(forced_type),
+        _unpack_configurable[forced_type],
         name=f"Unpack_{unpacked_loc.replace('/','_')}",
         outputs={'OutputName': force_location(unpacked_loc)},
         InputName=make_data_with_FetchDataFromFile(
@@ -125,33 +76,43 @@ def _unpacker(forced_type, packed_loc, unpacked_loc, **kwargs):
         **kwargs)
 
 
-def _reco_unpacker(key, **kwargs):
-    """Return unpacker that reads from file and unpacks to a forced output location."""
-    packed_type, packed_location = _packed_reco_from_file()[key]
-    # If the structure is not like this, pointers point to to the wrong place...
-    # The SmartRefs held by the unpacked MC objects only work if we unpack to these specific locations
-    unpacked_location = packed_location.replace('/pRec', '/Rec').replace(
-        '/pHLT2', '/HLT2')
-    return _unpacker(packed_type, packed_location, unpacked_location, **kwargs)
-
-
-def _mc_unpacker(key, **kwargs):
-    """Return unpacker that reads from file and unpacks to a forced output location."""
-    packed_type, packed_location = _packed_mc_from_file()[key]
-    # If the structure is not like this, pointers point to to the wrong place...
-    # The SmartRefs held by the unpacked MC objects only work if we unpack to these specific locations
-    unpacked_location = packed_location.replace('/pSim', '/MC').replace(
-        '/MC/MC', '/MC/')
-    return _unpacker(packed_type, packed_location, unpacked_location, **kwargs)
-
-
 def reco_unpackers():
-    muonPIDs = _reco_unpacker('PackedMuonPIDs')
-    richPIDs = _reco_unpacker('PackedRichPIDs', OutputLevel=ERROR)
-    electrons = _reco_unpacker('PackedCaloElectrons')
-    photons = _reco_unpacker('PackedCaloPhotons')
+    # for each key, specify both the location in the TES and the type that is
+    # expected to be made available by reading it from a file
+    _packed_reco_from_file = {
+        'PVs': ('LHCb::PackedRecVertices', '/Event/pRec/Vertex/Primary'),
+        'CaloElectrons': ('LHCb::PackedCaloHypos',
+                          '/Event/pRec/Calo/Electrons'),
+        'CaloPhotons': ('LHCb::PackedCaloHypos', '/Event/pRec/Calo/Photons'),
+        'CaloMergedPi0s': ('LHCb::PackedCaloHypos',
+                           '/Event/pRec/Calo/MergedPi0s'),
+        'CaloSplitPhotons': ('LHCb::PackedCaloHypos',
+                             '/Event/pRec/Calo/SplitPhotons'),
+        'MuonPIDs': ('LHCb::PackedMuonPID', '/Event/pRec/Muon/MuonPID'),
+        'RichPIDs': ('LHCb::PackedRichPID', '/Event/pRec/Rich/PIDs'),
+        'Tracks': ('LHCb::PackedTracks', '/Event/pRec/Track/Best'),
+        'NeutralProtos': ('LHCb::PackedProtoParticles',
+                          '/Event/pRec/ProtoP/Neutrals'),
+        'ChargedProtos': ('LHCb::PackedProtoParticles',
+                          '/Event/pRec/ProtoP/Charged')
+    }
+
+    def _reco_unpacker(key, **kwargs):
+        """Return unpacker that reads from file and unpacks to a forced output location."""
+        packed_type, packed_location = _packed_reco_from_file[key]
+        # If the structure is not like this, pointers point to to the wrong place...
+        # The SmartRefs held by the unpacked MC objects only work if we unpack to these specific locations
+        unpacked_location = packed_location.replace('/pRec', '/Rec').replace(
+            '/pHLT2', '/HLT2')
+        return _unpacker(packed_type, packed_location, unpacked_location,
+                         **kwargs)
+
+    muonPIDs = _reco_unpacker('MuonPIDs')
+    richPIDs = _reco_unpacker('RichPIDs', OutputLevel=ERROR)
+    electrons = _reco_unpacker('CaloElectrons')
+    photons = _reco_unpacker('CaloPhotons')
     charged_protos = _reco_unpacker(
-        'PackedChargedProtos',
+        'ChargedProtos',
         AddInfo=[
             ChargedProtoParticleAddRichInfo(InputRichPIDLocation=richPIDs),
             ChargedProtoParticleAddMuonInfo(InputMuonPIDLocation=muonPIDs),
@@ -164,53 +125,65 @@ def reco_unpackers():
         ])
 
     # Ordered so that dependents are unpacked first
-    d = collections.OrderedDict([
-        ('PVs', _reco_unpacker('PackedPVs')),
+    return collections.OrderedDict([
+        ('PVs', _reco_unpacker('PVs')),
         ('CaloElectrons', electrons),
         ('CaloPhotons', photons),
-        ('CaloMergedPi0s', _reco_unpacker('PackedCaloMergedPi0s')),
-        ('CaloSplitPhotons', _reco_unpacker('PackedCaloSplitPhotons')),
+        ('CaloMergedPi0s', _reco_unpacker('CaloMergedPi0s')),
+        ('CaloSplitPhotons', _reco_unpacker('CaloSplitPhotons')),
         ('MuonPIDs', muonPIDs),
         ('RichPIDs', richPIDs),
-        ('Tracks', _reco_unpacker('PackedTracks')),
-        ('NeutralProtos', _reco_unpacker('PackedNeutralProtos')),
+        ('Tracks', _reco_unpacker('Tracks')),
+        ('NeutralProtos', _reco_unpacker('NeutralProtos')),
         ('ChargedProtos', charged_protos),
     ])
 
-    # Make sure we have consistent names, and that we're unpacking everything
-    # we load from the file
-    assert set(
-        ['Packed' + k for k in d.keys()]) == _packed_reco_from_file().keys()
 
-    return d
+def mc_unpackers():
+    # for each key, specify both the location in the TES and the type that is
+    # expected to be made available by reading it from a file
+    _packed_mc_from_file = {
+        'MCParticles': ('LHCb::PackedMCParticles', '/Event/pSim/MCParticles'),
+        'MCVertices': ('LHCb::PackedMCVertices', '/Event/pSim/MCVertices'),
+        'MCVPHits': ('LHCb::PackedMCHit', '/Event/pSim/VP/Hits'),
+        'MCUTHits': ('LHCb::PackedMCHit', '/Event/pSim/UT/Hits'),
+        'MCFTHits': ('LHCb::PackedMCHit', '/Event/pSim/FT/Hits'),
+        'MCRichHits': ('LHCb::PackedMCRichHits', '/Event/pSim/Rich/Hits'),
+        'MCEcalHits': ('LHCb::PackedMCCaloHits', '/Event/pSim/Ecal/Hits'),
+        'MCHcalHits': ('LHCb::PackedMCCaloHits', '/Event/pSim/Hcal/Hits'),
+        'MCMuonHits': ('LHCb::PackedMCHit', '/Event/pSim/Muon/Hits'),
+        'MCRichDigitSummaries': ('LHCb::PackedMCRichDigitSummarys',
+                                 '/Event/pSim/Rich/DigitSummaries'),
+    }
 
+    def _mc_unpacker(key, **kwargs):
+        """Return unpacker that reads from file and unpacks to a forced output location."""
+        packed_type, packed_location = _packed_mc_from_file[key]
+        # If the structure is not like this, pointers point to to the wrong place...
+        # The SmartRefs held by the unpacked MC objects only work if we unpack to these specific locations
+        unpacked_location = packed_location.replace('/pSim', '/MC').replace(
+            '/MC/MC', '/MC/')
+        return _unpacker(packed_type, packed_location, unpacked_location,
+                         **kwargs)
 
-def mc_unpackers():
     # Ordered so that dependents are unpacked first
     # Make sure that MC particles and MC vertices are unpacked together,
     # see https://gitlab.cern.ch/lhcb/LHCb/issues/57 for details.
-    mc_vertices = _mc_unpacker('PackedMCVertices')
-    d = collections.OrderedDict([
-        ('MCRichDigitSummaries', _mc_unpacker('PackedMCRichDigitSummaries')),
-        ('MCParticles',
-         _mc_unpacker('PackedMCParticles', ExtraInputs=[mc_vertices])),
+    mc_vertices = _mc_unpacker('MCVertices')
+    return collections.OrderedDict([
+        ('MCRichDigitSummaries', _mc_unpacker('MCRichDigitSummaries')),
+        ('MCParticles', _mc_unpacker('MCParticles',
+                                     ExtraInputs=[mc_vertices])),
         ('MCVertices', mc_vertices),
-        ('MCVPHits', _mc_unpacker('PackedMCVPHits')),
-        ('MCUTHits', _mc_unpacker('PackedMCUTHits')),
-        ('MCFTHits', _mc_unpacker('PackedMCFTHits')),
-        ('MCRichHits', _mc_unpacker('PackedMCRichHits')),
-        ('MCEcalHits', _mc_unpacker('PackedMCEcalHits')),
-        ('MCHcalHits', _mc_unpacker('PackedMCHcalHits')),
-        ('MCMuonHits', _mc_unpacker('PackedMCMuonHits')),
+        ('MCVPHits', _mc_unpacker('MCVPHits')),
+        ('MCUTHits', _mc_unpacker('MCUTHits')),
+        ('MCFTHits', _mc_unpacker('MCFTHits')),
+        ('MCRichHits', _mc_unpacker('MCRichHits')),
+        ('MCEcalHits', _mc_unpacker('MCEcalHits')),
+        ('MCHcalHits', _mc_unpacker('MCHcalHits')),
+        ('MCMuonHits', _mc_unpacker('MCMuonHits')),
     ])
 
-    # Make sure we have consistent names, and that we're unpacking everything
-    # we load from the file
-    assert set(['Packed' + k for k in d.keys()]) - set(
-        _packed_mc_from_file().keys()) == set()
-
-    return d
-
 
 def boole_links_digits_mcparticles():
     """Return a dict of locations for MC linker tables (to mcparticles) created by Boole."""
@@ -225,9 +198,10 @@ def boole_links_digits_mcparticles():
         "UTDigits": "/Event/Link/Raw/UT/TightDigits",
         "VPDigits": "/Event/Link/Raw/VP/Digits",
     }
+    mc_particles = mc_unpackers()['MCParticles']
     return {
         key: make_data_with_FetchDataFromFile(
-            loc, force_type='LHCb::LinksByKey')
+            loc, force_type='LHCb::LinksByKey', ExtraInputs=[mc_particles])
         for key, loc in locations.items()
     }
 
@@ -237,16 +211,19 @@ def boole_links_digits_mchits():
 
     These locations are only propagated out of Boole for eXtendend DIGI and DST types.
     """
-    locations = {
-        "FTLiteClusters": "/Event/Link/Raw/FT/LiteClusters2MCHits",
-        "UTClusters": "/Event/Link/Raw/UT/Clusters2MCHits",
-        "UTDigits": "/Event/Link/Raw/UT/TightDigits2MCHits",
-        "VPDigits": "/Event/Link/Raw/VP/Digits2MCHits",
+    locations_and_dep = {
+        "FTLiteClusters": ("/Event/Link/Raw/FT/LiteClusters2MCHits",
+                           'MCFTHits'),
+        "UTClusters": ("/Event/Link/Raw/UT/Clusters2MCHits", 'MCUTHits'),
+        "UTDigits": ("/Event/Link/Raw/UT/TightDigits2MCHits", 'MCUTHits'),
+        "VPDigits": ("/Event/Link/Raw/VP/Digits2MCHits", 'MCVPHits')
     }
     return {
         key: make_data_with_FetchDataFromFile(
-            loc, force_type='LHCb::LinksByKey')
-        for key, loc in locations.items()
+            loc,
+            force_type='LHCb::LinksByKey',
+            ExtraInputs=[mc_unpackers()[dep]])
+        for key, (loc, dep) in locations_and_dep.items()
     }