Merge branch 'sevda-unpacking' into 'master'

use simplified unpacking

See merge request !747

DaVinciExamples/example_data/Gauss_12143001_xgen.yaml

@@ -9,4 +9,4 @@ histo_file: DV_histo_xgen.root
 ntuple_file: DV_tuple_xgen.root
 input_raw_format: 0.5
 process: Hlt2
-unpack_only_mc: true
+enable_unpack: false

DaVinciExamples/example_data/test_read_xgen.yaml

@@ -5,5 +5,4 @@ input_type: ROOT
 simulation: true
 conddb_tag: sim-20171127-vc-md100
 dddb_tag: dddb-20171126
-# Required to read XGEN files
-unpack_only_mc: true
+enable_unpack: false
\ No newline at end of file

DaVinciExamples/python/DaVinciExamples/tupling/AllFunctors.py

@@ -17,8 +17,8 @@ __date__ = "2021-11-23"
 import Functors as F
 from FunTuple import FunctorCollection
 from FunTuple import FunTuple_Particles as Funtuple
-from PyConf.components import force_location
 from DaVinci.reco_objects import make_pvs
+from PyConf.reading import get_particles
 from DaVinci.algorithms import add_filter, get_decreports, get_odin
 from DecayTreeFitter import DTFAlg
 from DaVinci.truth_matching import configured_MCTruthAndBkgCatAlg
@@ -30,7 +30,7 @@ from DaVinci import Options, make_config
 # Definition of strucing line
 #
 bd2dsk_line = "SpruceB2OC_BdToDsmK_DsmToHHH_FEST_Line"
-bd2dsk_data = force_location(f"/Event/Spruce/{bd2dsk_line}/Particles")
+bd2dsk_data = get_particles(f"/Event/Spruce/{bd2dsk_line}/Particles")
 
 _basic = 'basic'
 _composite = 'composite'

DaVinciExamples/python/DaVinciExamples/tupling/DTF_filtered.py

@@ -22,6 +22,7 @@ from DaVinci.algorithms import add_filter  #, filter_on
 from DecayTreeFitter import DTFAlg
 from FunTuple import FunctorCollection as FC
 from FunTuple import FunTuple_Particles as Funtuple
+from PyConf.reading import get_particles
 
 
 def main(options: Options):
@@ -34,11 +35,11 @@ def main(options: Options):
     #Get filtered particles (Note decay_descriptor is optional, if specified only B0 decays will be selected for processing)
     spruce_line = "SpruceB2OC_BdToDsmK_DsmToHHH_FEST_Line"
     # REPLACING TEMPORARY THE INPUT DATA
-    from PyConf.components import force_location
     #data_filtered = filter_on(
-    #f"/Event/Spruce/{spruce_line}/Particles",
-    #decay_descriptor=fields['B0'])
-    data_filtered = force_location(f"/Event/Spruce/{spruce_line}/Particles")
+    #    f"/Event/Spruce/{spruce_line}/Particles",
+    #    options.process,
+    #    decay_descriptor=fields['B0'])
+    data_filtered = get_particles(f"/Event/Spruce/{spruce_line}/Particles")
 
     # DecayTreeFitter Algorithm.
     DTF = DTFAlg(

DaVinciExamples/python/DaVinciExamples/tupling/example-tupling-mc-reconstructible-reconstructed.py

@@ -14,20 +14,36 @@ and MCReconstructed.
 """
 
 from FunTuple import FunTuple_MCParticles as MCFuntuple
-from PyConf.components import force_location
 from DaVinciMCTools import MCReconstructible, MCReconstructed
 from DaVinci import Options, make_config
 from RecoConf.data_from_file import make_mc_track_info
 from FunTuple.functorcollections import MCReconstructed_Collection, MCReconstructible_Collection
+from PyConf.reading import get_mc_particles, get_pp2mcp_relations
+from DaVinci.reco_objects import make_charged_protoparticles, make_neutral_protoparticles
 
 
 def main(options: Options):
+
     # Input
-    MC_data = force_location("/Event/HLT2/MC/Particles")
+    MC_data = get_mc_particles("/Event/HLT2/MC/Particles")
+    # PP2MCP relations need MC particles and ProtoParticles
+    # Since MC is already unpacked, only unpack protos and relations
+    extra_inputs = []
+    extra_inputs += [make_charged_protoparticles(options.process)]
+    extra_inputs += [make_neutral_protoparticles(options.process)]
+
+    relations = [
+        get_pp2mcp_relations(
+            loc, process=options.process, extra_inputs=extra_inputs)
+        for loc in [
+            "/Event/HLT2/Relations/ChargedPP2MCP",
+            "/Event/HLT2/Relations/NeutralPP2MCP"
+        ]
+    ]
 
     #Get variables related to reconstructible information.
     mcrtible = MCReconstructible(
-        process='Hlt2', mc_track_info=make_mc_track_info())
+        process=options.process, mc_track_info=make_mc_track_info())
     #The option extra_info is set to False by default and can be set to True to get more information
     vars_rtible = MCReconstructible_Collection(mcrtible, extra_info=True)
     print('Reconstructible functors:', vars_rtible.functor_dict.keys())
@@ -44,7 +60,10 @@ def main(options: Options):
     #   tracks to the mc particle are used (tupling arrays).
     #   Here we set it to false for testing purposes.
     mcrted_all = MCReconstructed(
-        MC_data, use_best_mcmatch=False, process='Hlt2')
+        MC_data,
+        use_best_mcmatch=False,
+        relations_locs=relations,
+        process=options.process)
     #The option extra_info below is set to False by default in the functor collection.
     vars_rted = MCReconstructed_Collection(mcrted_all, extra_info=False)
     #Note:

DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_array_taggers.py

@@ -20,12 +20,10 @@ Then the MAP_INPUT_ARRAY functor takes in input this relation map and for each
 entry stores the output of an external functor (i.e F.P, F.PT) in a vector.
 """
 
-from PyConf.components import force_location
-from PyConf.Algorithms import ParticleTaggerAlg, ParticleContainerMerger
-
 import Functors as F
-
+from PyConf.Algorithms import ParticleTaggerAlg, ParticleContainerMerger
 from FunTuple import FunctorCollection, FunTuple_Particles as Funtuple
+from PyConf.reading import get_particles
 
 from DaVinci import Options, make_config
 from DaVinci.algorithms import add_filter
@@ -34,7 +32,7 @@ from DaVinci.common_particles import make_long_pions
 
 def main(options: Options):
     bd2dsk_line = "SpruceB2OC_BdToDsmK_DsmToHHH_FEST_Line"
-    bd2dsk_data = force_location(f"/Event/Spruce/{bd2dsk_line}/Particles")
+    bd2dsk_data = get_particles(f"/Event/Spruce/{bd2dsk_line}/Particles")
 
     pions = make_long_pions()
 

DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_eventinfo.py

@@ -15,12 +15,12 @@ Read an HLT2 file and create an ntuple with information from RecSumarry
 import Functors as F
 from FunTuple import FunctorCollection as FC
 from FunTuple import FunTuple_Particles as Funtuple
-from PyConf.components import force_location
 from DaVinci.algorithms import add_filter
 from DaVinci import make_config, Options
 from DaVinci.algorithms import get_odin
 from FunTuple.functorcollections import EventInfo
 from DaVinci.reco_objects import get_rec_summary
+from PyConf.reading import get_particles
 
 
 def main(options: Options):
@@ -35,7 +35,7 @@ def main(options: Options):
     #get RecSummary object that holds information about nPVs, nTracks, nFTClusters
     # Note more information can be added to the RecSummary object
     # (see MRs: https://gitlab.cern.ch/lhcb/Moore/-/merge_requests/1649)
-    rec_summary = get_rec_summary(options)
+    rec_summary = get_rec_summary(options.process)
     evt_vars = FC({
         'nTracks': F.VALUE_OR(-1) @ F.NTRACKS(rec_summary),
         'nPVs': F.VALUE_OR(-1) @ F.NPVS(rec_summary),
@@ -47,7 +47,8 @@ def main(options: Options):
 
     #get particles to run over
     line_name = 'Hlt2SLB_LbToLcMuNu_LcToPKPi_Line'
-    particles = force_location(f"/Event/HLT2/{line_name}/Particles")
+    particles = get_particles(
+        f"/Event/HLT2/{line_name}/Particles", process=options.process)
     my_filter = add_filter(options, "Myfilter", f"HLT_PASS('{line_name}')")
     #define tupling algorithm
     my_tuple = Funtuple(

DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_collections.py

@@ -13,7 +13,6 @@ Read an HLT2 file and create an ntuple using pre-defined Functor collections.
 """
 
 import Functors as F
-from PyConf.components import force_location
 from DaVinciMCTools import MCReconstructible, MCReconstructed
 from FunTuple import FunctorCollection, functorcollections
 from FunTuple import FunTuple_Particles as Funtuple, FunTuple_MCParticles as MCFuntuple
@@ -24,12 +23,15 @@ from DaVinci.truth_matching import configured_MCTruthAndBkgCatAlg
 from DaVinci.algorithms import get_odin, get_decreports
 from PyConf.Algorithms import WeightedRelTableAlg
 from Gaudi.Configuration import INFO
+from PyConf.reading import get_particles, get_mc_particles
 
 
 def main(options: Options):
     line_name = 'Hlt2Charm_D0ToKmPip_Line'
-    d02kpi_data = force_location(f"/Event/HLT2/{line_name}/Particles")
-    MC_data = force_location("/Event/HLT2/MC/Particles")
+
+    d02kpi_data = get_particles(
+        f"/Event/HLT2/{line_name}/Particles", process=options.process)
+    MC_data = get_mc_particles("/Event/HLT2/MC/Particles")
 
     #Get variables related to reconstructible information.
     mcrtible = MCReconstructible(

DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_hlt2.py

@@ -14,8 +14,8 @@ Read an HLT2 file and create an ntuple with the new DaVinci configuration.
 import Functors as F
 from FunTuple import FunctorCollection as FC
 from FunTuple import FunTuple_Particles as Funtuple
-from PyConf.components import force_location
 from DaVinci.reco_objects import make_pvs
+from PyConf.reading import get_particles
 from DaVinci.algorithms import add_filter
 from DaVinci import Options, make_config
 from DaVinci.truth_matching import configured_MCTruthAndBkgCatAlg
@@ -66,7 +66,8 @@ def main(options: Options):
     }
 
     line_name = 'Hlt2Charm_D0ToKmPip_Line'
-    d02kpi_data = force_location(f"/Event/HLT2/{line_name}/Particles")
+    d02kpi_data = get_particles(
+        f"/Event/HLT2/{line_name}/Particles", process=options.process)
 
     my_filter = add_filter(options, "HDRFilter_D0Kpi",
                            f"HLT_PASS('{line_name}')")

DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_passthrough.py

@@ -14,15 +14,15 @@ objects are persisted by means of a pass through line.
 """
 from FunTuple import FunTuple_Particles as Funtuple
 from FunTuple.functorcollections import Kinematics
-from PyConf.components import force_location
 from DaVinci import Options, make_config
 from DaVinci.algorithms import add_filter
+from PyConf.reading import get_particles
 
 
 def main(options: Options):
     bs2jpsiphi_line = "Hlt2BsToJpsiPhi_JPsi2MuMu_PhiToKK_Line"
-    bs2jpsiphi_data = force_location(
-        f"/Event/HLT2/{bs2jpsiphi_line}/Particles")
+    bs2jpsiphi_data = get_particles(
+        f"/Event/HLT2/{bs2jpsiphi_line}/Particles", process=options.process)
     fields = {
         'Bs': "[B_s0 -> (J/psi(1S) -> mu+ mu-) (phi(1020) -> K+ K-)]CC",
         'Jpsi': "[B_s0 -> ^(J/psi(1S) -> mu+ mu-) (phi(1020) -> K+ K-)]CC",

DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_spruce.py

@@ -14,16 +14,16 @@ Read the output of an Sprucing job with the new DaVinci configuration.
 from FunTuple import FunctorCollection
 from FunTuple import FunTuple_Particles as Funtuple
 from FunTuple.functorcollections import Kinematics
-from PyConf.components import force_location
 from DaVinci.algorithms import add_filter
 from DaVinci import Options, make_config
+from PyConf.reading import get_particles
 
 
 def main(options: Options):
     line_B0DsK = 'SpruceB2OC_BdToDsmK_DsmToHHH_FEST_Line'
     line_B0Dspi = 'SpruceB2OC_BdToDsmPi_DsmToKpKmPim_Line'
-    bd2dsk_line = force_location(f"/Event/Spruce/{line_B0DsK}/Particles")
-    bd2dspi_line = force_location(f"/Event/Spruce/{line_B0Dspi}/Particles")
+    bd2dsk_line = get_particles(f"/Event/Spruce/{line_B0DsK}/Particles")
+    bd2dspi_line = get_particles(f"/Event/Spruce/{line_B0Dspi}/Particles")
 
     fields_dsk = {
         'B0': "[B0 -> D_s- K+]CC",

DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_spruce_mc.py

@@ -15,7 +15,7 @@ from FunTuple.functorcollections import Kinematics
 from DaVinci import Options, make_config
 from DaVinci.truth_matching import configured_MCTruthAndBkgCatAlg
 from DaVinci.algorithms import add_filter
-from PyConf.components import force_location
+from PyConf.reading import get_particles
 
 
 def main(options: Options):
@@ -35,7 +35,7 @@ def main(options: Options):
     }
 
     line_name = 'Spruce_Test_line'
-    B_data = force_location(f"/Event/Spruce/{line_name}/Particles")
+    B_data = get_particles(f"/Event/Spruce/{line_name}/Particles")
 
     my_filter = add_filter(options, "HDRFilter_B", f"HLT_PASS('{line_name}')")
 

DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_xgen.py

@@ -13,9 +13,9 @@ Read and process a .xgen file with the new DaVinci configuration.
 """
 from FunTuple import FunctorCollection, FunTuple_MCParticles as FuntupleMC
 from PyConf.Algorithms import PrintMCTree
-from PyConf.dataflow import force_location
 import Functors as F
 from DaVinci import Options, make_config
+from PyConf.reading import get_mc_particles
 
 
 def main(options: Options):
@@ -49,11 +49,7 @@ def main(options: Options):
         'Bu': variables_B,
     }
 
-    # Since the "/Event/MC/Particles" location is already declared as output of the UnpackMCParticle algorithm,
-    # it's not possible to instantiate a new Gaudi__Hive__FetchDataFromFile object with the usual
-    # 'make_data_with_FetchDataFromFile' method with the same output location.
-    # It's there sufficient forcing FunTuple to use the same location as input.
-    bu2jpsik_line = force_location("/Event/MC/Particles")
+    bu2jpsik_line = get_mc_particles("/Event/MC/Particles")
 
     printMC = PrintMCTree(
         MCParticles=bu2jpsik_line, ParticleNames=["B+", "B-"], OutputLevel=4)

DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_weightedrelation_trackvariables.py

@@ -19,35 +19,28 @@ To run the example: $DVPATH/run lbexec option_davinci_tupling_weightedrelation_t
 """
 
 import Functors as F
-from PyConf.components import force_location
 from PyConf.Algorithms import WeightedRelTableAlg
 from FunTuple import FunctorCollection, FunTuple_Particles as Funtuple
 from FunTuple.functorcollections import TrackIsolation
 from DaVinci.reco_objects import make_pvs
-from DaVinci.algorithms import add_filter, unpack_locations
+from PyConf.reading import get_particles
+from DaVinci.algorithms import add_filter
 from DaVinci import Options, make_config
 
-b2jpsik_data = force_location("/Event/HLT2/Hlt2B2JpsiKLine/Particles")
 
-branches = {
-    'B': "[B+ -> (J/psi(1S) -> mu+ mu- ) K+]CC",
-    'Jpsi': "[B+ -> ^(J/psi(1S) -> mu+ mu- ) K+]CC",
-    'Kp': "[B+ -> (J/psi(1S) -> mu+ mu- ) ^K+]CC"
-}
+def main(options: Options):
 
+    branches = {
+        'B': "[B+ -> (J/psi(1S) -> mu+ mu- ) K+]CC",
+        'Jpsi': "[B+ -> ^(J/psi(1S) -> mu+ mu- ) K+]CC",
+        'Kp': "[B+ -> (J/psi(1S) -> mu+ mu- ) ^K+]CC"
+    }
 
-def main(options: Options):
-    # Hack used to unpack the tagged data for now
-    # follows discussion on Mattermost channel DPA WP3 Offline Analysis
-    # [https://mattermost.web.cern.ch/lhcb/pl/p6tbr8inetf99jgin5ihce63ic]
-    unpackers = unpack_locations(options, False)
-    tagged_data = None
-    for alg in unpackers:
-        if "OutputName" in alg.outputs.keys():
-            if (alg.OutputName.location ==
-                    "/Event/HLT2/Hlt2B2JpsiKLine/LongTaggingParticles/Particles"
-                ):
-                tagged_data = alg.OutputName
+    b2jpsik_data = get_particles(
+        "/Event/HLT2/Hlt2B2JpsiKLine/Particles", process=options.process)
+    tagged_data = get_particles(
+        "/Event/HLT2/Hlt2B2JpsiKLine/LongTaggingParticles/Particles",
+        process=options.process)
 
     pvs = make_pvs(process=options.process)
 

DaVinciExamples/python/DaVinciExamples/tupling/option_trigger_decisions.py

@@ -14,9 +14,9 @@ Read an HLT2 file and create an ntuple with the new DaVinci configuration, acces
 import Functors as F
 from FunTuple import FunctorCollection
 from FunTuple import FunTuple_Particles as Funtuple
-from PyConf.components import force_location
 from DaVinci import Options, make_config
 from DaVinci.algorithms import get_decreports
+from PyConf.reading import get_particles
 from FunTuple.functorcollections import SelectionInfo
 
 
@@ -27,8 +27,9 @@ def main(options: Options):
 
     variables = {"B0": B0_variables}
 
-    b02dpi_data = force_location(
-        "/Event/HLT2/Hlt2B2OC_BdToDmPi_DmToPimPimKp_Line/Particles")
+    b02dpi_data = get_particles(
+        "/Event/HLT2/Hlt2B2OC_BdToDmPi_DmToPimPimKp_Line/Particles",
+        process=options.process)
 
     hlt1_dec = get_decreports("Hlt1", options)
 

DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_trigger_decisions.qmt

@@ -24,7 +24,7 @@
   <argument name="validator"><text>
 
 from DaVinciTests.QMTest.DaVinciExclusions import remove_known_warnings
-countErrorLines({"FATAL": 0, "WARNING": 12, "ERROR": 0},
+countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0},
                 stdout=remove_known_warnings(stdout))
 
 import sys, os, glob

DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_tupling_eventinfo.qmt

@@ -53,7 +53,7 @@ if len(B_excluded_2) != 0: raise Exception('Number of stored variables is greate
 f.Close()
 print('Test successfully completed!')
 os.system(f"rm {ntuple}")
-countErrorLines({"FATAL": 0, "WARNING": 9, "ERROR": 0},
+countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0},
                 stdout=remove_known_warnings(stdout))
 
   </text></argument>

DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_tupling_mc_recinfo.qmt

@@ -23,7 +23,7 @@
   </text></argument>
   <argument name="validator"><text>
 from DaVinciTests.QMTest.DaVinciExclusions import remove_known_warnings
-countErrorLines({"FATAL": 0, "WARNING": 1, "ERROR": 0},
+countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0},
                 stdout=remove_known_warnings(stdout))
 import sys, os
 from ROOT import TFile

DaVinciExamples/tests/qmtest/tupling.qms/test_example-tupling-basic.qmt

@@ -21,6 +21,7 @@
     histo_file: DV-example-tupling-basic-his.root
     evt_max: 10
     input_raw_format: 4.3
+    enable_unpack: false
 </text></argument>
 <argument name="timeout"><integer>3600</integer></argument>
 <argument name="args"><set>

DaVinciTests/python/DaVinciTests/QMTest/DaVinciExclusions.py

@@ -53,6 +53,11 @@ remove_known_warnings = LineSkipper(regexps=[
     r"DetectorDataSvc +SUCCESS Detector description database: git:/lhcb.xml",
     r"[a-zA-Z0-9]* +WARNING TransportSvc is currently incompatible with DD4HEP. .*",
     r"[a-zA-Z0-9]* +WARNING See https://gitlab.cern.ch/lhcb/Rec/-/issues/326 for more details",
+    # backwards compatibility -- key is from manifest, not git
+    r"key 0x[0-9a-f]+ has an explicitly configured overrule -- using that",
+    # backwards compatibility -- old data
+    r"DstData raw bank  has a zero encoding key, and it is not explicitly specified for decoding *",
+    r"HltDecReports has a zero TCK, and it is not explicitly specified for decoding *",
 ])
 
 remove_known_fluctuating_counters = LineSkipper(regexps=[