From 68e96a469ed28678e724b32e6ebf7a5882feb6bf Mon Sep 17 00:00:00 2001 From: Sevda Esen <sevda.esen@cern.ch> Date: Fri, 30 Sep 2022 13:34:03 +0200 Subject: [PATCH] use simplified unpacking --- .../example_data/Gauss_12143001_xgen.yaml | 2 +- .../example_data/test_read_xgen.yaml | 3 +- .../DaVinciExamples/tupling/AllFunctors.py | 4 +- .../DaVinciExamples/tupling/DTF_filtered.py | 9 +- ...upling-mc-reconstructible-reconstructed.py | 27 +++- .../option_davinci_tupling_array_taggers.py | 8 +- .../option_davinci_tupling_eventinfo.py | 7 +- ...option_davinci_tupling_from_collections.py | 8 +- .../option_davinci_tupling_from_hlt2.py | 5 +- ...option_davinci_tupling_from_passthrough.py | 6 +- .../option_davinci_tupling_from_spruce.py | 6 +- .../option_davinci_tupling_from_spruce_mc.py | 4 +- .../option_davinci_tupling_from_xgen.py | 8 +- ...tupling_weightedrelation_trackvariables.py | 33 ++--- .../tupling/option_trigger_decisions.py | 7 +- .../test_davinci_trigger_decisions.qmt | 2 +- .../test_davinci_tupling_eventinfo.qmt | 2 +- .../test_davinci_tupling_mc_recinfo.qmt | 2 +- .../test_example-tupling-basic.qmt | 1 + .../DaVinciTests/QMTest/DaVinciExclusions.py | 5 + .../python/DaVinciTests/funtuple_array.py | 4 +- .../DaVinciTests/option_davinci_for_turbo.py | 6 +- .../python/DaVinciTests/recVertices.py | 4 +- .../options/option_davinci_funtuple_array.py | 4 +- .../tests/options/option_davinci_sprucing.py | 4 +- .../tests/qmtest/io.qms/test_read_mc_digi.qmt | 1 + .../tests/qmtest/io.qms/test_read_mc_ldst.qmt | 1 + .../tests/qmtest/io.qms/test_read_mc_mdf.qmt | 1 + .../qmtest/io.qms/test_read_mc_xdigi.qmt | 1 + .../tests/qmtest/io.qms/test_read_mc_xgen.qmt | 1 + .../qmtest/io.qms/test_read_moore_dst.qmt | 1 + .../DaVinciTutorials/tutorial0_basic_DVjob.py | 7 +- .../tutorial1_functors_specialfield.py | 5 +- .../python/DaVinciTutorials/tutorial2_LoKi.py | 5 +- .../tutorial3_ThOrfunctors.py | 5 +- .../tutorial4_trigger_eventinfo.py | 5 +- .../DaVinciTutorials/tutorial5_MCTruth.py | 5 +- .../tutorial6_DecayTreeFit.py | 5 +- .../tutorial7_multiple_sel_lines.py | 8 +- Phys/DaVinci/python/DaVinci/LbExec.py | 3 - Phys/DaVinci/python/DaVinci/algorithms.py | 115 +++++------------- Phys/DaVinci/python/DaVinci/config.py | 5 +- Phys/DaVinci/python/DaVinci/reco_objects.py | 93 ++++++-------- Phys/DaVinci/python/DaVinci/truth_matching.py | 24 +++- Phys/DaVinci/python/DaVinci/utils.py | 29 +++++ Phys/DaVinci/tests/config/test_algorithms.py | 40 +----- doc/configuration/davinci_configuration.rst | 1 - doc/tutorials/running.rst | 4 +- 48 files changed, 241 insertions(+), 295 deletions(-) create mode 100644 Phys/DaVinci/python/DaVinci/utils.py diff --git a/DaVinciExamples/example_data/Gauss_12143001_xgen.yaml b/DaVinciExamples/example_data/Gauss_12143001_xgen.yaml index 12a766bcc..33b60ed6c 100644 --- a/DaVinciExamples/example_data/Gauss_12143001_xgen.yaml +++ b/DaVinciExamples/example_data/Gauss_12143001_xgen.yaml @@ -9,4 +9,4 @@ histo_file: DV_histo_xgen.root ntuple_file: DV_tuple_xgen.root input_raw_format: 0.5 process: Hlt2 -unpack_only_mc: true +enable_unpack: false diff --git a/DaVinciExamples/example_data/test_read_xgen.yaml b/DaVinciExamples/example_data/test_read_xgen.yaml index e88643ffb..01e59bc66 100644 --- a/DaVinciExamples/example_data/test_read_xgen.yaml +++ b/DaVinciExamples/example_data/test_read_xgen.yaml @@ -5,5 +5,4 @@ input_type: ROOT simulation: true conddb_tag: sim-20171127-vc-md100 dddb_tag: dddb-20171126 -# Required to read XGEN files -unpack_only_mc: true +enable_unpack: false \ No newline at end of file diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/AllFunctors.py b/DaVinciExamples/python/DaVinciExamples/tupling/AllFunctors.py index b42df9fcf..a429ed4b9 100644 --- a/DaVinciExamples/python/DaVinciExamples/tupling/AllFunctors.py +++ b/DaVinciExamples/python/DaVinciExamples/tupling/AllFunctors.py @@ -17,8 +17,8 @@ __date__ = "2021-11-23" import Functors as F from FunTuple import FunctorCollection from FunTuple import FunTuple_Particles as Funtuple -from PyConf.components import force_location from DaVinci.reco_objects import make_pvs +from PyConf.reading import get_particles from DaVinci.algorithms import add_filter, get_decreports, get_odin from DecayTreeFitter import DTFAlg from DaVinci.truth_matching import configured_MCTruthAndBkgCatAlg @@ -30,7 +30,7 @@ from DaVinci import Options, make_config # Definition of strucing line # bd2dsk_line = "SpruceB2OC_BdToDsmK_DsmToHHH_FEST_Line" -bd2dsk_data = force_location(f"/Event/Spruce/{bd2dsk_line}/Particles") +bd2dsk_data = get_particles(f"/Event/Spruce/{bd2dsk_line}/Particles") _basic = 'basic' _composite = 'composite' diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/DTF_filtered.py b/DaVinciExamples/python/DaVinciExamples/tupling/DTF_filtered.py index f5673fbeb..21e992b9d 100644 --- a/DaVinciExamples/python/DaVinciExamples/tupling/DTF_filtered.py +++ b/DaVinciExamples/python/DaVinciExamples/tupling/DTF_filtered.py @@ -22,6 +22,7 @@ from DaVinci.algorithms import add_filter #, filter_on from DecayTreeFitter import DTFAlg from FunTuple import FunctorCollection as FC from FunTuple import FunTuple_Particles as Funtuple +from PyConf.reading import get_particles def main(options: Options): @@ -34,11 +35,11 @@ def main(options: Options): #Get filtered particles (Note decay_descriptor is optional, if specified only B0 decays will be selected for processing) spruce_line = "SpruceB2OC_BdToDsmK_DsmToHHH_FEST_Line" # REPLACING TEMPORARY THE INPUT DATA - from PyConf.components import force_location #data_filtered = filter_on( - #f"/Event/Spruce/{spruce_line}/Particles", - #decay_descriptor=fields['B0']) - data_filtered = force_location(f"/Event/Spruce/{spruce_line}/Particles") + # f"/Event/Spruce/{spruce_line}/Particles", + # options.process, + # decay_descriptor=fields['B0']) + data_filtered = get_particles(f"/Event/Spruce/{spruce_line}/Particles") # DecayTreeFitter Algorithm. DTF = DTFAlg( diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/example-tupling-mc-reconstructible-reconstructed.py b/DaVinciExamples/python/DaVinciExamples/tupling/example-tupling-mc-reconstructible-reconstructed.py index 624c6e832..742b7c128 100644 --- a/DaVinciExamples/python/DaVinciExamples/tupling/example-tupling-mc-reconstructible-reconstructed.py +++ b/DaVinciExamples/python/DaVinciExamples/tupling/example-tupling-mc-reconstructible-reconstructed.py @@ -14,20 +14,36 @@ and MCReconstructed. """ from FunTuple import FunTuple_MCParticles as MCFuntuple -from PyConf.components import force_location from DaVinciMCTools import MCReconstructible, MCReconstructed from DaVinci import Options, make_config from RecoConf.data_from_file import make_mc_track_info from FunTuple.functorcollections import MCReconstructed_Collection, MCReconstructible_Collection +from PyConf.reading import get_mc_particles, get_pp2mcp_relations +from DaVinci.reco_objects import make_charged_protoparticles, make_neutral_protoparticles def main(options: Options): + # Input - MC_data = force_location("/Event/HLT2/MC/Particles") + MC_data = get_mc_particles("/Event/HLT2/MC/Particles") + # PP2MCP relations need MC particles and ProtoParticles + # Since MC is already unpacked, only unpack protos and relations + extra_inputs = [] + extra_inputs += [make_charged_protoparticles(options.process)] + extra_inputs += [make_neutral_protoparticles(options.process)] + + relations = [ + get_pp2mcp_relations( + loc, process=options.process, extra_inputs=extra_inputs) + for loc in [ + "/Event/HLT2/Relations/ChargedPP2MCP", + "/Event/HLT2/Relations/NeutralPP2MCP" + ] + ] #Get variables related to reconstructible information. mcrtible = MCReconstructible( - process='Hlt2', mc_track_info=make_mc_track_info()) + process=options.process, mc_track_info=make_mc_track_info()) #The option extra_info is set to False by default and can be set to True to get more information vars_rtible = MCReconstructible_Collection(mcrtible, extra_info=True) print('Reconstructible functors:', vars_rtible.functor_dict.keys()) @@ -44,7 +60,10 @@ def main(options: Options): # tracks to the mc particle are used (tupling arrays). # Here we set it to false for testing purposes. mcrted_all = MCReconstructed( - MC_data, use_best_mcmatch=False, process='Hlt2') + MC_data, + use_best_mcmatch=False, + relations_locs=relations, + process=options.process) #The option extra_info below is set to False by default in the functor collection. vars_rted = MCReconstructed_Collection(mcrted_all, extra_info=False) #Note: diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_array_taggers.py b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_array_taggers.py index 56f37fe51..272b7726c 100644 --- a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_array_taggers.py +++ b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_array_taggers.py @@ -20,12 +20,10 @@ Then the MAP_INPUT_ARRAY functor takes in input this relation map and for each entry stores the output of an external functor (i.e F.P, F.PT) in a vector. """ -from PyConf.components import force_location -from PyConf.Algorithms import ParticleTaggerAlg, ParticleContainerMerger - import Functors as F - +from PyConf.Algorithms import ParticleTaggerAlg, ParticleContainerMerger from FunTuple import FunctorCollection, FunTuple_Particles as Funtuple +from PyConf.reading import get_particles from DaVinci import Options, make_config from DaVinci.algorithms import add_filter @@ -34,7 +32,7 @@ from DaVinci.common_particles import make_long_pions def main(options: Options): bd2dsk_line = "SpruceB2OC_BdToDsmK_DsmToHHH_FEST_Line" - bd2dsk_data = force_location(f"/Event/Spruce/{bd2dsk_line}/Particles") + bd2dsk_data = get_particles(f"/Event/Spruce/{bd2dsk_line}/Particles") pions = make_long_pions() diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_eventinfo.py b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_eventinfo.py index 19cf451a5..739adae45 100644 --- a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_eventinfo.py +++ b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_eventinfo.py @@ -15,12 +15,12 @@ Read an HLT2 file and create an ntuple with information from RecSumarry import Functors as F from FunTuple import FunctorCollection as FC from FunTuple import FunTuple_Particles as Funtuple -from PyConf.components import force_location from DaVinci.algorithms import add_filter from DaVinci import make_config, Options from DaVinci.algorithms import get_odin from FunTuple.functorcollections import EventInfo from DaVinci.reco_objects import get_rec_summary +from PyConf.reading import get_particles def main(options: Options): @@ -35,7 +35,7 @@ def main(options: Options): #get RecSummary object that holds information about nPVs, nTracks, nFTClusters # Note more information can be added to the RecSummary object # (see MRs: https://gitlab.cern.ch/lhcb/Moore/-/merge_requests/1649) - rec_summary = get_rec_summary(options) + rec_summary = get_rec_summary(options.process) evt_vars = FC({ 'nTracks': F.VALUE_OR(-1) @ F.NTRACKS(rec_summary), 'nPVs': F.VALUE_OR(-1) @ F.NPVS(rec_summary), @@ -47,7 +47,8 @@ def main(options: Options): #get particles to run over line_name = 'Hlt2SLB_LbToLcMuNu_LcToPKPi_Line' - particles = force_location(f"/Event/HLT2/{line_name}/Particles") + particles = get_particles( + f"/Event/HLT2/{line_name}/Particles", process=options.process) my_filter = add_filter(options, "Myfilter", f"HLT_PASS('{line_name}')") #define tupling algorithm my_tuple = Funtuple( diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_collections.py b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_collections.py index 5e0b736f2..d52546969 100644 --- a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_collections.py +++ b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_collections.py @@ -13,7 +13,6 @@ Read an HLT2 file and create an ntuple using pre-defined Functor collections. """ import Functors as F -from PyConf.components import force_location from DaVinciMCTools import MCReconstructible, MCReconstructed from FunTuple import FunctorCollection, functorcollections from FunTuple import FunTuple_Particles as Funtuple, FunTuple_MCParticles as MCFuntuple @@ -24,12 +23,15 @@ from DaVinci.truth_matching import configured_MCTruthAndBkgCatAlg from DaVinci.algorithms import get_odin, get_decreports from PyConf.Algorithms import WeightedRelTableAlg from Gaudi.Configuration import INFO +from PyConf.reading import get_particles, get_mc_particles def main(options: Options): line_name = 'Hlt2Charm_D0ToKmPip_Line' - d02kpi_data = force_location(f"/Event/HLT2/{line_name}/Particles") - MC_data = force_location("/Event/HLT2/MC/Particles") + + d02kpi_data = get_particles( + f"/Event/HLT2/{line_name}/Particles", process=options.process) + MC_data = get_mc_particles("/Event/HLT2/MC/Particles") #Get variables related to reconstructible information. mcrtible = MCReconstructible( diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_hlt2.py b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_hlt2.py index 13d497f1f..e0412fc7e 100644 --- a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_hlt2.py +++ b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_hlt2.py @@ -14,8 +14,8 @@ Read an HLT2 file and create an ntuple with the new DaVinci configuration. import Functors as F from FunTuple import FunctorCollection as FC from FunTuple import FunTuple_Particles as Funtuple -from PyConf.components import force_location from DaVinci.reco_objects import make_pvs +from PyConf.reading import get_particles from DaVinci.algorithms import add_filter from DaVinci import Options, make_config from DaVinci.truth_matching import configured_MCTruthAndBkgCatAlg @@ -66,7 +66,8 @@ def main(options: Options): } line_name = 'Hlt2Charm_D0ToKmPip_Line' - d02kpi_data = force_location(f"/Event/HLT2/{line_name}/Particles") + d02kpi_data = get_particles( + f"/Event/HLT2/{line_name}/Particles", process=options.process) my_filter = add_filter(options, "HDRFilter_D0Kpi", f"HLT_PASS('{line_name}')") diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_passthrough.py b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_passthrough.py index 2d8003f14..97c0057cd 100644 --- a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_passthrough.py +++ b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_passthrough.py @@ -14,15 +14,15 @@ objects are persisted by means of a pass through line. """ from FunTuple import FunTuple_Particles as Funtuple from FunTuple.functorcollections import Kinematics -from PyConf.components import force_location from DaVinci import Options, make_config from DaVinci.algorithms import add_filter +from PyConf.reading import get_particles def main(options: Options): bs2jpsiphi_line = "Hlt2BsToJpsiPhi_JPsi2MuMu_PhiToKK_Line" - bs2jpsiphi_data = force_location( - f"/Event/HLT2/{bs2jpsiphi_line}/Particles") + bs2jpsiphi_data = get_particles( + f"/Event/HLT2/{bs2jpsiphi_line}/Particles", process=options.process) fields = { 'Bs': "[B_s0 -> (J/psi(1S) -> mu+ mu-) (phi(1020) -> K+ K-)]CC", 'Jpsi': "[B_s0 -> ^(J/psi(1S) -> mu+ mu-) (phi(1020) -> K+ K-)]CC", diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_spruce.py b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_spruce.py index 193b96242..f50067cf1 100644 --- a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_spruce.py +++ b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_spruce.py @@ -14,16 +14,16 @@ Read the output of an Sprucing job with the new DaVinci configuration. from FunTuple import FunctorCollection from FunTuple import FunTuple_Particles as Funtuple from FunTuple.functorcollections import Kinematics -from PyConf.components import force_location from DaVinci.algorithms import add_filter from DaVinci import Options, make_config +from PyConf.reading import get_particles def main(options: Options): line_B0DsK = 'SpruceB2OC_BdToDsmK_DsmToHHH_FEST_Line' line_B0Dspi = 'SpruceB2OC_BdToDsmPi_DsmToKpKmPim_Line' - bd2dsk_line = force_location(f"/Event/Spruce/{line_B0DsK}/Particles") - bd2dspi_line = force_location(f"/Event/Spruce/{line_B0Dspi}/Particles") + bd2dsk_line = get_particles(f"/Event/Spruce/{line_B0DsK}/Particles") + bd2dspi_line = get_particles(f"/Event/Spruce/{line_B0Dspi}/Particles") fields_dsk = { 'B0': "[B0 -> D_s- K+]CC", diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_spruce_mc.py b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_spruce_mc.py index 565b55231..bb64a97a2 100644 --- a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_spruce_mc.py +++ b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_spruce_mc.py @@ -15,7 +15,7 @@ from FunTuple.functorcollections import Kinematics from DaVinci import Options, make_config from DaVinci.truth_matching import configured_MCTruthAndBkgCatAlg from DaVinci.algorithms import add_filter -from PyConf.components import force_location +from PyConf.reading import get_particles def main(options: Options): @@ -35,7 +35,7 @@ def main(options: Options): } line_name = 'Spruce_Test_line' - B_data = force_location(f"/Event/Spruce/{line_name}/Particles") + B_data = get_particles(f"/Event/Spruce/{line_name}/Particles") my_filter = add_filter(options, "HDRFilter_B", f"HLT_PASS('{line_name}')") diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_xgen.py b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_xgen.py index 94a9ae96c..716ff2609 100644 --- a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_xgen.py +++ b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_xgen.py @@ -13,9 +13,9 @@ Read and process a .xgen file with the new DaVinci configuration. """ from FunTuple import FunctorCollection, FunTuple_MCParticles as FuntupleMC from PyConf.Algorithms import PrintMCTree -from PyConf.dataflow import force_location import Functors as F from DaVinci import Options, make_config +from PyConf.reading import get_mc_particles def main(options: Options): @@ -49,11 +49,7 @@ def main(options: Options): 'Bu': variables_B, } - # Since the "/Event/MC/Particles" location is already declared as output of the UnpackMCParticle algorithm, - # it's not possible to instantiate a new Gaudi__Hive__FetchDataFromFile object with the usual - # 'make_data_with_FetchDataFromFile' method with the same output location. - # It's there sufficient forcing FunTuple to use the same location as input. - bu2jpsik_line = force_location("/Event/MC/Particles") + bu2jpsik_line = get_mc_particles("/Event/MC/Particles") printMC = PrintMCTree( MCParticles=bu2jpsik_line, ParticleNames=["B+", "B-"], OutputLevel=4) diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_weightedrelation_trackvariables.py b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_weightedrelation_trackvariables.py index b2e05b481..4f6c0eb44 100644 --- a/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_weightedrelation_trackvariables.py +++ b/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_weightedrelation_trackvariables.py @@ -19,35 +19,28 @@ To run the example: $DVPATH/run lbexec option_davinci_tupling_weightedrelation_t """ import Functors as F -from PyConf.components import force_location from PyConf.Algorithms import WeightedRelTableAlg from FunTuple import FunctorCollection, FunTuple_Particles as Funtuple from FunTuple.functorcollections import TrackIsolation from DaVinci.reco_objects import make_pvs -from DaVinci.algorithms import add_filter, unpack_locations +from PyConf.reading import get_particles +from DaVinci.algorithms import add_filter from DaVinci import Options, make_config -b2jpsik_data = force_location("/Event/HLT2/Hlt2B2JpsiKLine/Particles") -branches = { - 'B': "[B+ -> (J/psi(1S) -> mu+ mu- ) K+]CC", - 'Jpsi': "[B+ -> ^(J/psi(1S) -> mu+ mu- ) K+]CC", - 'Kp': "[B+ -> (J/psi(1S) -> mu+ mu- ) ^K+]CC" -} +def main(options: Options): + branches = { + 'B': "[B+ -> (J/psi(1S) -> mu+ mu- ) K+]CC", + 'Jpsi': "[B+ -> ^(J/psi(1S) -> mu+ mu- ) K+]CC", + 'Kp': "[B+ -> (J/psi(1S) -> mu+ mu- ) ^K+]CC" + } -def main(options: Options): - # Hack used to unpack the tagged data for now - # follows discussion on Mattermost channel DPA WP3 Offline Analysis - # [https://mattermost.web.cern.ch/lhcb/pl/p6tbr8inetf99jgin5ihce63ic] - unpackers = unpack_locations(options, False) - tagged_data = None - for alg in unpackers: - if "OutputName" in alg.outputs.keys(): - if (alg.OutputName.location == - "/Event/HLT2/Hlt2B2JpsiKLine/LongTaggingParticles/Particles" - ): - tagged_data = alg.OutputName + b2jpsik_data = get_particles( + "/Event/HLT2/Hlt2B2JpsiKLine/Particles", process=options.process) + tagged_data = get_particles( + "/Event/HLT2/Hlt2B2JpsiKLine/LongTaggingParticles/Particles", + process=options.process) pvs = make_pvs(process=options.process) diff --git a/DaVinciExamples/python/DaVinciExamples/tupling/option_trigger_decisions.py b/DaVinciExamples/python/DaVinciExamples/tupling/option_trigger_decisions.py index 6c85f54b7..ccb69d697 100644 --- a/DaVinciExamples/python/DaVinciExamples/tupling/option_trigger_decisions.py +++ b/DaVinciExamples/python/DaVinciExamples/tupling/option_trigger_decisions.py @@ -14,9 +14,9 @@ Read an HLT2 file and create an ntuple with the new DaVinci configuration, acces import Functors as F from FunTuple import FunctorCollection from FunTuple import FunTuple_Particles as Funtuple -from PyConf.components import force_location from DaVinci import Options, make_config from DaVinci.algorithms import get_decreports +from PyConf.reading import get_particles from FunTuple.functorcollections import SelectionInfo @@ -27,8 +27,9 @@ def main(options: Options): variables = {"B0": B0_variables} - b02dpi_data = force_location( - "/Event/HLT2/Hlt2B2OC_BdToDmPi_DmToPimPimKp_Line/Particles") + b02dpi_data = get_particles( + "/Event/HLT2/Hlt2B2OC_BdToDmPi_DmToPimPimKp_Line/Particles", + process=options.process) hlt1_dec = get_decreports("Hlt1", options) diff --git a/DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_trigger_decisions.qmt b/DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_trigger_decisions.qmt index ce0b1384a..9fb75425a 100644 --- a/DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_trigger_decisions.qmt +++ b/DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_trigger_decisions.qmt @@ -24,7 +24,7 @@ <argument name="validator"><text> from DaVinciTests.QMTest.DaVinciExclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 12, "ERROR": 0}, +countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, stdout=remove_known_warnings(stdout)) import sys, os, glob diff --git a/DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_tupling_eventinfo.qmt b/DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_tupling_eventinfo.qmt index 5c3922595..520a42bd7 100644 --- a/DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_tupling_eventinfo.qmt +++ b/DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_tupling_eventinfo.qmt @@ -53,7 +53,7 @@ if len(B_excluded_2) != 0: raise Exception('Number of stored variables is greate f.Close() print('Test successfully completed!') os.system(f"rm {ntuple}") -countErrorLines({"FATAL": 0, "WARNING": 9, "ERROR": 0}, +countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, stdout=remove_known_warnings(stdout)) </text></argument> diff --git a/DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_tupling_mc_recinfo.qmt b/DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_tupling_mc_recinfo.qmt index aef259b56..c5edbd2f2 100644 --- a/DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_tupling_mc_recinfo.qmt +++ b/DaVinciExamples/tests/qmtest/tupling.qms/test_davinci_tupling_mc_recinfo.qmt @@ -23,7 +23,7 @@ </text></argument> <argument name="validator"><text> from DaVinciTests.QMTest.DaVinciExclusions import remove_known_warnings -countErrorLines({"FATAL": 0, "WARNING": 1, "ERROR": 0}, +countErrorLines({"FATAL": 0, "WARNING": 0, "ERROR": 0}, stdout=remove_known_warnings(stdout)) import sys, os from ROOT import TFile diff --git a/DaVinciExamples/tests/qmtest/tupling.qms/test_example-tupling-basic.qmt b/DaVinciExamples/tests/qmtest/tupling.qms/test_example-tupling-basic.qmt index ef5fba35d..9e7117609 100644 --- a/DaVinciExamples/tests/qmtest/tupling.qms/test_example-tupling-basic.qmt +++ b/DaVinciExamples/tests/qmtest/tupling.qms/test_example-tupling-basic.qmt @@ -21,6 +21,7 @@ histo_file: DV-example-tupling-basic-his.root evt_max: 10 input_raw_format: 4.3 + enable_unpack: false </text></argument> <argument name="timeout"><integer>3600</integer></argument> <argument name="args"><set> diff --git a/DaVinciTests/python/DaVinciTests/QMTest/DaVinciExclusions.py b/DaVinciTests/python/DaVinciTests/QMTest/DaVinciExclusions.py index 726924c56..a35ba192b 100755 --- a/DaVinciTests/python/DaVinciTests/QMTest/DaVinciExclusions.py +++ b/DaVinciTests/python/DaVinciTests/QMTest/DaVinciExclusions.py @@ -53,6 +53,11 @@ remove_known_warnings = LineSkipper(regexps=[ r"DetectorDataSvc +SUCCESS Detector description database: git:/lhcb.xml", r"[a-zA-Z0-9]* +WARNING TransportSvc is currently incompatible with DD4HEP. .*", r"[a-zA-Z0-9]* +WARNING See https://gitlab.cern.ch/lhcb/Rec/-/issues/326 for more details", + # backwards compatibility -- key is from manifest, not git + r"key 0x[0-9a-f]+ has an explicitly configured overrule -- using that", + # backwards compatibility -- old data + r"DstData raw bank has a zero encoding key, and it is not explicitly specified for decoding *", + r"HltDecReports has a zero TCK, and it is not explicitly specified for decoding *", ]) remove_known_fluctuating_counters = LineSkipper(regexps=[ diff --git a/DaVinciTests/python/DaVinciTests/funtuple_array.py b/DaVinciTests/python/DaVinciTests/funtuple_array.py index 9b8919688..e02672bb6 100644 --- a/DaVinciTests/python/DaVinciTests/funtuple_array.py +++ b/DaVinciTests/python/DaVinciTests/funtuple_array.py @@ -13,17 +13,17 @@ Option file for testing if FunTuple handles correctly an array with a user-defin """ import Functors as F -from PyConf.components import force_location from PyConf.Algorithms import ParticleTaggerAlg, ParticleContainerMerger from FunTuple import FunctorCollection, FunTuple_Particles as Funtuple from DaVinci.algorithms import add_filter from DaVinci.common_particles import make_long_pions +from PyConf.reading import get_particles from DaVinci import Options, make_config def main(options: Options): bd2dsk_line = "SpruceB2OC_BdToDsmK_DsmToHHH_FEST_Line" - bd2dsk_data = force_location(f"/Event/Spruce/{bd2dsk_line}/Particles") + bd2dsk_data = get_particles(f"/Event/Spruce/{bd2dsk_line}/Particles") # In this test we want to save the information regarding long pions available in the event # storing them in a set of arrays. diff --git a/DaVinciTests/python/DaVinciTests/option_davinci_for_turbo.py b/DaVinciTests/python/DaVinciTests/option_davinci_for_turbo.py index 16334fe3b..42e2dd1e9 100644 --- a/DaVinciTests/python/DaVinciTests/option_davinci_for_turbo.py +++ b/DaVinciTests/python/DaVinciTests/option_davinci_for_turbo.py @@ -14,15 +14,15 @@ objects are persisted by means of a pass through line. """ from FunTuple import FunTuple_Particles as Funtuple from FunTuple.functorcollections import Kinematics -from PyConf.components import force_location +from PyConf.reading import get_particles from DaVinci import Options, make_config from DaVinci.algorithms import add_filter def main(options: Options): bs2jpsiphi_line = "Hlt2Generic_Bs0ToJpsiPhi_JPsiToMupMum_Line" - bs2jpsiphi_data = force_location( - f"/Event/HLT2/{bs2jpsiphi_line}/Particles") + bs2jpsiphi_data = get_particles( + f"/Event/HLT2/{bs2jpsiphi_line}/Particles", process=options.process) fields = { 'Bs': "[B_s0 -> (J/psi(1S) -> mu+ mu-) (phi(1020) -> K+ K-)]CC", 'Jpsi': "[B_s0 -> ^(J/psi(1S) -> mu+ mu-) (phi(1020) -> K+ K-)]CC", diff --git a/DaVinciTests/python/DaVinciTests/recVertices.py b/DaVinciTests/python/DaVinciTests/recVertices.py index bdd27959d..3ab692ea2 100644 --- a/DaVinciTests/python/DaVinciTests/recVertices.py +++ b/DaVinciTests/python/DaVinciTests/recVertices.py @@ -15,14 +15,14 @@ import Functors as F from FunTuple import FunctorCollection from FunTuple import FunTuple_Particles as Funtuple from DaVinci.reco_objects import make_pvs +from PyConf.reading import get_particles from DaVinci.algorithms import add_filter -from PyConf.components import force_location from DaVinci import Options, make_config def main(options: Options): bd2dsk_line = "SpruceB2OC_BdToDsmK_DsmToHHH_FEST_Line" - bd2dsk_data = force_location(f"/Event/Spruce/{bd2dsk_line}/Particles") + bd2dsk_data = get_particles(f"/Event/Spruce/{bd2dsk_line}/Particles") fields_dsk = { 'B0': "[B0 -> D_s- K+]CC", diff --git a/DaVinciTests/tests/options/option_davinci_funtuple_array.py b/DaVinciTests/tests/options/option_davinci_funtuple_array.py index 53b13a5f6..8a6708bb8 100644 --- a/DaVinciTests/tests/options/option_davinci_funtuple_array.py +++ b/DaVinciTests/tests/options/option_davinci_funtuple_array.py @@ -16,11 +16,11 @@ """ import Functors as F -from PyConf.components import force_location from PyConf.Algorithms import ParticleTaggerAlg, ParticleContainerMerger from FunTuple import FunctorCollection, FunTuple_Particles as Funtuple from DaVinci.algorithms import add_filter from DaVinci.common_particles import make_long_pions +from PyConf.reading import get_particles from DaVinci import options options.annsvc_config = 'root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp3/tests/spruce_all_lines_realtime_newPacking.tck.json' @@ -30,7 +30,7 @@ options.ntuple_file = 'DV-test-array-ntp.root' options.process = 'Spruce' bd2dsk_line = "SpruceB2OC_BdToDsmK_DsmToHHH_FEST_Line" -bd2dsk_data = force_location(f"/Event/Spruce/{bd2dsk_line}/Particles") +bd2dsk_data = get_particles(f"/Event/Spruce/{bd2dsk_line}/Particles") # In this test we want to save the information regarding long pions available in the event # storing them in a set of arrays. diff --git a/DaVinciTests/tests/options/option_davinci_sprucing.py b/DaVinciTests/tests/options/option_davinci_sprucing.py index bb643a07a..24c539117 100644 --- a/DaVinciTests/tests/options/option_davinci_sprucing.py +++ b/DaVinciTests/tests/options/option_davinci_sprucing.py @@ -15,12 +15,12 @@ from FunTuple import FunctorCollection from FunTuple import FunTuple_Particles as Funtuple from FunTuple.functorcollections import Kinematics from DaVinci.algorithms import add_filter +from PyConf.reading import get_particles from DaVinci import Options, make_config -from PyConf.components import force_location def main(options: Options): - bd2dsk_line = force_location( + bd2dsk_line = get_particles( "/Event/Spruce/SpruceB2OC_BdToDsmK_DsmToHHH_FEST_Line/Particles") fields = { diff --git a/DaVinciTests/tests/qmtest/io.qms/test_read_mc_digi.qmt b/DaVinciTests/tests/qmtest/io.qms/test_read_mc_digi.qmt index 976ce5801..d37376071 100644 --- a/DaVinciTests/tests/qmtest/io.qms/test_read_mc_digi.qmt +++ b/DaVinciTests/tests/qmtest/io.qms/test_read_mc_digi.qmt @@ -19,6 +19,7 @@ <argument name="test_file_db_options_yaml"><text>upgrade-magdown-sim09c-up02-34112100-digi</text></argument> <argument name="extra_options_yaml"><text> evt_max: 200 + enable_unpack: false </text></argument> <argument name="reference"><text>../refs/test_davinci_read_mc_digi.ref</text></argument> <argument name="error_reference"><text>../refs/empty.ref</text></argument> diff --git a/DaVinciTests/tests/qmtest/io.qms/test_read_mc_ldst.qmt b/DaVinciTests/tests/qmtest/io.qms/test_read_mc_ldst.qmt index e0f1b1ff2..277904cb2 100644 --- a/DaVinciTests/tests/qmtest/io.qms/test_read_mc_ldst.qmt +++ b/DaVinciTests/tests/qmtest/io.qms/test_read_mc_ldst.qmt @@ -19,6 +19,7 @@ <argument name="test_file_db_options_yaml"><text>upgrade-magdown-sim09c-up02-reco-up01-minbias-ldst</text></argument> <argument name="extra_options_yaml"><text> evt_max: 200 + enable_unpack: false </text></argument> <argument name="reference"><text>../refs/test_davinci_read_mc_ldst.ref</text></argument> <argument name="error_reference"><text>../refs/empty.ref</text></argument> diff --git a/DaVinciTests/tests/qmtest/io.qms/test_read_mc_mdf.qmt b/DaVinciTests/tests/qmtest/io.qms/test_read_mc_mdf.qmt index b67e99bd9..6cf262f94 100644 --- a/DaVinciTests/tests/qmtest/io.qms/test_read_mc_mdf.qmt +++ b/DaVinciTests/tests/qmtest/io.qms/test_read_mc_mdf.qmt @@ -20,6 +20,7 @@ <argument name="extra_options_yaml"><text> input_type: RAW evt_max: 200 + enable_unpack: false </text></argument> <argument name="reference"><text>../refs/test_davinci_read_mc_mdf.ref</text></argument> <argument name="error_reference"><text>../refs/empty.ref</text></argument> diff --git a/DaVinciTests/tests/qmtest/io.qms/test_read_mc_xdigi.qmt b/DaVinciTests/tests/qmtest/io.qms/test_read_mc_xdigi.qmt index b5d41cdc0..c43a1d3a1 100644 --- a/DaVinciTests/tests/qmtest/io.qms/test_read_mc_xdigi.qmt +++ b/DaVinciTests/tests/qmtest/io.qms/test_read_mc_xdigi.qmt @@ -19,6 +19,7 @@ <argument name="test_file_db_options_yaml"><text>upgrade_DC19_01_Bs2JPsiPhi_MD</text></argument> <argument name="extra_options_yaml"><text> evt_max: 200 + enable_unpack: false </text></argument> <argument name="reference"><text>../refs/test_davinci_read_mc_xdigi.ref</text></argument> <argument name="error_reference"><text>../refs/empty.ref</text></argument> diff --git a/DaVinciTests/tests/qmtest/io.qms/test_read_mc_xgen.qmt b/DaVinciTests/tests/qmtest/io.qms/test_read_mc_xgen.qmt index e9cae67a0..2a1a725bd 100644 --- a/DaVinciTests/tests/qmtest/io.qms/test_read_mc_xgen.qmt +++ b/DaVinciTests/tests/qmtest/io.qms/test_read_mc_xgen.qmt @@ -21,6 +21,7 @@ <argument name="options_yaml_fn"><text>$DAVINCIEXAMPLESROOT/example_data/test_read_xgen.yaml</text></argument> <argument name="extra_options_yaml"><text> evt_max: -1 + enable_unpack: false </text></argument> <argument name="reference"><text>../refs/test_davinci_read_mc_xgen.ref</text></argument> <argument name="error_reference"><text>../refs/empty.ref</text></argument> diff --git a/DaVinciTests/tests/qmtest/io.qms/test_read_moore_dst.qmt b/DaVinciTests/tests/qmtest/io.qms/test_read_moore_dst.qmt index fe772296a..10c068d6f 100755 --- a/DaVinciTests/tests/qmtest/io.qms/test_read_moore_dst.qmt +++ b/DaVinciTests/tests/qmtest/io.qms/test_read_moore_dst.qmt @@ -23,6 +23,7 @@ evt_max: -1 print_freq: 1 msg_svc_format: "% F%60W%S%7W%R%T %0W%M" + enable_unpack: false </text></argument> <argument name="validator"><text> findReferenceBlock("""StdLooseD02KK INFO Number of counters : 6 diff --git a/DaVinciTutorials/python/DaVinciTutorials/tutorial0_basic_DVjob.py b/DaVinciTutorials/python/DaVinciTutorials/tutorial0_basic_DVjob.py index 15a352518..7af2b2804 100644 --- a/DaVinciTutorials/python/DaVinciTutorials/tutorial0_basic_DVjob.py +++ b/DaVinciTutorials/python/DaVinciTutorials/tutorial0_basic_DVjob.py @@ -10,8 +10,8 @@ ############################################################################### from DaVinci import Options, make_config from DaVinci.algorithms import add_filter +from PyConf.reading import get_particles from PyConf.Algorithms import PrintDecayTree -from PyConf.dataflow import force_location def main(options: Options): @@ -20,10 +20,9 @@ def main(options: Options): # For the TES path checkout spruce_passthrough.tck.json or you can do a dst dump # (see https://lhcb.github.io/starterkit-lessons/first-analysis-steps/interactive-dst.html) # - # The TES location input to the algorithms must of type "PyConf.DataHandle" and not pure strings. - # Therefore we wrap the TES location string below with "force_location" wrapper class. turbo_line = "Hlt2BsToJpsiPhi_JPsi2MuMu_PhiToKK_Line" - input_data = force_location(f"/Event/HLT2/{turbo_line}/Particles") + input_data = get_particles( + f"/Event/HLT2/{turbo_line}/Particles", process=options.process) # Add a filter: We are not really filtering over particles, we are getting over a technical hurdle here. # The hurdle being that if the event hasn't fired a HLT2 line then no TES location exists diff --git a/DaVinciTutorials/python/DaVinciTutorials/tutorial1_functors_specialfield.py b/DaVinciTutorials/python/DaVinciTutorials/tutorial1_functors_specialfield.py index 503be1ef2..2be112727 100644 --- a/DaVinciTutorials/python/DaVinciTutorials/tutorial1_functors_specialfield.py +++ b/DaVinciTutorials/python/DaVinciTutorials/tutorial1_functors_specialfield.py @@ -11,9 +11,9 @@ import Functors as F from DaVinci import Options, make_config from DaVinci.algorithms import add_filter +from PyConf.reading import get_particles from FunTuple import FunctorCollection as FC from FunTuple import FunTuple_Particles as Funtuple -from PyConf.dataflow import force_location def main(options: Options): @@ -63,7 +63,8 @@ def main(options: Options): #Define the TES location (see previous example for explanation) turbo_line = "Hlt2BsToJpsiPhi_JPsi2MuMu_PhiToKK_Line" - input_data = force_location(f"/Event/HLT2/{turbo_line}/Particles") + input_data = get_particles( + f"/Event/HLT2/{turbo_line}/Particles", process=options.process) #Define a filter (see previous example for explaination) my_filter = add_filter(options, "HDRFilter_SeeNoEvil", diff --git a/DaVinciTutorials/python/DaVinciTutorials/tutorial2_LoKi.py b/DaVinciTutorials/python/DaVinciTutorials/tutorial2_LoKi.py index 7e17e05f8..ab1668b1d 100644 --- a/DaVinciTutorials/python/DaVinciTutorials/tutorial2_LoKi.py +++ b/DaVinciTutorials/python/DaVinciTutorials/tutorial2_LoKi.py @@ -11,9 +11,9 @@ import Functors as F from DaVinci import Options, make_config from DaVinci.algorithms import add_filter +from PyConf.reading import get_particles from FunTuple import FunctorCollection as FC from FunTuple import FunTuple_Particles as Funtuple -from PyConf.dataflow import force_location def main(options: Options): @@ -64,7 +64,8 @@ def main(options: Options): #Load data from dst onto a TES turbo_line = "Hlt2BsToJpsiPhi_JPsi2MuMu_PhiToKK_Line" - input_data = force_location(f"/Event/HLT2/{turbo_line}/Particles") + input_data = get_particles( + f"/Event/HLT2/{turbo_line}/Particles", process=options.process) #Add a filter my_filter = add_filter(options, "HDRFilter_SeeNoEvil", diff --git a/DaVinciTutorials/python/DaVinciTutorials/tutorial3_ThOrfunctors.py b/DaVinciTutorials/python/DaVinciTutorials/tutorial3_ThOrfunctors.py index 7abead0f8..82edb42ad 100644 --- a/DaVinciTutorials/python/DaVinciTutorials/tutorial3_ThOrfunctors.py +++ b/DaVinciTutorials/python/DaVinciTutorials/tutorial3_ThOrfunctors.py @@ -12,9 +12,9 @@ import Functors as F from DaVinci import Options, make_config from DaVinci.algorithms import add_filter from DaVinci.reco_objects import make_pvs +from PyConf.reading import get_particles from FunTuple import FunctorCollection as FC from FunTuple import FunTuple_Particles as Funtuple -from PyConf.dataflow import force_location def main(options: Options): @@ -89,7 +89,8 @@ def main(options: Options): #Load data from dst onto a TES turbo_line = "Hlt2BsToJpsiPhi_JPsi2MuMu_PhiToKK_Line" - input_data = force_location(f"/Event/HLT2/{turbo_line}/Particles") + input_data = get_particles( + f"/Event/HLT2/{turbo_line}/Particles", process=options.process) #Add a filter my_filter = add_filter(options, "HDRFilter_SeeNoEvil", diff --git a/DaVinciTutorials/python/DaVinciTutorials/tutorial4_trigger_eventinfo.py b/DaVinciTutorials/python/DaVinciTutorials/tutorial4_trigger_eventinfo.py index 63a45539a..aea11c22d 100644 --- a/DaVinciTutorials/python/DaVinciTutorials/tutorial4_trigger_eventinfo.py +++ b/DaVinciTutorials/python/DaVinciTutorials/tutorial4_trigger_eventinfo.py @@ -11,9 +11,9 @@ import Functors as F from DaVinci import Options, make_config from DaVinci.algorithms import add_filter +from PyConf.reading import get_particles from FunTuple import FunctorCollection as FC from FunTuple import FunTuple_Particles as Funtuple -from PyConf.dataflow import force_location def main(options: Options): @@ -90,7 +90,8 @@ def main(options: Options): variables = {"ALL": kin} #Load data from dst onto a TES - input_data = force_location(f"/Event/HLT2/{turbo_line}/Particles") + input_data = get_particles( + f"/Event/HLT2/{turbo_line}/Particles", process=options.process) #Add a filter my_filter = add_filter(options, "HDRFilter_SeeNoEvil", diff --git a/DaVinciTutorials/python/DaVinciTutorials/tutorial5_MCTruth.py b/DaVinciTutorials/python/DaVinciTutorials/tutorial5_MCTruth.py index 24e7e32e3..8b627c1ad 100644 --- a/DaVinciTutorials/python/DaVinciTutorials/tutorial5_MCTruth.py +++ b/DaVinciTutorials/python/DaVinciTutorials/tutorial5_MCTruth.py @@ -11,9 +11,9 @@ import Functors as F from DaVinci import Options, make_config from DaVinci.algorithms import add_filter +from PyConf.reading import get_particles from FunTuple import FunctorCollection as FC from FunTuple import FunTuple_Particles as Funtuple -from PyConf.dataflow import force_location def main(options: Options): @@ -43,7 +43,8 @@ def main(options: Options): #Load data from dst onto a TES turbo_line = "Hlt2BsToJpsiPhi_JPsi2MuMu_PhiToKK_Line" - input_data = force_location(f"/Event/HLT2/{turbo_line}/Particles") + input_data = get_particles( + f"/Event/HLT2/{turbo_line}/Particles", process=options.process) #Define an algorithm that builds a map i.e. one-to-one relation b/w Reco Particle -> Truth MC Particle. mctruth = configured_MCTruthAndBkgCatAlg(inputs=input_data) #print(mctruth.MCAssocTable) diff --git a/DaVinciTutorials/python/DaVinciTutorials/tutorial6_DecayTreeFit.py b/DaVinciTutorials/python/DaVinciTutorials/tutorial6_DecayTreeFit.py index 0ca88c22f..81d86f169 100644 --- a/DaVinciTutorials/python/DaVinciTutorials/tutorial6_DecayTreeFit.py +++ b/DaVinciTutorials/python/DaVinciTutorials/tutorial6_DecayTreeFit.py @@ -11,10 +11,10 @@ import Functors as F from DaVinci import Options, make_config from DaVinci.algorithms import add_filter +from PyConf.reading import get_particles from FunTuple import FunctorCollection as FC from FunTuple import FunTuple_Particles as Funtuple from FunTuple.functorcollections import Kinematics -from PyConf.dataflow import force_location def main(options: Options): @@ -35,7 +35,8 @@ def main(options: Options): #Load data from dst onto a TES (See Example7) turbo_line = "Hlt2BsToJpsiPhi_JPsi2MuMu_PhiToKK_Line" - input_data = force_location(f"/Event/HLT2/{turbo_line}/Particles") + input_data = get_particles( + f"/Event/HLT2/{turbo_line}/Particles", process=options.process) #get kinematic functors kin = Kinematics() diff --git a/DaVinciTutorials/python/DaVinciTutorials/tutorial7_multiple_sel_lines.py b/DaVinciTutorials/python/DaVinciTutorials/tutorial7_multiple_sel_lines.py index d22d18d50..fdd441dc2 100644 --- a/DaVinciTutorials/python/DaVinciTutorials/tutorial7_multiple_sel_lines.py +++ b/DaVinciTutorials/python/DaVinciTutorials/tutorial7_multiple_sel_lines.py @@ -10,9 +10,9 @@ ############################################################################### from DaVinci import Options, make_config from DaVinci.algorithms import add_filter +from PyConf.reading import get_particles from FunTuple import FunTuple_Particles as Funtuple from FunTuple.functorcollections import Kinematics -from PyConf.dataflow import force_location def main(options: Options): @@ -33,7 +33,8 @@ def main(options: Options): #Load data from dst onto a TES turbo_line1 = "Hlt2BsToJpsiPhi_JPsi2MuMu_PhiToKK_Line" - input_data1 = force_location(f"/Event/HLT2/{turbo_line1}/Particles") + input_data1 = get_particles( + f"/Event/HLT2/{turbo_line1}/Particles", process=options.process) my_filter1 = add_filter(options, "HDRFilter_SeeNoEvil1", f"HLT_PASS('{turbo_line1}')") mytuple1 = Funtuple( @@ -45,7 +46,8 @@ def main(options: Options): # If running over several sprucing lines (e.g. for calibration) one can define multiple instances of FunTuple turbo_line2 = "Hlt2BsToJpsiPhi_JPsi2ee_PhiToKK_Line" - input_data2 = force_location(f"/Event/HLT2/{turbo_line2}/Particles") + input_data2 = get_particles( + f"/Event/HLT2/{turbo_line2}/Particles", process=options.process) my_filter2 = add_filter(options, "HDRFilter_SeeNoEvil2", f"HLT_PASS('{turbo_line2}')") mytuple2 = Funtuple( diff --git a/Phys/DaVinci/python/DaVinci/LbExec.py b/Phys/DaVinci/python/DaVinci/LbExec.py index 47427f920..a1aa89175 100644 --- a/Phys/DaVinci/python/DaVinci/LbExec.py +++ b/Phys/DaVinci/python/DaVinci/LbExec.py @@ -31,6 +31,3 @@ class Options(DefaultOptions): enable_unpack: bool = True write_fsr: bool = True merge_genfsr: bool = False - # For xgen files we need to unpack only mc particles and vertices - # Needed until unpacking will become functional - unpack_only_mc: bool = False diff --git a/Phys/DaVinci/python/DaVinci/algorithms.py b/Phys/DaVinci/python/DaVinci/algorithms.py index aad3d60c2..968643811 100644 --- a/Phys/DaVinci/python/DaVinci/algorithms.py +++ b/Phys/DaVinci/python/DaVinci/algorithms.py @@ -10,7 +10,6 @@ ############################################################################### import re import logging -import click from PyConf.Algorithms import ( FilterDecays, @@ -22,9 +21,7 @@ from PyConf.application import ( make_odin, ) from Hlt2Conf.algorithms import make_dvalgorithm -from PyConf.components import force_location -from Gaudi.Configuration import WARNING -from GaudiConf import reading +from PyConf.reading import get_particles log = logging.getLogger(__name__) @@ -96,15 +93,15 @@ def add_filter(options, name, code): return algFilter -def apply_filters_and_unpacking(options, algs_dict, unpack_only_mc): +def apply_filters_and_unpacking(options, algs_dict): """ Adding filter and unpacking algorithms. Args: options (DaVinci.Options): lbexec provided options object algs_dict (dict): dict of the user algorithms. - unpack_only_mc (bool): flag to unpack only mc particles and vertices. (TO BE REMOVED WHEN THE UNPACKING WILL BECOME FUNCTIONAL) + (UNPACKING IS FUNCTIONAL, time to remove maybe?) Returns: Dict where at each node filters and unpacking algorithms are prepended to the initial list of user algorithms. @@ -120,8 +117,9 @@ def apply_filters_and_unpacking(options, algs_dict, unpack_only_mc): evt_pre_filters.append(evt_filter) algs_list += evt_pre_filters if options.enable_unpack: - unpackers = unpack_locations(options, unpack_only_mc) - algs_list += unpackers + # This only unpacks the events to get DecReports and ODIN + unpacker = unpack_event(options) + algs_list += [unpacker] algs_list += algs @@ -157,89 +155,29 @@ def define_fsr_writer(options): return algs -def __configure_unpacking(manifest, process, stream, simulation): - unpack_raw_event = reading.unpack_rawevent( - bank_types=['ODIN', 'DstData', 'HltDecReports'], - process=process, - stream=stream, - output_level=5, - configurables=False) - - if process == "Spruce": - TES_ROOT = '/Event/Spruce' - else: - TES_ROOT = '/Event/HLT2' - - reading_algs = [] - - locations = reading.make_locations(manifest, TES_ROOT) - reading_algs += [unpack_raw_event] - - packed_decoder = reading.decoder( - process=process, stream=stream, output_level=5, configurables=False) - reading_algs += [packed_decoder] +def __configure_unpacking(process, stream): + from GaudiConf.reading import unpack_rawevent - mc_unpackers = [] - if simulation: - mc_unpackers = reading.mc_unpackers( - process=process, configurables=False) - - reading_algs += mc_unpackers - - reading_algs += reading.unpackers( - locations, - manifest, - packed_decoder.OutputBuffers, - mc=mc_unpackers, + unpack_raw_event = unpack_rawevent( + bank_types=['ODIN', 'HltDecReports'], process=process, - output_level=5, + stream=stream, configurables=False) - # TODO: return a dictionary of handles to the data instead!!! - return reading_algs + return unpack_raw_event -def unpack_locations(options, unpack_only_mc): +def unpack_event(options): """ - Configures algorithms for reading HLT2/Spruce output. Location are found using stream as prefix. - (TO BE REMOVED WHEN THE UNPACKING WILL BECOME FUNCTIONAL) - + Configures the algorithm for reading HLT2/Spruce output. Location are found using stream as prefix. + Args: - options (DaVinci.Options): lbexec provided options object - - unpack_only_mc (bool): flag to unpack only mc particles and vertices. Returns: - - List of unpacking algorithms. + - List of decoding algorithms. """ - from GaudiConf import reading - - process = options.process - stream = options.stream - reading_algs = [] - - if unpack_only_mc: - if options.simulation: - reading_algs += reading.mc_unpackers( - process=process, filtered_mc=False, configurables=False) - else: - message = 'Requested unpacking MC but simulation is set False. Check your DV options.' - click.echo( - f"{click.style('WARNING', bold=True, fg='yellow')} {click.style(message, fg='white')}" - ) - elif options.annsvc_config: - # NOTE: this has _nothing_ to do with the ANNSvc - # but unfortunately that's what the option is currently called... - assert not options.input_manifest_file, "annsvc_config and input_manifest_file are mutually exclusive" - log.warning( - 'deprecated option: \'annsvc_config\'. Please use \'input_manifest_file\' instead' - ) - reading_algs += __configure_unpacking( - reading.load_manifest(options.annsvc_config), process, stream, - options.simulation) - elif options.input_manifest_file: - reading_algs += __configure_unpacking( - reading.load_manifest(options.input_manifest_file), process, - stream, options.simulation) + reading_algs = __configure_unpacking(options.process, options.stream) return reading_algs @@ -257,22 +195,21 @@ def get_hlt_reports(options, source=''): - HltDecReportsDecoder containing the configuration for hlt2 lines. - HltDecReportsDecoder containing the configuration for spruced lines. """ - from GaudiConf import reading + from GaudiConf.reading import hlt_decisions process = options.process stream = options.stream if source == 'Hlt1' or source == 'Hlt2': - dec_reports = reading.hlt_decisions( + dec_reports = hlt_decisions( process=process, stream=stream, output_loc="/Event/%s/DecReports" % source, configurables=False, - source=source, - output_level=WARNING) + source=source) elif source == 'Spruce': - dec_reports = reading.hlt_decisions( + dec_reports = hlt_decisions( process=process, stream=stream, configurables=False, source=source) return dec_reports @@ -299,7 +236,8 @@ def configured_FunTuple(options, config): dictAlgs = {} for key in config.keys(): - inputs = force_location(config[key]["location"]) + inputs = get_particles( + config[key]["location"], process=options.process) dictAlgs[key] = [] i = 0 @@ -376,7 +314,10 @@ def apply_algorithm(list_particles, algorithm, **kwargs): return dv_algorithm(ParticlesA=list_particles, **kwargs).Particles -def filter_on(location, decay_descriptor=None, bank_type=None): +def filter_on(location, + process="Spruce", + decay_descriptor=None, + bank_type=None): """ Function to get particles from Hlt2 or Spruce sample. A FilterDecays is applied before returning the requested particles if a decay descriptor @@ -389,7 +330,7 @@ def filter_on(location, decay_descriptor=None, bank_type=None): Returns: data: TES location of the particles that are loaded from the input samples """ - data = force_location(location) + data = get_particles(location, process=process) if decay_descriptor: data = apply_algorithm([data], FilterDecays, Code=decay_descriptor) return data diff --git a/Phys/DaVinci/python/DaVinci/config.py b/Phys/DaVinci/python/DaVinci/config.py index b781a7185..97854509f 100644 --- a/Phys/DaVinci/python/DaVinci/config.py +++ b/Phys/DaVinci/python/DaVinci/config.py @@ -158,8 +158,6 @@ def add_davinci_configurables(options, user_algorithms, public_tools): if not public_tools: public_tools = [] - unpack_only_mc = options.unpack_only_mc - config = configure_input(options) if options.input_manifest_file: @@ -179,8 +177,7 @@ def add_davinci_configurables(options, user_algorithms, public_tools): # do in apply_filters_and_unpacking... config.add(configured_ann_svc(json_file=options.annsvc_config)) - dvMainFlow = apply_filters_and_unpacking(options, user_algorithms, - unpack_only_mc) + dvMainFlow = apply_filters_and_unpacking(options, user_algorithms) fsrAlgs = {} if options.write_fsr: diff --git a/Phys/DaVinci/python/DaVinci/reco_objects.py b/Phys/DaVinci/python/DaVinci/reco_objects.py index c6b5c9a63..26fca98b6 100644 --- a/Phys/DaVinci/python/DaVinci/reco_objects.py +++ b/Phys/DaVinci/python/DaVinci/reco_objects.py @@ -18,14 +18,12 @@ packed data on file. and the definition of what gets persisted gets formalised. 2) Code very heavily relies on its Moore equivalent. Thank you, RTA team. """ -from GaudiConf.PersistRecoConf import PersistRecoPacking -from PyConf.location_prefix import prefix, packed_prefix +from PyConf.packing import reco_locations, unpackers_map + from PyConf.components import force_location from PyConf.tonic import configurable - -from RecoConf.data_from_file import unpacked_reco_locations -from DaVinci.algorithms import unpack_locations +from PyConf.reading import upfront_decoder @configurable @@ -36,42 +34,52 @@ def upfront_reconstruction(process='Spruce'): other algorithms, but only to define the control flow, i.e. the return value of this function should be ran before all HLT2 lines. """ - TES_ROOT = '/Event/Spruce' - RECO = 'HLT2' + stream = '/Event/Spruce/HLT2' if process in ['Hlt2', 'Turbo']: - TES_ROOT = '/Event/HLT2' - RECO = '' + stream = '/Event/HLT2' - conf = PersistRecoPacking( - stream=TES_ROOT, reco_stream=RECO, data_type='Upgrade') + reco_loc = reco_locations(stream) + unpacker_algs = [] + unpackers = unpackers_map() - unpackers = list(conf.unpackers()) + for loc, k in reco_loc.values(): + unp = unpackers[k] + unpacker_algs += [ + unp(InputName=upfront_decoder(process), + outputs={"OutputName": force_location(loc)}, + OutputLevel=5) + ] - return unpackers + return unpacker_algs @configurable def reconstruction(process='Spruce'): """Return a {name: DataHandle} dict that define the reconstruction output.""" - map = {} - - TES_ROOT = '/Event/Spruce/HLT2' + stream = '/Event/Spruce/HLT2' if process in ['Hlt2', 'Turbo']: - TES_ROOT = '/Event/HLT2' + stream = '/Event/HLT2' - packed_loc = unpacked_reco_locations() + unpackers = upfront_reconstruction(process) - for key, value in packed_loc.items(): - map[key.replace('Packed', '')] = force_location( - prefix(value, TES_ROOT)) + reco_map = {} + reco_loc = reco_locations(stream) + for key, value in reco_loc.items(): + for v in unpackers: + if "OutputName" in v.outputs.keys( + ) and v.OutputName.location == value[0]: + # values are in format (location, type), need location here + reco_map[key] = v.OutputName + break ### Temporary: as long as we persist v1, we need to insert a converter for the new PVs from PyConf.Algorithms import RecV1ToPVConverter - map["PVs_v1"] = map["PVs"] - map["PVs"] = RecV1ToPVConverter(InputVertices=map["PVs_v1"]).OutputVertices + reco_map["PVs_v1"] = reco_map["PVs"] + reco_map["PVs"] = RecV1ToPVConverter( + InputVertices=reco_map["PVs_v1"]).OutputVertices - return map + return reco_map def make_rich_pids(process='Spruce'): @@ -102,38 +110,5 @@ def make_tracks(process='Spruce'): return reconstruction(process=process)['Tracks'] -def get_rec_summary(options): - #Would ideally want to do reconstruction(process=process)['RecSummary'] - # However throws an error: "multiple algorithms declare /Event/HLT2/Rec/Summary" - # For now use a "hack" (FIX ME) - unpackers = unpack_locations(options, False) - rec_summary = None - for alg in unpackers: - if "OutputName" in alg.outputs.keys(): - if (alg.OutputName.location == '/Event/HLT2/Rec/Summary'): - rec_summary = alg.OutputName - - return rec_summary - - -def get_particles(process="Spruce", location=""): - - if process == 'Spruce': - stream = '/Event/Spruce' - stream_reco = 'HLT2' - else: - stream = '/Event/HLT2' - stream_reco = '' - - conf = PersistRecoPacking( - stream=stream, - reco_stream=stream_reco, - packed={"Particles": [packed_prefix(location, stream)]}, - unpacked={"Particles": [location]}) - - particles = conf.unpackers_by_key()["Particles"] - - if len(particles) > 0: - return particles[0].OutputName - else: - assert "no particles found in location: " + location +def get_rec_summary(process="Spruce"): + return reconstruction(process=process)['RecSummary'] diff --git a/Phys/DaVinci/python/DaVinci/truth_matching.py b/Phys/DaVinci/python/DaVinci/truth_matching.py index cd9f8c092..c62235482 100644 --- a/Phys/DaVinci/python/DaVinci/truth_matching.py +++ b/Phys/DaVinci/python/DaVinci/truth_matching.py @@ -15,6 +15,9 @@ from PyConf.Algorithms import MCTruthAndBkgCatAlg from PyConf.Tools import DaVinciSmartAssociator, MCMatchObjP2MCRelator, ParticleDescendants from PyConf.Tools import BackgroundCategory, BackgroundCategoryViaRelations from PyConf.Tools import P2MCPFromProtoP +from PyConf.reading import get_pp2mcp_relations, get_mc_particles +from PyConf.location_prefix import prefix +from DaVinci.reco_objects import make_charged_protoparticles, make_neutral_protoparticles def configured_MCTruthAndBkgCatAlg( @@ -66,15 +69,28 @@ def configured_MCTruthAndBkgCatAlg( f"The specified 'process' {process} not recognised. Can only be 'Hlt2' or 'Spruce'. Please check!" ) + # No algorithm is asking for mc particles and proto particles explicitely + # But pp2mcp relation unpacker needs them, so give them as extra_inputs + extra_inputs = [] + extra_inputs += [make_charged_protoparticles(process)] + extra_inputs += [make_neutral_protoparticles(process)] + + mc_loc = root_in_tes + "/MC/Particles" + extra_inputs += [get_mc_particles(mc_loc)] + + relations = [ + get_pp2mcp_relations(prefix(rel, root_in_tes), process, extra_inputs) + for rel in relations_locs + ] + # Tool used by DaVinciSmartAssociator p2mctool = P2MCPFromProtoP( - Locations=relations_locs, - RootInTES=root_in_tes, - OutputLevel=output_level) + Locations=relations, RootInTES=root_in_tes, OutputLevel=output_level) # Tools used by MCTruthAndBkgCatAlg bkg_cat = BackgroundCategory( P2MCTool=p2mctool, + ExtraInputs=relations, vetoNeutralRedo=not redo_neutral_assoc, RootInTES=root_in_tes, OutputLevel=output_level) @@ -87,7 +103,7 @@ def configured_MCTruthAndBkgCatAlg( BackgroundCategoryTool=bkg_cat, OutputLevel=output_level) mcrel_assc = MCMatchObjP2MCRelator( - RelTableLocations=relations_locs, + RelTableLocations=relations, #Setting RootInTES for this tool gives a segfaul (why?), we anyway use DaVinciSmartAssociator for association. #RootInTES=root_in_tes, OutputLevel=output_level) diff --git a/Phys/DaVinci/python/DaVinci/utils.py b/Phys/DaVinci/python/DaVinci/utils.py new file mode 100644 index 000000000..061a775c8 --- /dev/null +++ b/Phys/DaVinci/python/DaVinci/utils.py @@ -0,0 +1,29 @@ +############################################################################### +# (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +import os, XRootD.client + + +def load_file(fname): + if not fname: return None + + if fname.startswith("root://eoslhcb.cern.ch//"): + with XRootD.client.File() as f: + status, _ = f.open(str(fname)) + if not status.ok: + raise RuntimeError(f"could not open {fname}: {status.message}") + status, data = f.read() + if not status.ok: + raise RuntimeError(f"could not read {fname}: {status.message}") + return data.decode('utf-8') + else: + with open(os.path.expandvars(fname)) as f: + data = f.read() + return data.decode('utf-8') diff --git a/Phys/DaVinci/tests/config/test_algorithms.py b/Phys/DaVinci/tests/config/test_algorithms.py index 47c1cda4b..2084f412f 100644 --- a/Phys/DaVinci/tests/config/test_algorithms.py +++ b/Phys/DaVinci/tests/config/test_algorithms.py @@ -15,7 +15,6 @@ from DaVinci.algorithms import ( define_fsr_writer, add_filter, #filter_on apply_filters_and_unpacking, - unpack_locations, configured_FunTuple, get_odin, get_decreports) @@ -82,46 +81,9 @@ def test_add_void_filter(): assert "VoidFilter" in test_filter.fullname -def test_unpack_locations(): - """ - Check if the unpacking algorithms are retrieved correctly from GaudiConf. - TO BE REMOVED WHEN THE UNPACKING WILL BECOME FULLY FUNCTIONAL. - """ - options = Options( - data_type="Upgrade", - evt_max=1, - simulation=True, - ) - test_algs = unpack_locations(options, False) - - assert isinstance(test_algs, list) - #assert any("Unpack" in alg.fullname for alg in test_algs) - - -def test_unpack_locations_xgen(): - """ - Check if DaVinci unpacks only the MC locations if run on an xgen file. - (i.e. 'unpack_only_mc' is set to True.) - TO BE REMOVED WHEN THE UNPACKING WILL BECOME FULLY FUNCTIONAL. - """ - options = Options( - data_type="Upgrade", - evt_max=1, - simulation=True, - ) - unpack_only_mc = True - test_algs = unpack_locations(options, unpack_only_mc) - - assert isinstance(test_algs, list) - assert any("UnpackMCParticle" in alg.fullname for alg in test_algs) - assert not any("TrackUnpacker" in alg.fullname for alg in test_algs) - - def test_apply_filters_and_unpack(): """ Check if DaVinci applies correctly a filter in front of a given algorithm - and instantiates correctly the main unpacking algorithms. - TO BE UPDATED WHEN THE UNPACKING WILL BECOME FULLY FUNCTIONAL. """ options = Options( data_type="Upgrade", @@ -131,7 +93,7 @@ def test_apply_filters_and_unpack(): simulation=True, ) alg_dict = {"test_alg": [VoidConsumer()]} - test_alg_dict = apply_filters_and_unpacking(options, alg_dict, False) + test_alg_dict = apply_filters_and_unpacking(options, alg_dict) list_of_main_expected_algs = ["LoKi__VoidFilter"] for exp_alg in list_of_main_expected_algs: diff --git a/doc/configuration/davinci_configuration.rst b/doc/configuration/davinci_configuration.rst index 1a44012f7..36920726c 100644 --- a/doc/configuration/davinci_configuration.rst +++ b/doc/configuration/davinci_configuration.rst @@ -88,7 +88,6 @@ How to run a job on an XGEN file The DaVinci application can also be run over an XGEN (extended generator) file by setting two keys in the ``options.yaml`` file: * Setting the ``input_type: ROOT`` -* Setting ``unpack_only_mc: true`` (this is a temporary workaround until unpacking is functional) The new FunTupleMC algorithm is used to create the tuple. A working example can be found `here <https://gitlab.cern.ch/lhcb/DaVinci/-/blob/master/DaVinciExamples/python/DaVinciExamples/tupling/option_davinci_tupling_from_xgen.py>`__. diff --git a/doc/tutorials/running.rst b/doc/tutorials/running.rst index 0fb67413d..7c2d0eeaa 100644 --- a/doc/tutorials/running.rst +++ b/doc/tutorials/running.rst @@ -32,12 +32,12 @@ Make a file named ``my_module.py`` that contains a function that takes an ``opti from DaVinci import Options, make_config from DaVinci.algorithms import add_filter from PyConf.Algorithms import PrintDecayTree - from PyConf.dataflow import force_location + from Davinci.reco_ocjects import get_particles def print_decay_tree(options: Options): turbo_line = "Hlt2BsToJpsiPhi_JPsi2MuMu_PhiToKK_Line" - input_data = force_location(f"/Event/HLT2/{turbo_line}/Particles") + input_data = get_particles(f"/Event/HLT2/{turbo_line}/Particles") user_algorithms = [ add_filter(options, "HDRFilter_SeeNoEvil", f"HLT_PASS('{turbo_line}')"), -- GitLab