Crash in VELO tracking

This crash was observed when processing the lead data with HLT2, see https://lblogbook.cern.ch/HLT/483 .

libPrPixel.so!LHCb::Event::NumericProxy<(SIMDWrapper::InstructionSet)2, (LHCb::Pr::ProxyBehaviour)1, LHCb::Pr::Velo::(anonymous namespace)::HitsPlane, LHCb::Event::SOAPath<LHCb::Pr::Velo::(anonymous namespace)::HitsTag::pos, unsigned long>, float, false>::get(const LHCb::Event::NumericProxy<(SIMDWrapper::InstructionSet)2, (LHCb::Pr::ProxyBehaviour)1, LHCb::Pr::Velo::(anonymous namespace)::HitsPlane, LHCb::Event::SOAPath<LHCb::Pr::Velo::(anonymous namespace)::HitsTag::pos, unsigned long>, float, false> * const this) (/home/rmatev/stack/LHCb/InstallArea/x86_64_v2-el9-gcc12-dbg/include/Event/Proxy.h:174)
libPrPixel.so!LHCb::Event::Vec_field<3>::VecProxy<(SIMDWrapper::InstructionSet)2, (LHCb::Pr::ProxyBehaviour)1, LHCb::Pr::Velo::(anonymous namespace)::HitsPlane, LHCb::Event::SOAPath<LHCb::Pr::Velo::(anonymous namespace)::HitsTag::pos> >::get(const LHCb::Event::Vec_field<3>::VecProxy<(SIMDWrapper::InstructionSet)2, (LHCb::Pr::ProxyBehaviour)1, LHCb::Pr::Velo::(anonymous namespace)::HitsPlane, LHCb::Event::SOAPath<LHCb::Pr::Velo::(anonymous namespace)::HitsTag::pos> > * const this) (/home/rmatev/stack/LHCb/InstallArea/x86_64_v2-el9-gcc12-dbg/include/Event/SIMDEventTypes.h:398)
libPrPixel.so!LHCb::Event::Proxy<(SIMDWrapper::InstructionSet)2, (LHCb::Pr::ProxyBehaviour)1, LHCb::Pr::Velo::(anonymous namespace)::HitsPlane>::get<LHCb::Pr::Velo::(anonymous namespace)::HitsTag::pos>(const LHCb::Event::Proxy<(SIMDWrapper::InstructionSet)2, (LHCb::Pr::ProxyBehaviour)1, LHCb::Pr::Velo::(anonymous namespace)::HitsPlane> * const this) (/home/rmatev/stack/LHCb/InstallArea/x86_64_v2-el9-gcc12-dbg/include/Event/Proxy.h:726)
libPrPixel.so!LHCb::Event::ZipProxy<LHCb::Event::Proxy<(SIMDWrapper::InstructionSet)2, (LHCb::Pr::ProxyBehaviour)1, LHCb::Pr::Velo::(anonymous namespace)::HitsPlane> >::get<LHCb::Pr::Velo::(anonymous namespace)::HitsTag::pos>(const LHCb::Event::ZipProxy<LHCb::Event::Proxy<(SIMDWrapper::InstructionSet)2, (LHCb::Pr::ProxyBehaviour)1, LHCb::Pr::Velo::(anonymous namespace)::HitsPlane> > * const this) (/home/rmatev/stack/LHCb/InstallArea/x86_64_v2-el9-gcc12-dbg/include/Event/Proxy.h:849)
libPrPixel.so!LHCb::Pr::Velo::(anonymous namespace)::closestsHitsInPhi<3, SIMDWrapper::sse::float_v, SIMDWrapper::sse::int_v>(SIMDWrapper::sse::int_v * p0_idx, LHCb::LinAlg::Vec<SIMDWrapper::sse::float_v, 3> * p0_pos, LHCb::Pr::Velo::(anonymous namespace)::HitsPlane * P0) (/home/rmatev/stack/Rec/Pr/PrPixel/src/VeloClusterTrackingSIMD.cpp:594)
libPrPixel.so!LHCb::Pr::Velo::ClusterTrackingSIMD<(LHCb::RawBank::BankType)63, (LHCb::Pr::Velo::SearchMode)0>::TrackSeeding(const LHCb::Pr::Velo::ClusterTrackingSIMD<(LHCb::RawBank::BankType)63, (LHCb::Pr::Velo::SearchMode)0> * const this, LHCb::Pr::Velo::(anonymous namespace)::HitsPlane * P0, LHCb::Pr::Velo::(anonymous namespace)::HitsPlane * P1, LHCb::Pr::Velo::(anonymous namespace)::HitsPlane * P2, LHCb::Pr::Velo::(anonymous namespace)::LightTracksSoA * tracks) (/home/rmatev/stack/Rec/Pr/PrPixel/src/VeloClusterTrackingSIMD.cpp:675)
libPrPixel.so!LHCb::Pr::Velo::ClusterTrackingSIMD<(LHCb::RawBank::BankType)63, (LHCb::Pr::Velo::SearchMode)0>::operator()(const LHCb::Pr::Velo::ClusterTrackingSIMD<(LHCb::RawBank::BankType)63, (LHCb::Pr::Velo::SearchMode)0> * const this,  evtCtx,  tBanks, const DeVP & devp) (/home/rmatev/stack/Rec/Pr/PrPixel/src/VeloClusterTrackingSIMD.cpp:1081)
libPrPixel.so!_ZZN5Gaudi10Functional7details19filter_evtcontext_tIJ12EventContextN3gsl4spanIKPKN4LHCb7RawBankELm18446744073709551615EEENS6_8Detector4DeVPEEE5applyINS1_16MultiTransformerIFSt5tupleIJNS6_2Pr4HitsILNSI_7HitTypeE0EEENSI_4Velo6TracksESN_EERKS3_RKSB_RKSD_ENS0_6Traits4use_IJNS6_3Det8LbDD4hep21useConditionHandleForIJSD_EEENSW_11BaseClass_tINSZ_23ConditionAccessorHolderI10FixTESPathINS_9AlgorithmEEEEEEEEELb0EEEKSH_IJ20DataObjectReadHandleISB_ENSZ_17ConditionAccessorISD_EEEEEEDaRKT_SQ_RT0_ENKUlDpRKT_E_clIJS1C_S1E_EEEDaS1P_(const struct {...} * const __closure) (/home/rmatev/stack/Gaudi/InstallArea/x86_64_v2-el9-gcc12-dbg/include/GaudiAlg/FunctionalDetails.h:491)
libPrPixel.so!_ZSt13__invoke_implISt5tupleIJN4LHCb2Pr4HitsILNS2_7HitTypeE0EEENS2_4Velo6TracksES7_EEZN5Gaudi10Functional7details19filter_evtcontext_tIJ12EventContextN3gsl4spanIKPKNS1_7RawBankELm18446744073709551615EEENS1_8Detector4DeVPEEE5applyINSB_16MultiTransformerIFS8_RKSD_RKSK_RKSM_ENSA_6Traits4use_IJNS1_3Det8LbDD4hep21useConditionHandleForIJSM_EEENSX_11BaseClass_tINS10_23ConditionAccessorHolderI10FixTESPathINS9_9AlgorithmEEEEEEEEELb0EEEKS0_IJ20DataObjectReadHandleISK_ENS10_17ConditionAccessorISM_EEEEEEDaRKT_SR_RT0_EUlDpRKT_E_JRKS1D_RKS1F_EES1I_St14__invoke_otherOS1L_DpOT1_(struct {...} && __f) (/cvmfs/lhcb.cern.ch/lib/lcg/releases/gcc/12.1.0-57c96/x86_64-centos9/include/c++/12.1.0/bits/invoke.h:61)
libPrPixel.so!_ZSt8__invokeIZN5Gaudi10Functional7details19filter_evtcontext_tIJ12EventContextN3gsl4spanIKPKN4LHCb7RawBankELm18446744073709551615EEENS7_8Detector4DeVPEEE5applyINS2_16MultiTransformerIFSt5tupleIJNS7_2Pr4HitsILNSJ_7HitTypeE0EEENSJ_4Velo6TracksESO_EERKS4_RKSC_RKSE_ENS1_6Traits4use_IJNS7_3Det8LbDD4hep21useConditionHandleForIJSE_EEENSX_11BaseClass_tINS10_23ConditionAccessorHolderI10FixTESPathINS0_9AlgorithmEEEEEEEEELb0EEEKSI_IJ20DataObjectReadHandleISC_ENS10_17ConditionAccessorISE_EEEEEEDaRKT_SR_RT0_EUlDpRKT_E_JRKS1D_RKS1F_EENSt15__invoke_resultIS1I_JDpT0_EE4typeEOS1I_DpOS1X_(struct {...} && __fn) (/cvmfs/lhcb.cern.ch/lib/lcg/releases/gcc/12.1.0-57c96/x86_64-centos9/include/c++/12.1.0/bits/invoke.h:97)
libPrPixel.so!_ZSt12__apply_implIZN5Gaudi10Functional7details19filter_evtcontext_tIJ12EventContextN3gsl4spanIKPKN4LHCb7RawBankELm18446744073709551615EEENS7_8Detector4DeVPEEE5applyINS2_16MultiTransformerIFSt5tupleIJNS7_2Pr4HitsILNSJ_7HitTypeE0EEENSJ_4Velo6TracksESO_EERKS4_RKSC_RKSE_ENS1_6Traits4use_IJNS7_3Det8LbDD4hep21useConditionHandleForIJSE_EEENSX_11BaseClass_tINS10_23ConditionAccessorHolderI10FixTESPathINS0_9AlgorithmEEEEEEEEELb0EEEKSI_IJ20DataObjectReadHandleISC_ENS10_17ConditionAccessorISE_EEEEEEDaRKT_SR_RT0_EUlDpRKT_E_RS1H_JLm0ELm1EEEDcOS1I_OS1L_St16integer_sequenceImJXspT1_EEE( __t, struct {...} && __f) (/cvmfs/lhcb.cern.ch/lib/lcg/releases/gcc/12.1.0-57c96/x86_64-centos9/include/c++/12.1.0/tuple:1853)
libPrPixel.so!_ZSt5applyIZN5Gaudi10Functional7details19filter_evtcontext_tIJ12EventContextN3gsl4spanIKPKN4LHCb7RawBankELm18446744073709551615EEENS7_8Detector4DeVPEEE5applyINS2_16MultiTransformerIFSt5tupleIJNS7_2Pr4HitsILNSJ_7HitTypeE0EEENSJ_4Velo6TracksESO_EERKS4_RKSC_RKSE_ENS1_6Traits4use_IJNS7_3Det8LbDD4hep21useConditionHandleForIJSE_EEENSX_11BaseClass_tINS10_23ConditionAccessorHolderI10FixTESPathINS0_9AlgorithmEEEEEEEEELb0EEEKSI_IJ20DataObjectReadHandleISC_ENS10_17ConditionAccessorISE_EEEEEEDaRKT_SR_RT0_EUlDpRKT_E_RS1H_EDcOS1I_OS1L_( __t, struct {...} && __f) (/cvmfs/lhcb.cern.ch/lib/lcg/releases/gcc/12.1.0-57c96/x86_64-centos9/include/c++/12.1.0/tuple:1865)
libPrPixel.so!Gaudi::Functional::details::filter_evtcontext_t<EventContext, gsl::span<LHCb::RawBank const* const, 18446744073709551615ul>, LHCb::Detector::DeVP>::apply<Gaudi::Functional::details::MultiTransformer<std::tuple<LHCb::Pr::Hits<(LHCb::Pr::HitType)0>, LHCb::Pr::Velo::Tracks, LHCb::Pr::Velo::Tracks> (EventContext const&, gsl::span<LHCb::RawBank const* const, 18446744073709551615ul> const&, LHCb::Detector::DeVP const&), Gaudi::Functional::Traits::use_<LHCb::Det::LbDD4hep::useConditionHandleFor<LHCb::Detector::DeVP>, Gaudi::Functional::Traits::BaseClass_t<LHCb::Det::LbDD4hep::ConditionAccessorHolder<FixTESPath<Gaudi::Algorithm> > > >, false>, std::tuple<DataObjectReadHandle<gsl::span<LHCb::RawBank const* const, 18446744073709551615ul> >, LHCb::Det::LbDD4hep::ConditionAccessor<LHCb::Detector::DeVP> > const>(Gaudi::Functional::details::MultiTransformer<std::tuple<LHCb::Pr::Hits<(LHCb::Pr::HitType)0>, LHCb::Pr::Velo::Tracks, LHCb::Pr::Velo::Tracks> (EventContext const&, gsl::span<LHCb::RawBank const* const, 18446744073709551615ul> const&, LHCb::Detector::DeVP const&), Gaudi::Functional::Traits::use_<LHCb::Det::LbDD4hep::useConditionHandleFor<LHCb::Detector::DeVP>, Gaudi::Functional::Traits::BaseClass_t<LHCb::Det::LbDD4hep::ConditionAccessorHolder<FixTESPath<Gaudi::Algorithm> > > >, false> const&, EventContext const&, std::tuple<DataObjectReadHandle<gsl::span<LHCb::RawBank const* const, 18446744073709551615ul> >, LHCb::Det::LbDD4hep::ConditionAccessor<LHCb::Detector::DeVP> > const&)( handles,  ctx,  algo) (/home/rmatev/stack/Gaudi/InstallArea/x86_64_v2-el9-gcc12-dbg/include/GaudiAlg/FunctionalDetails.h:491)
libPrPixel.so!_ZZNK5Gaudi10Functional7details16MultiTransformerIFSt5tupleIJN4LHCb2Pr4HitsILNS5_7HitTypeE0EEENS5_4Velo6TracksESA_EERK12EventContextRKN3gsl4spanIKPKNS4_7RawBankELm18446744073709551615EEERKNS4_8Detector4DeVPEENS0_6Traits4use_IJNS4_3Det8LbDD4hep21useConditionHandleForIJSP_EEENST_11BaseClass_tINSW_23ConditionAccessorHolderI10FixTESPathINS_9AlgorithmEEEEEEEEELb0EE7executeESE_ENKUlDpRT_E_clIJK21DataObjectWriteHandleIS8_S8_EKS1D_ISA_SA_ES1H_EEEDaS1A_(const struct {...} * const __closure) (/home/rmatev/stack/Gaudi/InstallArea/x86_64_v2-el9-gcc12-dbg/include/GaudiAlg/Transformer.h:148)
libPrPixel.so!_ZSt13__invoke_implIvZNK5Gaudi10Functional7details16MultiTransformerIFSt5tupleIJN4LHCb2Pr4HitsILNS6_7HitTypeE0EEENS6_4Velo6TracksESB_EERK12EventContextRKN3gsl4spanIKPKNS5_7RawBankELm18446744073709551615EEERKNS5_8Detector4DeVPEENS1_6Traits4use_IJNS5_3Det8LbDD4hep21useConditionHandleForIJSQ_EEENSU_11BaseClass_tINSX_23ConditionAccessorHolderI10FixTESPathINS0_9AlgorithmEEEEEEEEELb0EE7executeESF_EUlDpRT_E_JRK21DataObjectWriteHandleIS9_S9_ERKS1D_ISB_SB_ES1J_EET_St14__invoke_otherOT0_DpOT1_(struct {...} && __f) (/cvmfs/lhcb.cern.ch/lib/lcg/releases/gcc/12.1.0-57c96/x86_64-centos9/include/c++/12.1.0/bits/invoke.h:61)
libPrPixel.so!_ZSt8__invokeIZNK5Gaudi10Functional7details16MultiTransformerIFSt5tupleIJN4LHCb2Pr4HitsILNS6_7HitTypeE0EEENS6_4Velo6TracksESB_EERK12EventContextRKN3gsl4spanIKPKNS5_7RawBankELm18446744073709551615EEERKNS5_8Detector4DeVPEENS1_6Traits4use_IJNS5_3Det8LbDD4hep21useConditionHandleForIJSQ_EEENSU_11BaseClass_tINSX_23ConditionAccessorHolderI10FixTESPathINS0_9AlgorithmEEEEEEEEELb0EE7executeESF_EUlDpRT_E_JRK21DataObjectWriteHandleIS9_S9_ERKS1D_ISB_SB_ES1J_EENSt15__invoke_resultIT_JDpT0_EE4typeEOS1L_DpOS1M_(struct {...} && __fn) (/cvmfs/lhcb.cern.ch/lib/lcg/releases/gcc/12.1.0-57c96/x86_64-centos9/include/c++/12.1.0/bits/invoke.h:96)
libPrPixel.so!_ZSt12__apply_implIZNK5Gaudi10Functional7details16MultiTransformerIFSt5tupleIJN4LHCb2Pr4HitsILNS6_7HitTypeE0EEENS6_4Velo6TracksESB_EERK12EventContextRKN3gsl4spanIKPKNS5_7RawBankELm18446744073709551615EEERKNS5_8Detector4DeVPEENS1_6Traits4use_IJNS5_3Det8LbDD4hep21useConditionHandleForIJSQ_EEENSU_11BaseClass_tINSX_23ConditionAccessorHolderI10FixTESPathINS0_9AlgorithmEEEEEEEEELb0EE7executeESF_EUlDpRT_E_RKS4_IJ21DataObjectWriteHandleIS9_S9_ES1D_ISB_SB_ES1F_EEJLm0ELm1ELm2EEEDcOT_OT0_St16integer_sequenceImJXspT1_EEE(const std::tuple<DataObjectWriteHandle<LHCb::Pr::Hits<(LHCb::Pr::HitType)0>, LHCb::Pr::Hits<(LHCb::Pr::HitType)0> >, DataObjectWriteHandle<LHCb::Pr::Velo::Tracks, LHCb::Pr::Velo::Tracks>, DataObjectWriteHandle<LHCb::Pr::Velo::Tracks, LHCb::Pr::Velo::Tracks> > & __t, struct {...} && __f) (/cvmfs/lhcb.cern.ch/lib/lcg/releases/gcc/12.1.0-57c96/x86_64-centos9/include/c++/12.1.0/tuple:1852)
libPrPixel.so!_ZSt5applyIZNK5Gaudi10Functional7details16MultiTransformerIFSt5tupleIJN4LHCb2Pr4HitsILNS6_7HitTypeE0EEENS6_4Velo6TracksESB_EERK12EventContextRKN3gsl4spanIKPKNS5_7RawBankELm18446744073709551615EEERKNS5_8Detector4DeVPEENS1_6Traits4use_IJNS5_3Det8LbDD4hep21useConditionHandleForIJSQ_EEENSU_11BaseClass_tINSX_23ConditionAccessorHolderI10FixTESPathINS0_9AlgorithmEEEEEEEEELb0EE7executeESF_EUlDpRT_E_RKS4_IJ21DataObjectWriteHandleIS9_S9_ES1D_ISB_SB_ES1F_EEEDcOT_OT0_(const std::tuple<DataObjectWriteHandle<LHCb::Pr::Hits<(LHCb::Pr::HitType)0>, LHCb::Pr::Hits<(LHCb::Pr::HitType)0> >, DataObjectWriteHandle<LHCb::Pr::Velo::Tracks, LHCb::Pr::Velo::Tracks>, DataObjectWriteHandle<LHCb::Pr::Velo::Tracks, LHCb::Pr::Velo::Tracks> > & __t, struct {...} && __f) (/cvmfs/lhcb.cern.ch/lib/lcg/releases/gcc/12.1.0-57c96/x86_64-centos9/include/c++/12.1.0/tuple:1863)
libPrPixel.so!Gaudi::Functional::details::MultiTransformer<std::tuple<LHCb::Pr::Hits<(LHCb::Pr::HitType)0>, LHCb::Pr::Velo::Tracks, LHCb::Pr::Velo::Tracks> (EventContext const&, gsl::span<LHCb::RawBank const* const, 18446744073709551615ul> const&, LHCb::Detector::DeVP const&), Gaudi::Functional::Traits::use_<LHCb::Det::LbDD4hep::useConditionHandleFor<LHCb::Detector::DeVP>, Gaudi::Functional::Traits::BaseClass_t<LHCb::Det::LbDD4hep::ConditionAccessorHolder<FixTESPath<Gaudi::Algorithm> > > >, false>::execute(EventContext const&) const(const Gaudi::Functional::details::MultiTransformer<std::tuple<LHCb::Pr::Hits<(LHCb::Pr::HitType)0>, LHCb::Pr::Velo::Tracks, LHCb::Pr::Velo::Tracks>(const EventContext&, const gsl::span<const LHCb::RawBank* const, 18446744073709551615>&, const LHCb::Detector::DeVP&), Gaudi::Functional::Traits::use_<LHCb::Det::LbDD4hep::useConditionHandleFor<LHCb::Detector::DeVP>, Gaudi::Functional::Traits::BaseClass_t<LHCb::Det::LbDD4hep::ConditionAccessorHolder<FixTESPath<Gaudi::Algorithm> > > >, false> * const this,  ctx) (/home/rmatev/stack/Gaudi/InstallArea/x86_64_v2-el9-gcc12-dbg/include/GaudiAlg/Transformer.h:137)
libAlgflowManagerComp.so!Online::AlgWrapper::execute(const Online::AlgWrapper * const this, EventContext & evtCtx, gsl::span<LHCb::Interfaces::ISchedulerConfiguration::State::AlgState, 18446744073709551615> AlgoStates) (/home/rmatev/stack/Online/Online/AlgFlowManager/components/ControlFlowNode.h:184)

Input data is in

/eos/lhcb/wg/rta/samples/data/279201-LHCb-RAW-errors/errors.mdf

and the crash can be reproduced with

MooreOnline/run MooreOnline/MooreScripts/scripts/testbench.py MooreOnline/MooreScripts/tests/options/HLT2/Arch.xml --data-dir=/eos/lhcb/wg/rta/samples/data/279201-LHCb-RAW-errors --measure-throughput=0 --hlt-type=hlt2_PbPb_default --working-dir=/tmp/wd-hlt2

A more straightforward way to reproduce must be possible with

Moore/run gaudirun.py my_input.py Moore/Hlt/Hlt2Conf/options/hlt2_PbPb_default.py

Edited Dec 11, 2023 by Rosen Matev