CTA gRPC Frontend crashing when multiple Retrieve requests are submitted simultaneously
Details can be found in the community issue: https://cta-community.web.cern.ch/t/downgrading-just-grpc-frontend/347
Stacktrace:
#0 0x00007f4be754d1de _ZNKSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE7compareERKS4_ (libstdc++.so.6 + 0x14d1de)
#1 0x000000000047a6a0 _ZStltIcSt11char_traitsIcESaIcEEbRKNSt7__cxx1112basic_stringIT_T0_T1_EESA_ (cta-frontend-grpc + 0x7a6a0)
#2 0x000000000047b5b0 _ZNKSt4lessINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEEclERKS5_S8_ (cta-frontend-grpc + 0x7b5b0)
#3 0x000000000048b5f7 _ZNKSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES5_St9_IdentityIS5_ESt4lessIS5_ESaIS5_EE14_M_lower_boundEPKSt13_Rb_tree_nodeIS5_EPKSt18_Rb_tree_node_baseRKS5_ (cta-frontend-grpc + 0x8b5f7)
#4 0x0000000000489c14 _ZNKSt8_Rb_treeINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES5_St9_IdentityIS5_ESt4lessIS5_ESaIS5_EE4findERKS5_ (cta-frontend-grpc + 0x89c14)
#5 0x00007f4becbc498a _ZNKSt3setINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4lessIS5_ESaIS5_EE4findERKS5_ (libctacommon.so.0 + 0x1c498a)
#6 0x00007f4becbc4533 _ZNK3cta3log10LogContext16ParamNameMatcherclERKNS0_5ParamE (libctacommon.so.0 + 0x1c4533)
#7 0x00007f4becbc6dc9 _ZN9__gnu_cxx5__ops10_Iter_predIN3cta3log10LogContext16ParamNameMatcherEEclISt14_List_iteratorINS3_5ParamEEEEbT_ (libctacommon.so.0 + 0x1c6dc9)
#8 0x00007f4becbc6232 _ZSt11__remove_ifISt14_List_iteratorIN3cta3log5ParamEEN9__gnu_cxx5__ops10_Iter_predINS2_10LogContext16ParamNameMatcherEEEET_SB_SB_T0_ (libctacommon.so.0 + 0x1c6232)
#9 0x00007f4becbc50ec _ZSt9remove_ifISt14_List_iteratorIN3cta3log5ParamEENS2_10LogContext16ParamNameMatcherEET_S7_S7_T0_ (libctacommon.so.0 + 0x1c50ec)
#10 0x00007f4becbc3b34 _ZN3cta3log10LogContext5eraseERKSt3setINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt4lessIS8_ESaIS8_EE (libctacommon.so.0 + 0x1c3b34)
#11 0x00000000004795b7 _ZN3cta3log20ScopedParamContainerD2Ev (cta-frontend-grpc + 0x795b7)
#12 0x00000000004787cb _ZN3cta8frontend4grpc10CtaRpcImpl8RetrieveEPN4grpc13ServerContextEPKNS_3xrd7RequestEPNS6_8ResponseE (cta-frontend-grpc + 0x787cb)
#13 0x00000000004fbf5e _ZZN3cta3xrd6CtaRpc7ServiceC4EvENKUlPS2_PN4grpc13ServerContextEPKNS0_7RequestEPNS0_8ResponseEE2_clES3_S6_S9_SB_ (cta-frontend-grpc + 0xfbf5e)
#14 0x0000000000500ec0 __invoke_impl<grpc::Status, cta::xrd::CtaRpc::Service::Service()::<lambda(cta::xrd::CtaRpc::Service*, grpc::ServerContext*, const cta::xrd::Request*, cta::xrd::Response*)>&, cta::xrd::CtaRpc::Service*, grpc::ServerContext*, const cta::xrd::Request*, cta::xrd::Response*> (cta-frontend-grpc + 0x100ec0)
#15 0x00000000004ffa6b __invoke_r<grpc::Status, cta::xrd::CtaRpc::Service::Service()::<lambda(cta::xrd::CtaRpc::Service*, grpc::ServerContext*, const cta::xrd::Request*, cta::xrd::Response*)>&, cta::xrd::CtaRpc::Service*, grpc::ServerContext*, const cta::xrd::Request*, cta::xrd::Response*> (cta-frontend-grpc + 0xffa6b)
#16 0x00000000004fe6b6 _M_invoke (cta-frontend-grpc + 0xfe6b6)
#17 0x000000000052d2b1 _ZNKSt8functionIFN4grpc6StatusEPN3cta3xrd6CtaRpc7ServiceEPNS0_13ServerContextEPKNS3_7RequestEPNS3_8ResponseEEEclES6_S8_SB_SD_ (cta-frontend-grpc + 0x12d2b1)
#18 0x0000000000524773 _ZZN4grpc8internal16RpcMethodHandlerIN3cta3xrd6CtaRpc7ServiceENS3_7RequestENS3_8ResponseEN6google8protobuf11MessageLiteESA_E10RunHandlerERKNS0_13MethodHandler16HandlerParameterEENKUlvE_clEv (cta-frontend-grpc + 0x124773)
#19 0x000000000052d309 _ZN4grpc8internal23CatchingFunctionHandlerIZNS0_16RpcMethodHandlerIN3cta3xrd6CtaRpc7ServiceENS4_7RequestENS4_8ResponseEN6google8protobuf11MessageLiteESB_E10RunHandlerERKNS0_13MethodHandler16HandlerParameterEEUlvE_EENS_6StatusEOT_ (cta-frontend-grpc + 0x12d309)
#20 0x0000000000524839 _ZN4grpc8internal16RpcMethodHandlerIN3cta3xrd6CtaRpc7ServiceENS3_7RequestENS3_8ResponseEN6google8protobuf11MessageLiteESA_E10RunHandlerERKNS0_13MethodHandler16HandlerParameterE (cta-frontend-grpc + 0x124839)
#21 0x00007f4bef281c1d _ZN4grpc6Server11SyncRequest28ContinueRunAfterInterceptionEv (libgrpc++.so.1.46 + 0xb1c1d)
#22 0x00007f4bef2884dc _ZN4grpc13ThreadManager12MainWorkLoopEv (libgrpc++.so.1.46 + 0xb84dc)
#23 0x00007f4bef288610 _ZN4grpc13ThreadManager12WorkerThread3RunEv (libgrpc++.so.1.46 + 0xb8610)
#24 0x00007f4bef173f81 _ZZN9grpc_core12_GLOBAL__N_120ThreadInternalsPosixC4EPKcPFvPvES4_PbRKNS_6Thread7OptionsEENUlS4_E_4_FUNES4_ (libgpr.so.24 + 0x11f81)
#25 0x00007f4be7089c02 start_thread (libc.so.6 + 0x89c02)
#26 0x00007f4be710ec40 __clone3 (libc.so.6 + 0x10ec40)
Stack trace of thread 2619011:
#0 0x00007f4be710e21e epoll_wait (libc.so.6 + 0x10e21e)
#1 0x00007f4be63fe748 _ZN11EpollDriver10event_waitERSt6vectorI14FiredFileEventSaIS1_EEP7timeval (libceph-common.so.2 + 0x3fe748)
#2 0x00007f4be63fcda6 _ZN11EventCenter14process_eventsEjPNSt6chrono8durationImSt5ratioILl1ELl1000000000EEEE (libceph-common.so.2 + 0x3fcda6)
#3 0x00007f4be63fd916 _ZNSt17_Function_handlerIFvvEZN12NetworkStack10add_threadEP6WorkerEUlvE_E9_M_invokeERKSt9_Any_data (libceph-common.so.2 + 0x3fd916)
#4 0x00007f4be74dbad4 execute_native_thread_routine (libstdc++.so.6 + 0xdbad4)
#5 0x00007f4be7089c02 start_thread (libc.so.6 + 0x89c02)
#6 0x00007f4be710ec40 __clone3 (libc.so.6 + 0x10ec40)```