Random crashes when registering metrics variables
Dear @engamber,
I've noticed that I often getting crashes during configuration when new metrics variables are registered, but it is not fully reproducible. The trace back is below and I suspect it is because the vector holding the metrics variables is being extended while the publishing is on-going causing the use of deleted memory.
Cheers, Brian
2019-10-22 10:03:03.799 [core] [info] [9580] [Statistics.cpp:52] daqling::core::Statistics::CheckStatistics() Statistics thread about to spawn...
=================================================================
==9530==ERROR: AddressSanitizer: heap-use-after-free on address 0x60600000c390 at pc 0x7f38622cc096 bp 0x7f38526be520 sp 0x7f38526be518
READ of size 8 at 0x60600000c390 thread T12
#0 0x7f38622cc095 in std::__shared_ptr<daqling::core::Metric_base, (__gnu_cxx::_Lock_policy)2>::get() const /opt/ohpc/pub/compiler/gcc/8.3.0/include/c++/8.3.0/bits/shared_ptr_base.h:1308
#1 0x7f38622c98a9 in daqling::core::Statistics::CheckStatistics() /home/aagaard/daq/daqling/src/Core/Statistics.cpp:57
#2 0x7f38622f3dfa in void std::__invoke_impl<void, void (daqling::core::Statistics::*)(), daqling::core::Statistics*>(std::__invoke_memfun_deref, void (daqling::core::Statistics::*&&)(), daqling::core::Statistics*&&) /opt/ohpc/pub/compiler/gcc/8.3.0/include/c++/8.3.0/bits/invoke.h:73
#3 0x7f38622cb588 in std::__invoke_result<void (daqling::core::Statistics::*)(), daqling::core::Statistics*>::type std::__invoke<void (daqling::core::Statistics::*)(), daqling::core::Statistics*>(void (daqling::core::Statistics::*&&)(), daqling::core::Statistics*&&) /opt/ohpc/pub/compiler/gcc/8.3.0/include/c++/8.3.0/bits/invoke.h:95
#4 0x7f3862315f91 in decltype (__invoke((_S_declval<0ul>)(), (_S_declval<1ul>)())) std::thread::_Invoker<std::tuple<void (daqling::core::Statistics::*)(), daqling::core::Statistics*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /opt/ohpc/pub/compiler/gcc/8.3.0/include/c++/8.3.0/thread:244
#5 0x7f3862315e2a in std::thread::_Invoker<std::tuple<void (daqling::core::Statistics::*)(), daqling::core::Statistics*> >::operator()() /opt/ohpc/pub/compiler/gcc/8.3.0/include/c++/8.3.0/thread:253
#6 0x7f3862315de4 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (daqling::core::Statistics::*)(), daqling::core::Statistics*> > >::_M_run() /opt/ohpc/pub/compiler/gcc/8.3.0/include/c++/8.3.0/thread:196
#7 0x7f3860d40d9e in execute_native_thread_routine ../../../../../libstdc++-v3/src/c++11/thread.cc:80
#8 0x7f385f837e64 in start_thread (/lib64/libpthread.so.0+0x7e64)
#9 0x7f385f56088c in clone (/lib64/libc.so.6+0xfe88c)
0x60600000c390 is located 16 bytes inside of 64-byte region [0x60600000c380,0x60600000c3c0)
freed by thread T3 here:
#0 0x7f3862f59f00 in operator delete(void*) ../../../../libsanitizer/asan/asan_new_delete.cc:135
#1 0x7f3862305749 in __gnu_cxx::new_allocator<std::shared_ptr<daqling::core::Metric_base> >::deallocate(std::shared_ptr<daqling::core::Metric_base>*, unsigned long) /opt/ohpc/pub/compiler/gcc/8.3.0/include/c++/8.3.0/ext/new_allocator.h:125
#2 0x7f38622ffd71 in std::allocator_traits<std::allocator<std::shared_ptr<daqling::core::Metric_base> > >::deallocate(std::allocator<std::shared_ptr<daqling::core::Metric_base> >&, std::shared_ptr<daqling::core::Metric_base>*, unsigned long) /opt/ohpc/pub/compiler/gcc/8.3.0/include/c++/8.3.0/bits/alloc_traits.h:462
#3 0x7f38622f864d in std::_Vector_base<std::shared_ptr<daqling::core::Metric_base>, std::allocator<std::shared_ptr<daqling::core::Metric_base> > >::_M_deallocate(std::shared_ptr<daqling::core::Metric_base>*, unsigned long) /opt/ohpc/pub/compiler/gcc/8.3.0/include/c++/8.3.0/bits/stl_vector.h:304
#4 0x7f3853bef343 in void std::vector<std::shared_ptr<daqling::core::Metric_base>, std::allocator<std::shared_ptr<daqling::core::Metric_base> > >::_M_realloc_insert<std::shared_ptr<daqling::core::Metric_base> const&>(__gnu_cxx::__normal_iterator<std::shared_ptr<daqling::core::Metric_base>*, std::vector<std::shared_ptr<daqling::core::Metric_base>, std::allocator<std::shared_ptr<daqling::core::Metric_base> > > >, std::shared_ptr<daqling::core::Metric_base> const&) /opt/ohpc/pub/compiler/gcc/8.3.0/include/c++/8.3.0/bits/vector.tcc:469
#5 0x7f3853bd47ad in std::vector<std::shared_ptr<daqling::core::Metric_base>, std::allocator<std::shared_ptr<daqling::core::Metric_base> > >::push_back(std::shared_ptr<daqling::core::Metric_base> const&) /opt/ohpc/pub/compiler/gcc/8.3.0/include/c++/8.3.0/bits/stl_vector.h:1085
#6 0x7f3853bba7e3 in void daqling::core::Statistics::registerVariable<std::atomic<int>, int>(std::atomic<int>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, daqling::core::metrics::metric_type, daqling::core::metrics::variable_type, float) /home/aagaard/daq/daqling/src/Core/Statistics.hpp:73
#7 0x7f3853ba4f8d in FaserProcess::registerVariable(std::atomic<int>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, daqling::core::metrics::metric_type, float) /home/aagaard/daq/src/Commons/FaserProcess.hpp:56
#8 0x7f3853b796e3 in EventBuilderFaserModule::configure() /home/aagaard/daq/src/Modules/EventBuilderFaser/EventBuilderFaserModule.cpp:32