Commit fd97df4b authored by Christopher Rob Jones's avatar Christopher Rob Jones Committed by Charles Leggett
Browse files

Add build and runtime support for gcc/clang sanitizers

parent 69f3f1ff
......@@ -23,3 +23,47 @@ gaudi_add_test(nose
COMMAND nosetests --with-doctest -v
${CMAKE_CURRENT_SOURCE_DIR}/tests/nose
${CMAKE_CURRENT_SOURCE_DIR}/python/Gaudi)
# Settings relating to sanitizer builds
if (SANITIZER_ENABLED)
message(STATUS "Enabled sanitizer ${SANITIZER_ENABLED}")
# Enable automatic LD_PRELOAD
gaudi_env( SET PRELOAD_SANITIZER_LIB ${SANITIZER_ENABLED})
gaudi_build_env(SET PRELOAD_SANITIZER_LIB ${SANITIZER_ENABLED})
# Address sanitizer runtime options
set(GAUDI_ASAN_OPTS detect_leaks=0,alloc_dealloc_mismatch=0,halt_on_error=0,suppressions=\${GAUDIROOT}/job/Gaudi-ASan.supp
CACHE STRING "Runtime options for AddressSanitizer")
message(STATUS "ASAN OPTIONS ${GAUDI_ASAN_OPTS}")
gaudi_env( SET ASAN_OPTIONS ${GAUDI_ASAN_OPTS} )
gaudi_build_env(SET ASAN_OPTIONS ${GAUDI_ASAN_OPTS} )
# Leak sanitizer runtime options
set(GAUDI_LSAN_OPTS print_suppressions=0,halt_on_error=0,suppressions=\${GAUDIROOT}/job/Gaudi-LSan.supp
CACHE STRING "Runtime options for LeakSanitizer")
message(STATUS "LSAN OPTIONS ${GAUDI_LSAN_OPTS}")
gaudi_env( SET LSAN_OPTIONS ${GAUDI_LSAN_OPTS} )
gaudi_build_env(SET LSAN_OPTIONS ${GAUDI_LSAN_OPTS} )
# Thread sanitizer runtime options
# Note this sanitizer is known to prefer to be run with *everything* built using the
# sanitizer, which is not possible here (i.e. LCG externals). See
# https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual#non-instrumented-code
# Using the options ignore_interceptors_accesses=1,ignore_noninstrumented_modules=1
# seems to help reduce the noise a bit, but might remove real issues..
set(GAUDI_TSAN_OPTS print_suppressions=0,halt_on_error=0,ignore_interceptors_accesses=1,ignore_noninstrumented_modules=1,suppressions=\${GAUDIROOT}/job/Gaudi-TSan.supp
CACHE STRING "Runtime options for ThreadSanitizer")
message(STATUS "TSAN OPTIONS ${GAUDI_TSAN_OPTS}")
gaudi_env( SET TSAN_OPTIONS ${GAUDI_TSAN_OPTS} )
gaudi_build_env(SET TSAN_OPTIONS ${GAUDI_TSAN_OPTS} )
# Undefined Behaviour sanitizer runtime options
set(GAUDI_UBSAN_OPTS print_stacktrace=1,print_suppressions=0,halt_on_error=0,suppressions=\${GAUDIROOT}/job/Gaudi-UBSan.supp
CACHE STRING "Runtime options for UndefinedSanitizer")
message(STATUS "UBSAN OPTIONS ${GAUDI_UBSAN_OPTS}")
gaudi_env( SET UBSAN_OPTIONS ${GAUDI_UBSAN_OPTS} )
gaudi_build_env(SET UBSAN_OPTIONS ${GAUDI_UBSAN_OPTS} )
endif()
# --------------------------------------------------
# Address sanitizer specific suppressions
# http://clang.llvm.org/docs/AddressSanitizer.html
#
# Note the leak sanitizer options in Gaudi-LSan.supp
# are also used by the address sanitizer.
# --------------------------------------------------
# Nothing yet ..
# --------------------------------------------------
# Leak sanitizer suppressions
# http://clang.llvm.org/docs/LeakSanitizer.html
# --------------------------------------------------
# PyROOT
leak:PyROOT::MethodProxy_New
leak:PyROOT::CreateConverter
leak:PyROOT::CreateScopeProxy
leak:PyROOT::CreateExecutor
leak:PyROOT::TMethodHolder
leak:PyROOT::TClassMethodHolder::Clone
leak:PyROOT::MethodProxy::MethodInfo_t::MethodInfo_t
# Python
leak:PyEval_EvalFrameEx
leak:^PyObject_Call$
leak:^builtin_hasattr$
# ROOT
leak:TClingLookupHelper__ExistingTypeCheck
leak:^mp_new$
leak:^pp_new$
leak:^mp_setthreaded$
leak:TFormula::HandleParamRanges
# XrootD
leak:XrdSys
# Xerces
leak:xercesc_3_1::MemoryManagerImpl::allocate
# TBB
leak:tbb::internal::task_stream
# xgboost
leak:libxgboost.so
# These are certainly not leaks from std::string, but a limitation
# of the traceback coming out of the LCG externals.
# Perhaps would help if the debug builds used -fno-omit-frame-pointer
# Perhaps coming out of python in some way ...
# Suppressed to reduce noise ...
#
# Direct leak of 745 byte(s) in 24 object(s) allocated from:
# #0 0x7f895ee8a116 in operator new(unsigned long) ../../../../gcc-7.3.0/libsanitizer/lsan/lsan_interceptors.cc:162
# #1 0x7f895d90227e in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_assign(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) /build/pmendez-sftnight/build-730binutils/x86_64-pc-linux-gnu/libstdc++-v3/include/bits/basic_string.tcc:265
#
# Direct leak of 82 byte(s) in 1 object(s) allocated from:
# #0 0x7f895ee8a116 in operator new(unsigned long) ../../../../gcc-7.3.0/libsanitizer/lsan/lsan_interceptors.cc:162
# #1 0x7f895d8f4a2e in void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct<char*>(char*, char*, std::forward_iterator_tag) /build/pmendez-sftnight/build-730binutils/x86_64-pc-linux-gnu/libstdc++-v3/include/bits/basic_string.tcc:219
leak:std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_assign
leak:std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct
leak:std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::reserve
leak:std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_mutate
#
# Indirect leak of 36 byte(s) in 9 object(s) allocated from:
# #0 0x7fc2f0ecf116 in operator new(unsigned long) ../../../../gcc-7.3.0/libsanitizer/lsan/lsan_interceptors.cc:162
# #1 0x7fc2dc2d5cbb in std::vector<unsigned int, std::allocator<unsigned int> >::_M_fill_insert(__gnu_cxx::__normal_iterator<unsigned int*, std::vector<unsigned int, std::allocator<unsigned int> > >, unsigned long, unsigned int const&) (/cvmfs/lhcb.cern.ch/lib/lcg/releases/LCG_93/ROOT/6.12.06/x86_64-centos7-gcc7-dbg/lib/libCling.so+0xc3acbb)
leak:std::vector<unsigned int, std::allocator<unsigned int> >::_M_fill_insert
# Suppresses *everything* that comes via the runtime interceptor.
leak:_interceptor_
# --------------------------------------------------
# Thread sanitizer suppressions
# http://clang.llvm.org/docs/ThreadSanitizer.html
# --------------------------------------------------
# Nothing yet ..
# --------------------------------------------------
# Undefined Behaviour sanitizer suppressions
# http://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html
# --------------------------------------------------
# Boost - https://svn.boost.org/trac10/ticket/13233
vptr:boost/log/attributes/attribute_value.hpp
vptr:boost/smart_ptr/detail/shared_count.hpp
# Xerces / XMLFileCatalog
# /workspace/build/GAUDI/GAUDI_HEAD/GaudiUtils/src/component/XMLFileCatalog.cpp:84:67: runtime error: downcast of address 0x000001927a58 which does not point to an object of type 'DOMElement'
# 0x000001927a58: note: object is of type 'xercesc_3_1::DOMTextImpl'
# 00 00 00 00 a0 64 36 8c 0c 7f 00 00 38 78 92 01 00 00 00 00 58 04 00 00 00 00 00 00 c8 80 92 01
# ^~~~~~~~~~~~~~~~~~~~~~~
# vptr for 'xercesc_3_1::DOMTextImpl'
vptr:GaudiUtils/src/component/XMLFileCatalog.cpp
# STL ?
# /cvmfs/lhcb.cern.ch/lib/lcg/releases/gcc/7.3.0/x86_64-centos7/include/c++/7.3.0/bits/shared_ptr_base.h:154:16: runtime error: member call on address 0x0000016c3f90 which does not point to an object of type '_Sp_counted_base'
# 0x0000016c3f90: note: object has invalid vptr
# 00 00 00 00 00 b8 b6 4f 44 7f 00 00 00 00 00 00 01 00 00 00 20 74 72 79 69 6e 67 20 60 3e 6c 01
# ^~~~~~~~~~~~~~~~~~~~~~~
# invalid vptr
# Include gcc version so we don't automatically
# apply these to newer releases.
vptr:include/c++/7.3.0/bits/shared_ptr_base.h
# TBB
# /cvmfs/lhcb.cern.ch/lib/lcg/releases/tbb/2018_U1-d3621/x86_64-centos7-gcc7-dbg/include/tbb/task.h:780:34: runtime error: member call on address 0x7fccb4ac8e00 which does not point to an object of type 'scheduler'
# 0x7fccb4ac8e00: note: object is of type 'tbb::internal::custom_scheduler<tbb::internal::IntelSchedulerTraits>'
# 00 00 00 00 58 2a 22 cf cc 7f 00 00 00 00 00 00 00 00 00 00 60 f6 ac b4 cc 7f 00 00 60 f6 ac b4
# ^~~~~~~~~~~~~~~~~~~~~~~
# vptr for 'tbb::internal::custom_scheduler<tbb::internal::IntelSchedulerTraits>'
vptr:include/tbb/task.h
......@@ -274,9 +274,22 @@ if __name__ == "__main__":
InstallRootLoggingHandler(prefix, level=level, with_time=opts.debug)
root_logger = logging.getLogger()
# Sanitizer support
sanitizer = os.environ.get("PRELOAD_SANITIZER_LIB", "")
if sanitizer:
if sanitizer not in os.environ.get("LD_PRELOAD", ""):
opts.preload.insert(0, sanitizer)
os.environ["PRELOAD_SANITIZER_LIB"] = ""
# tcmalloc support
if opts.tcmalloc:
opts.preload.insert(0, os.environ.get("TCMALLOCLIB", "libtcmalloc.so"))
# Disable tcmalloc if sanitizer is selected
if sanitizer:
logging.warning("tcmalloc preload disabled when using a sanitizer")
else:
opts.preload.insert(0, os.environ.get(
"TCMALLOCLIB", "libtcmalloc.so"))
# allow preloading of libraries
if opts.preload:
preload = os.environ.get("LD_PRELOAD", "")
......
......@@ -57,4 +57,10 @@ EventWatchdog INFO The last event took ~10s
ApplicationMgr INFO Application Manager Stopped successfully
""", stdout = stdout)
</text></argument>
<argument name="unsupported_platforms"><set>
<text>asan</text>
<text>lsan</text>
<text>ubsan</text>
<text>tsan</text>
</set></argument>
</extension>
......@@ -38,4 +38,10 @@ if not 'in GaudiTesting::SleepyAlg::execute' in stderr:
causes.append('invalid stack trace')
</text></argument>
<argument name="unsupported_platforms"><set>
<text>asan</text>
<text>lsan</text>
<text>ubsan</text>
<text>tsan</text>
</set></argument>
</extension>
......@@ -22,7 +22,13 @@ ApplicationMgr(EvtMax=10,
<argument name="reference"><text>refs/google_auditors/cpu_profiler.ref</text></argument>
<argument name="error_reference"><text>refs/google_auditors/cpu_profiler_err.ref</text></argument>
<argument name="use_temp_dir"><enumeral>true</enumeral></argument>
<argument name="unsupported_platforms"><set><text>i686</text></set></argument>
<argument name="unsupported_platforms"><set>
<text>i686</text>
<text>asan</text>
<text>lsan</text>
<text>ubsan</text>
<text>tsan</text>
</set></argument>
<argument name="validator"><text>
# hide a difference between JobOptionsSvc and Configurables
preprocessor = normalizeExamples + \
......
......@@ -29,7 +29,13 @@ ApplicationMgr(EvtMax=10,
<argument name="reference"><text>refs/google_auditors/heap_checker.ref</text></argument>
<argument name="error_reference"><text>refs/google_auditors/heap_checker_err.ref</text></argument>
<argument name="use_temp_dir"><enumeral>true</enumeral></argument>
<argument name="unsupported_platforms"><set><text>i686</text></set></argument>
<argument name="unsupported_platforms"><set>
<text>i686</text>
<text>asan</text>
<text>lsan</text>
<text>ubsan</text>
<text>tsan</text>
</set></argument>
<argument name="validator"><text>
# hide a difference between JobOptionsSvc and Configurables
preprocessor = (normalizeExamples +
......
......@@ -22,5 +22,11 @@ ApplicationMgr(EvtMax=10,
<argument name="reference"><text>refs/google_auditors/heap_profiler.ref</text></argument>
<argument name="error_reference"><text>refs/google_auditors/heap_profiler_err.ref</text></argument>
<argument name="use_temp_dir"><enumeral>true</enumeral></argument>
<argument name="unsupported_platforms"><set><text>i686</text></set></argument>
<argument name="unsupported_platforms"><set>
<text>i686</text>
<text>asan</text>
<text>lsan</text>
<text>ubsan</text>
<text>tsan</text>
</set></argument>
</extension>
......@@ -20,4 +20,10 @@ app = ApplicationMgr(TopAlg = [sigalg(EventCount=3, Signal=signal.SIGSEGV)],
# no-op to avoid the check on the stderr
</text></argument>
<argument name="exit_code"><integer>139</integer></argument>
<argument name="unsupported_platforms"><set>
<text>asan</text>
<text>lsan</text>
<text>ubsan</text>
<text>tsan</text>
</set></argument>
</extension>
......@@ -80,8 +80,10 @@ if( CPPUNIT_FOUND )
endif()
gaudi_add_unit_test(test_GaudiTime tests/src/test_GaudiTime.cpp
LINK_LIBRARIES GaudiKernel)
gaudi_add_unit_test(test_GaudiTiming tests/src/test_GaudiTiming.cpp
LINK_LIBRARIES GaudiKernel)
if (NOT SANITIZER_ENABLED)
gaudi_add_unit_test(test_GaudiTiming tests/src/test_GaudiTiming.cpp
LINK_LIBRARIES GaudiKernel)
endif()
gaudi_add_unit_test(Parsers_test tests/src/parsers.cpp
LINK_LIBRARIES GaudiKernel)
gaudi_add_unit_test(Memory_test tests/src/Memory_test.cpp
......
......@@ -60,8 +60,10 @@ void StatusCode::check()
if ( System::backTrace( addresses, depth ) ) {
for ( size_t idx : {2, 3} ) {
if ( System::getStackLevel( addresses[idx], addr, fnc, lib ) && fnc != "StatusCode::~StatusCode()" ) {
// When running address sanitizer builds with -fno-omit-frame-pointer
// StatusCode::check() might appear as the function name, so skip.
if ( System::getStackLevel( addresses[idx], addr, fnc, lib ) && fnc != "StatusCode::~StatusCode()" &&
fnc != "StatusCode::check()" ) {
if ( scs ) {
scs->regFnc( fnc, lib );
} else {
......
......@@ -508,7 +508,10 @@ class BaseTest(object):
return ""
# function to split an extension in constituents parts
def platformSplit(p): return set(p.split('-' in p and '-' or '_'))
def platformSplit(p):
import re
delim = re.compile('-' in p and r"[-+]" or r"_")
return set(delim.split(p))
reference = os.path.normpath(os.path.join(self.basedir,
os.path.expandvars(reffile)))
......
......@@ -133,6 +133,15 @@ def main():
os.environ.get('CMTCONFIG', '')):
os.environ['TERM'] = 'dumb'
# If running sanitizer builds, set LD_PRELOAD in environment
sanitizer = os.environ.get('PRELOAD_SANITIZER_LIB', '')
ld_preload = os.environ.get('LD_PRELOAD', '')
if sanitizer and sanitizer not in ld_preload:
if ld_preload:
os.environ['LD_PRELOAD'] = sanitizer+" "+ld_preload
else:
os.environ['LD_PRELOAD'] = sanitizer
# Testing the file beginning with "Test" or if it is a qmt file and doing the test
logging.debug('processing %s', filename)
if filename.endswith('_test.py'):
......
......@@ -806,6 +806,7 @@ def main():
NamedMeasurementFile.write("<pre></pre>")
NamedMeasurementFile.close()
else:
executionTime = float(0)
# write the other files
for NamedMeasurement in Results.getiterator("NamedMeasurement"):
value = NamedMeasurement.find("Value")
......
......@@ -109,16 +109,20 @@ foreach(undef_symbol mallctl)
endforeach()
set_target_properties(GaudiJemalloc PROPERTIES LINK_FLAGS "${GaudiJemalloc_linker_flags}")
find_library(jemalloc_lib NAMES jemalloc)
if(jemalloc_lib)
message(STATUS "Found jemalloc: ${jemalloc_lib}")
gaudi_add_test(jira.gaudi_1045
COMMAND gaudirun.py
ENVIRONMENT LD_PRELOAD=${jemalloc_lib})
get_filename_component(jemalloc_libdir "${jemalloc_lib}" PATH)
gaudi_env(PREPEND LD_LIBRARY_PATH "${jemalloc_libdir}")
if (NOT SANITIZER_ENABLED)
find_library(jemalloc_lib NAMES jemalloc)
if(jemalloc_lib)
message(STATUS "Found jemalloc: ${jemalloc_lib}")
gaudi_add_test(jira.gaudi_1045
COMMAND gaudirun.py
ENVIRONMENT LD_PRELOAD=${jemalloc_lib})
get_filename_component(jemalloc_libdir "${jemalloc_lib}" PATH)
gaudi_env(PREPEND LD_LIBRARY_PATH "${jemalloc_libdir}")
else()
message(STATUS "jemalloc not found, cannot test GAUDI-1045")
endif()
else()
message(STATUS "jemalloc not found, cannot test GAUDI-1045")
message(STATUS "jemalloc test disabled when running a sanitizer")
endif()
#---Installation------------------------------------------------------------
......
......@@ -132,6 +132,22 @@ set(_opt_ext_DEBUG "-g")
set(_opt_level_RELWITHDEBINFO "${_opt_level_RELEASE}")
set(_opt_ext_RELWITHDEBINFO "${_opt_ext_RELEASE} -g")
# Sanitizer options
# http://clang.llvm.org/docs/AddressSanitizer.html
# http://clang.llvm.org/docs/LeakSanitizer.html
# http://clang.llvm.org/docs/ThreadSanitizer.html
# http://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html
# https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html
set(GAUDI_asan_FLAGS "-fsanitize=address -fsanitize-recover=all -fno-omit-frame-pointer -lasan"
CACHE STRING "Build options for AddressSanitizer")
set(GAUDI_lsan_FLAGS "-fsanitize=leak -fsanitize-recover=all -fno-omit-frame-pointer -llsan"
CACHE STRING "Build options for LeakSanitizer")
set(GAUDI_tsan_FLAGS "-fsanitize=thread -fsanitize-recover=all -fno-omit-frame-pointer -ltsan"
CACHE STRING "Build options for ThreadSanitizer")
set(GAUDI_ubsan_FLAGS "-fsanitize=undefined -fsanitize-recover=all -fno-omit-frame-pointer -lubsan"
CACHE STRING "Build options for UndefinedSanitizer")
set(GAUDI_DEFAULT_SANITIZER)
# - parse subtype flags
string(TOUPPER "${CMAKE_BUILD_TYPE}" _up_bt)
foreach(_subtype ${BINARY_TAG_SUBTYPE})
......@@ -142,9 +158,21 @@ foreach(_subtype ${BINARY_TAG_SUBTYPE})
set(_opt_ext_${_up_bt} "${_opt_ext_${_up_bt}} -g")
elseif(_subtype STREQUAL "cov")
set(_opt_ext_${_up_bt} "${_opt_ext_${_up_bt}} --coverage")
elseif(_subtype MATCHES "^(a|l|t|ub)san$")
set(GAUDI_DEFAULT_SANITIZER ${_subtype})
endif()
endforeach()
set(GAUDI_USE_SANITIZER ${GAUDI_DEFAULT_SANITIZER} CACHE STRING "Enabled given sanitizer")
if (GAUDI_USE_SANITIZER)
if(NOT GAUDI_USE_SANITIZER MATCHES "^(a|l|t|ub)san$")
message(FATAL_ERROR "Invalid sanitizer name: ${GAUDI_USE_SANITIZER}")
else()
set(SANITIZER_ENABLED "lib${GAUDI_USE_SANITIZER}.so")
set(_opt_ext_${_up_bt} "${_opt_ext_${_up_bt}} ${GAUDI_${GAUDI_USE_SANITIZER}_FLAGS}")
endif()
endif()
if(_opt_level_${_up_bt})
message(STATUS "Optimization: ${_opt_level_${_up_bt}} ${_opt_ext_${_up_bt}}")
endif()
......
......@@ -1783,12 +1783,19 @@ function(gaudi_generate_configurables library)
set(genconf_products ${genconf_products} ${outdir}/__init__.py)
endif()
# If a sanitizer is enabled force genconf to always return true status
# to allow builds to continue even if running genconf triggered a sanitizer error
set(genconf_force_status "")
if (SANITIZER_ENABLED)
set(genconf_force_status "||true")
endif()
add_custom_command(
OUTPUT ${genconf_products} ${outdir}/${library}.confdb
COMMAND ${env_cmd} --xml ${env_xml}
${genconf_cmd} ${library_preload} -o ${outdir} -p ${package}
${genconf_opts}
-i ${library}
-i ${library} ${genconf_force_status}
DEPENDS ${conf_depends})
add_custom_target(${library}Conf ALL DEPENDS ${outdir}/${library}.confdb)
# Add the target to the target that groups all of them for the package.
......@@ -2470,11 +2477,19 @@ function(gaudi_add_test name)
if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/tests_tmp)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests_tmp)
endif()
set(_test_environment_ "${ARG_ENVIRONMENT}" )
if (SANITIZER_ENABLED)
if (_test_environment_)
set(_test_environment_ "LD_PRELOAD=${SANITIZER_ENABLED};${_test_environment_}" )
else()
set(_test_environment_ "LD_PRELOAD=${SANITIZER_ENABLED}" )
endif()
endif()
gaudi_add_test(${qmt_name}
COMMAND ${test_cmd}
WORKING_DIRECTORY ${qmtest_root_dir}
LABELS QMTest ${ARG_LABELS}
ENVIRONMENT ${ARG_ENVIRONMENT})
ENVIRONMENT ${_test_environment_})
# we need to reapply the logic to qmt_name
if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
set(test_name ${qmt_name})
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment