From ae95dc9819f544f4476624cbafb8d7ac31432582 Mon Sep 17 00:00:00 2001 From: Benjamin Curtice Corbett Date: Mon, 10 Aug 2020 17:32:36 -0700 Subject: [PATCH] Reimplemented stackTrace, added line info output. --- CMakeLists.txt | 4 +- cmake/Config.cmake | 18 +- docs/doxygen/LvArrayConfig.hpp | 2 + host-configs/LLNL/lassen-clang@upstream.cmake | 1 + host-configs/LLNL/lassen-gcc@8.3.1.cmake | 1 + host-configs/LLNL/quartz-base.cmake | 2 + src/LvArrayConfig.hpp.in | 2 + src/Macros.hpp | 2 +- src/system.cpp | 335 +++++++++++++++--- src/system.hpp | 12 +- unitTests/CMakeLists.txt | 1 + unitTests/testStackTrace.cpp | 64 ++++ 12 files changed, 381 insertions(+), 63 deletions(-) create mode 100644 unitTests/testStackTrace.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 01e7ff80..f37979ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,8 @@ if( NOT is_submodule ) option( ENABLE_OPENMP "Build with OpenMP" ON ) option( ENABLE_CALIPER "Build with Caliper" ON ) + option( ENABLE_ADDR2LINE "Enable addr2line usage in stacktraces" ON ) + include( cmake/blt/SetupBLT.cmake ) include( cmake/CMakeBasics.cmake ) include( cmake/SetupTPL.cmake ) @@ -34,7 +36,7 @@ endif() include(cmake/Macros.cmake) include(cmake/Config.cmake) -set( lvarray_dependencies "" ) +set( lvarray_dependencies dl ) if( ENABLE_CHAI ) set( lvarray_dependencies ${lvarray_dependencies} chai umpire ) diff --git a/cmake/Config.cmake b/cmake/Config.cmake index f391e68d..7ecd7d35 100644 --- a/cmake/Config.cmake +++ b/cmake/Config.cmake @@ -14,15 +14,27 @@ foreach( DEP in ${PREPROCESSOR_DEFINES}) endif() endforeach() +if( USE_ADDR2LINE ) + if ( NOT DEFINED ADDR2LINE_EXEC ) + set( ADDR2LINE_EXEC /usr/bin/addr2line CACHE PATH "" ) + endif() + + if ( NOT EXISTS ${ADDR2LINE_EXEC} ) + message( FATAL_ERROR "The addr2line executable does not exist: ${ADDR2LINE_EXEC}" ) + endif() + + set( LVARRAY_ADDR2LINE_EXEC ${ADDR2LINE_EXEC} ) +endif() + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/src/LvArrayConfig.hpp.in ${CMAKE_BINARY_DIR}/include/LvArrayConfig.hpp ) function( make_full_config_file PREPROCESSOR_VARS ) foreach( DEP in ${PREPROCESSOR_VARS}) - set(USE_${DEP} TRUE ) - set(GEOSX_USE_${DEP} TRUE ) - set(${DEP} TRUE ) + set(USE_${DEP} TRUE ) + set(GEOSX_USE_${DEP} TRUE ) + set(${DEP} TRUE ) endforeach() configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/src/LvArrayConfig.hpp.in diff --git a/docs/doxygen/LvArrayConfig.hpp b/docs/doxygen/LvArrayConfig.hpp index 0e8ebad3..a0fab3fc 100644 --- a/docs/doxygen/LvArrayConfig.hpp +++ b/docs/doxygen/LvArrayConfig.hpp @@ -37,3 +37,5 @@ #ifndef USE_CALIPER #define USE_CALIPER #endif + +#define LVARRAY_ADDR2LINE_EXEC "/usr/bin/addr2line" diff --git a/host-configs/LLNL/lassen-clang@upstream.cmake b/host-configs/LLNL/lassen-clang@upstream.cmake index b209b3f9..9ca46355 100644 --- a/host-configs/LLNL/lassen-clang@upstream.cmake +++ b/host-configs/LLNL/lassen-clang@upstream.cmake @@ -6,6 +6,7 @@ set(GEOSX_TPL_DIR ${GEOSX_TPL_ROOT_DIR}/2020-07-08/install-${CONFIG_NAME}-releas set(ENABLE_UMPIRE ON CACHE BOOL "") set(ENABLE_CHAI ON CACHE BOOL "") +set(USE_ADDR2LINE ON CACHE BOOL "") # C options set(CMAKE_C_COMPILER /usr/tce/packages/clang/clang-upstream-2019.03.26/bin/clang CACHE PATH "") diff --git a/host-configs/LLNL/lassen-gcc@8.3.1.cmake b/host-configs/LLNL/lassen-gcc@8.3.1.cmake index 5575e749..a13a94b1 100644 --- a/host-configs/LLNL/lassen-gcc@8.3.1.cmake +++ b/host-configs/LLNL/lassen-gcc@8.3.1.cmake @@ -7,6 +7,7 @@ set(GEOSX_TPL_DIR ${GEOSX_TPL_ROOT_DIR}/2020-07-08/install-${CONFIG_NAME}-releas set(ENABLE_UMPIRE ON CACHE BOOL "") set(ENABLE_CHAI ON CACHE BOOL "") +set(USE_ADDR2LINE ON CACHE BOOL "") set(CMAKE_C_COMPILER /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc CACHE PATH "") set(CMAKE_CXX_COMPILER /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ CACHE PATH "") diff --git a/host-configs/LLNL/quartz-base.cmake b/host-configs/LLNL/quartz-base.cmake index b015fcca..7a297e78 100644 --- a/host-configs/LLNL/quartz-base.cmake +++ b/host-configs/LLNL/quartz-base.cmake @@ -37,6 +37,8 @@ set(ENABLE_CALIPER ON CACHE BOOL "") set(ENABLE_PAPI ON CACHE BOOL "") set(PAPI_PREFIX /usr/tce/packages/papi/papi-5.4.3 CACHE PATH "") +set(USE_ADDR2LINE ON CACHE BOOL "") + set(ENABLE_OPENMP ON CACHE BOOL "") set(CUDA_ENABLED OFF CACHE BOOL "") diff --git a/src/LvArrayConfig.hpp.in b/src/LvArrayConfig.hpp.in index 9d266bb8..bb52bef2 100644 --- a/src/LvArrayConfig.hpp.in +++ b/src/LvArrayConfig.hpp.in @@ -37,3 +37,5 @@ #ifndef USE_CALIPER #cmakedefine USE_CALIPER #endif + +#cmakedefine LVARRAY_ADDR2LINE_EXEC "@LVARRAY_ADDR2LINE_EXEC@" diff --git a/src/Macros.hpp b/src/Macros.hpp index c5e4c6df..ad776ff9 100644 --- a/src/Macros.hpp +++ b/src/Macros.hpp @@ -108,7 +108,7 @@ __oss << "***** LOCATION: " LOCATION "\n"; \ __oss << "***** Controlling expression (should be false): " STRINGIZE( EXP ) "\n"; \ __oss << MSG << "\n"; \ - __oss << LvArray::system::stackTrace(); \ + __oss << LvArray::system::stackTrace( true ); \ std::cout << __oss.str() << std::endl; \ LvArray::system::abort(); \ } \ diff --git a/src/system.cpp b/src/system.cpp index 305652ec..82c31b5c 100644 --- a/src/system.cpp +++ b/src/system.cpp @@ -5,7 +5,9 @@ * SPDX-License-Identifier: (BSD-3-Clause) */ +// Source includes #include "system.hpp" +#include "Macros.hpp" // System includes #include @@ -14,82 +16,300 @@ #include #include #include -#include #include -#include #if defined(__x86_64__) - #include #include #endif -std::string demangleBacktrace( char * backtraceString, int frame ) +#include +#include + +#if defined( LVARRAY_ADDR2LINE_EXEC ) + #include + #include +#endif + +#if defined( USE_MPI ) + #include +#endif + +/** + * @struct UnwindState + * @brief Holds info used in unwindCallback. + * @note Adapted from https://github.com/boostorg/stacktrace + */ +struct UnwindState +{ + std::size_t frames_to_skip; + void * * current; + void * * end; +}; + +/** + * @brief Callback used with _Unwind_Backtrace. + * @param context + * @param arg The UnwindState. + * @note Adapted from https://github.com/boostorg/stacktrace + */ +static _Unwind_Reason_Code unwindCallback( _Unwind_Context * const context, void * const arg ) +{ + // Note: do not write `::_Unwind_GetIP` because it is a macro on some platforms. + // Use `_Unwind_GetIP` instead! + UnwindState * const state = static_cast< UnwindState * >(arg); + if( state->frames_to_skip ) + { + --state->frames_to_skip; + return _Unwind_GetIP( context ) ? _URC_NO_REASON : _URC_END_OF_STACK; + } + + *state->current = reinterpret_cast< void * >( _Unwind_GetIP( context ) ); + + ++state->current; + if( !*(state->current - 1) || state->current == state->end ) + { + return _URC_END_OF_STACK; + } + + return _URC_NO_REASON; +} + +/** + * @brief Populate @p frames with the stack return addresses. + * @param frames A pointer to the buffer to fill, must have length at least @p maxFrames. + * @param maxFrames The maximum number of frames to collect. + * @param skip The number of initial frames to skip. + * @note Adapted from https://github.com/boostorg/stacktrace + */ +static std::size_t collect( void * * const frames, std::size_t const maxFrames, std::size_t const skip ) { - char * mangledName = nullptr; - char * functionOffset = nullptr; - char * returnOffset = nullptr; - -#ifdef __APPLE__ - /* On apple machines the mangled function name always starts at the 58th - * character */ - constexpr int APPLE_OFFSET = 58; - mangledName = backtraceString + APPLE_OFFSET; - for( char * p = backtraceString; *p; ++p ) + std::size_t frames_count = 0; + if( !maxFrames ) { - if( *p == '+' ) + return frames_count; + } + + UnwindState state = { skip + 1, frames, frames + maxFrames }; + _Unwind_Backtrace( &unwindCallback, &state ); + frames_count = state.current - frames; + + if( frames_count && frames[frames_count - 1] == 0 ) + { + --frames_count; + } + + return frames_count; +} + +/** + * @brief Return the demangled name of the function at @p address. + * @param address The address of the function gotten from the stack frame. + * @return the demangled name of the function at @p address. + */ +static std::string getFunctionNameFromFrame( void const * const address ) +{ + Dl_info dli; + const bool dl_ok = dladdr( address, &dli ); + if( dl_ok && dli.dli_sname ) + { + return LvArray::system::demangle( dli.dli_sname ); + } + + return ""; +} + +#if defined( LVARRAY_ADDR2LINE_EXEC ) + +/** + * @brief Return @c true iff @p path is an absolute path. + * @param path The file path to inspect. + * @return @c true iff @p path is an absolute path. + */ +static constexpr bool isAbsPath( char const * path ) +{ return *path != '\0' && ( *path == ':' || *path == '/' || isAbsPath( path + 1 ) ); } + +/** + * @class UnwindState + * @brief Used to fork a subprocess that executes @c LVARRAY_ADDR2LINE_EXEC and get the results. + * @note Adapted from https://github.com/boostorg/stacktrace + */ +class Addr2LinePipe +{ +public: + + /** + * @brief Constructor. + * @param flag The flag(s) to pass to addr2line. + * @param execPath The path to the executable the address is from, usually the current executable. + * @param addr The address to query. + */ + Addr2LinePipe( char const * const flag, char const * const execPath, char const * const addr ): + m_file( nullptr ), + m_pid( 0 ) + { + int pdes[ 2 ]; + char prog_name[] = STRINGIZE( LVARRAY_ADDR2LINE_EXEC ); + static_assert( isAbsPath( STRINGIZE( LVARRAY_ADDR2LINE_EXEC ) ), + "LVARRAY_ADDR2LINE_EXEC = " STRINGIZE( LVARRAY_ADDR2LINE_EXEC ) ); + + char * argp[] = { + prog_name, + const_cast< char * >( flag ), + const_cast< char * >( execPath ), + const_cast< char * >( addr ), + 0 + }; + + if( pipe( pdes ) < 0 ) + { return; } + + m_pid = fork(); + switch( m_pid ) { - functionOffset = p; + case -1: + { + // Failed... + close( pdes[ 0 ] ); + close( pdes[ 1 ] ); + return; + + } + case 0: + // We are the child. + close( STDERR_FILENO ); + close( pdes[ 0 ] ); + if( pdes[ 1 ] != STDOUT_FILENO ) + { dup2( pdes[ 1 ], STDOUT_FILENO ); } + + // Do not use `execlp()`, `execvp()`, and `execvpe()` here! + // `exec*p*` functions are vulnerable to PATH variable evaluation attacks. + execv( prog_name, argp ); + _exit( 127 ); } - returnOffset = p; + + m_file = fdopen( pdes[ 0 ], "r" ); + close( pdes[ 1 ] ); } -#else - for( char * p = backtraceString; *p; ++p ) + + /** + * @brief User defined conversion to a file pointer. + * @return A file pointer to the output of the addr2line execution. + */ + operator FILE *() const + { return m_file; } + + /// Destructor + ~Addr2LinePipe() { - if( *p == '(' ) + if( m_file ) { - mangledName = p; + fclose( m_file ); + int pstat = 0; + kill( m_pid, SIGKILL ); + waitpid( m_pid, &pstat, 0 ); } - else if( *p == '+' ) + } + +private: + /// A file pointer to the output of the addr2line execution. + FILE * m_file; + + /// The process ID of the forked child process. + pid_t m_pid; +}; + +/** + * @brief Return the result of calling @c addr2line @p flag @c pathToCurrentExecutable @p addr. + * @param flag The flag to pass to addr2line. + * @param addr The address to pass to addr2line. + * @return The result of calling @c addr2line @p flag @c pathToCurrentExecutable @p addr. + */ +static std::string addr2line( const char * flag, const void * addr ) +{ + std::string res; + + Dl_info dli; + if( dladdr( addr, &dli ) ) + { + res = dli.dli_fname; + } + else + { + res.resize( 16 ); + int rlin_size = readlink( "/proc/self/exe", &res[ 0 ], res.size() - 1 ); + while( rlin_size == static_cast< int >( res.size() - 1 ) ) { - functionOffset = p; + res.resize( res.size() * 4 ); + rlin_size = readlink( "/proc/self/exe", &res[ 0 ], res.size() - 1 ); } - else if( *p == ')' ) + if( rlin_size == -1 ) { - returnOffset = p; - break; + res.clear(); + return res; } + res.resize( rlin_size ); } -#endif std::ostringstream oss; + oss << addr; - if( mangledName && functionOffset && returnOffset && - mangledName < functionOffset ) + Addr2LinePipe p( flag, res.c_str(), oss.str().c_str() ); + res.clear(); + + if( !p ) { - // if the line could be processed, attempt to demangle the symbol - *mangledName = 0; - mangledName++; -#ifdef __APPLE__ - *(functionOffset - 1) = 0; -#endif - *functionOffset = 0; - ++functionOffset; - *returnOffset = 0; - ++returnOffset; + return res; + } + + char data[ 32 ]; + while( !feof( p ) ) + { + if( fgets( data, sizeof( data ), p ) ) + { + res += data; + } + else + { + break; + } + } - oss << "Frame " << frame << ": " << LvArray::system::demangle( mangledName ) << "\n"; + // Trimming + while( !res.empty() && ( res[ res.size() - 1 ] == '\n' || res[ res.size() - 1 ] == '\r' ) ) + { + res.erase( res.size() - 1 ); } - else + + return res; +} + +#endif + +/** + * @brief Return the source location of @p address iff LVARRAY_ADDR2LINE_EXEC is defined. + * @param address The address to get the source location of. + * @return The source location of @p address iff LVARRAY_ADDR2LINE_EXEC is defined. + */ +static std::string getSourceLocationFromFrame( void const * const address ) +{ + #if defined( LVARRAY_ADDR2LINE_EXEC ) + std::string const source_line = addr2line( "-Cpe", address ); + if( !source_line.empty() && source_line[0] != '?' ) { - // otherwise, print the whole line - oss << "Frame " << frame << ": " << backtraceString << "\n"; + return source_line; } + #else + LVARRAY_UNUSED_VARIABLE( address ); + #endif - return ( oss.str() ); + return ""; } -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -std::string getFpeDetails() +/** + * @brief Return a string representing the current floating point exception. + * @return A string representing the current floating point exception. + */ +static std::string getFpeDetails() { std::ostringstream oss; int const fpe = fetestexcept( FE_ALL_EXCEPT ); @@ -132,27 +352,30 @@ using handle_type = void ( * )( int ); static std::map< int, handle_type > initialHandler; /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -std::string stackTrace() +std::string stackTrace( bool const location ) { constexpr int MAX_FRAMES = 25; void * array[ MAX_FRAMES ]; - const int size = backtrace( array, MAX_FRAMES ); - char * * strings = backtrace_symbols( array, size ); + std::size_t const size = collect( array, MAX_FRAMES, 1 ); - // skip first stack frame (points here) std::ostringstream oss; oss << "\n** StackTrace of " << size - 1 << " frames **\n"; - for( int i = 1; i < size && strings != nullptr; ++i ) + for( std::size_t i = 0; i < size; ++i ) { - oss << demangleBacktrace( strings[ i ], i ); + oss << "Frame " << i << ": " << getFunctionNameFromFrame( array[ i ] ); + + if( location ) + { + oss << " " << getSourceLocationFromFrame( array[ i ] ); + } + + oss << "\n"; } oss << "=====\n"; - free( strings ); - - return ( oss.str() ); + return oss.str(); } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -231,7 +454,7 @@ void stackTraceHandler( int const sig, bool const exit ) } } - oss << stackTrace() << std::endl; + oss << stackTrace( true ) << std::endl; std::cout << oss.str(); if( exit ) diff --git a/src/system.hpp b/src/system.hpp index 22870b88..f1bec372 100644 --- a/src/system.hpp +++ b/src/system.hpp @@ -13,6 +13,8 @@ #pragma once // System includes +#include +#include #include #include @@ -25,8 +27,14 @@ namespace LvArray namespace system { -/// @return Return a demangled stack trace of the last 25 frames. -std::string stackTrace(); +/** + * @brief Return a demangled stack trace of the last 25 frames. + * @param lineInfo If @c true then file and line numbers will be added to the + * trace if available. This is only supported if @c LVARRAY_ADDR2LINE_EXEC is defined and + * normally only works in debug builds. + * @return A demangled stack trace of the last 25 frames. + */ +std::string stackTrace( bool const lineInfo ); /** * @return The demangled name corresponding to the given diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt index b492aaeb..3dbf2d77 100644 --- a/unitTests/CMakeLists.txt +++ b/unitTests/CMakeLists.txt @@ -64,6 +64,7 @@ set(testSources testTensorOpsSymInverseOneArg.cpp testTensorOpsSymInverseTwoArgs.cpp testTypeManipulation.cpp + testStackTrace.cpp ) # diff --git a/unitTests/testStackTrace.cpp b/unitTests/testStackTrace.cpp new file mode 100644 index 00000000..624545a9 --- /dev/null +++ b/unitTests/testStackTrace.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC and LvArray contributors. + * All rights reserved. + * See the LICENSE file for details. + * SPDX-License-Identifier: (BSD-3-Clause) + */ + +// Source includes +#include "system.hpp" +#include "Macros.hpp" + +// TPL includes +#include +#include + + +namespace LvArray +{ +namespace testing +{ + +std::string __attribute__((noinline)) baz() +{ + return system::stackTrace( true ); +} + +std::string __attribute__((noinline)) bar() +{ + return baz(); +} + +std::string __attribute__((noinline)) foo() +{ + bar(); + std::string const result = bar(); + bar(); + return result; +} + +#if defined( LVARRAY_USE_OPENMP ) + +TEST( stackTrace, OpenMP ) +{ + RAJA::forall< RAJA::omp_parallel_for_exec >( RAJA::TypedRangeSegment< int >( 0, 100 ), []( int ){} ); + LVARRAY_LOG( foo() ); +} + +#endif + +TEST( stackTrace, normal ) +{ + LVARRAY_LOG( foo() ); +} + +} // namespace testing +} // namespace LvArray + +// This is the default gtest main method. It is included for ease of debugging. +int main( int argc, char * * argv ) +{ + ::testing::InitGoogleTest( &argc, argv ); + int const result = RUN_ALL_TESTS(); + return result; +}