Add portable stacktraces

- Add stacktraces to
   1. arbor_exception (and all derived) as `where`
   2. arbor_internal_error as `where`
   3. arb_assert
- Propagate stacktraces to Python exceptions derived from the above.
- Expand dev/doc on debugging

CMakeLists.txt

@@ -40,7 +40,7 @@ set(ARB_MODCC "" CACHE STRING "path to external modcc NMODL compiler")
 
 # Use libunwind to generate stack traces on errors?
 
-option(ARB_UNWIND "Use libunwind for stack trace printing if available" OFF)
+option(ARB_BACKTRACE "Enable stacktraces on assertion and exceptions (requires Boost)." OFF)
 
 # Specify GPU build type
 
@@ -386,17 +386,15 @@ if(ARB_WITH_GPU)
     endif()
 endif()
 
-# Use libunwind if requested for pretty printing stack traces
-#-------------------------------------------------------------
+# Use boost::stacktrace if requested for pretty printing stack traces
+#--------------------------------------------------------------------
 
-if (ARB_UNWIND)
-    find_package(Unwind REQUIRED)
-    if(Unwind_FOUND)
-        target_link_libraries(arbor-private-deps INTERFACE Unwind::unwind)
-        target_compile_definitions(arbor-private-deps INTERFACE WITH_UNWIND)
-
-        list(APPEND arbor_export_dependencies "Unwind")
-    endif()
+if (ARB_BACKTRACE)
+    find_package(Boost REQUIRED
+                 COMPONENTS stacktrace_basic
+                            stacktrace_addr2line)
+    target_link_libraries(arbor-private-deps INTERFACE Boost::stacktrace_basic Boost::stacktrace_addr2line ${CMAKE_DL_LIBS})
+    target_compile_definitions(arbor-private-deps INTERFACE WITH_BACKTRACE)
 endif()
 
 # Build and use modcc unless explicit path given
@@ -547,7 +545,6 @@ install(
     FILES
         "${CMAKE_CURRENT_BINARY_DIR}/arbor-config.cmake"
         "${CMAKE_CURRENT_BINARY_DIR}/arbor-config-version.cmake"
-        cmake/FindUnwind.cmake
     DESTINATION "${cmake_config_dir}")
 
 add_subdirectory(lmorpho)

arbor/arbexcept.cpp

@@ -4,12 +4,25 @@
 #include <arbor/arbexcept.hpp>
 #include <arbor/common_types.hpp>
 
+#include "util/unwind.hpp"
 #include "util/strprintf.hpp"
 
 namespace arb {
 
 using arb::util::pprintf;
 
+arbor_exception::arbor_exception(const std::string& what):
+    std::runtime_error{what} {
+    // Backtrace w/o this c'tor and that of backtrace.
+    where = util::backtrace{}.pop(2).to_string();
+}
+
+arbor_internal_error::arbor_internal_error(const std::string& what):
+        std::logic_error(what) {
+    // Backtrace w/o this c'tor and that of backtrace.
+    where = util::backtrace{}.pop(2).to_string();
+}
+
 domain_error::domain_error(const std::string& w): arbor_exception(w) {}
 
 bad_cell_probe::bad_cell_probe(cell_kind kind, cell_gid_type gid):

arbor/assert.cpp

@@ -12,7 +12,7 @@ void ARB_ARBOR_API abort_on_failed_assertion(
     int line,
     const char* func)
 {
-    // Emit stack trace If libunwind is being used.
+    // Emit stack trace if enabled.
     std::cerr << util::backtrace();
 
     // Explicit flush, as we can't assume default buffering semantics on stderr/cerr,

arbor/include/arbor/arbexcept.hpp

@@ -15,18 +15,15 @@ namespace arb {
 // there is a bug in the library.)
 
 struct ARB_SYMBOL_VISIBLE arbor_internal_error: std::logic_error {
-    arbor_internal_error(const std::string& what_arg):
-        std::logic_error(what_arg)
-    {}
+    arbor_internal_error(const std::string&);
+    std::string where;
 };
 
-
 // Common base-class for arbor run-time errors.
 
 struct ARB_SYMBOL_VISIBLE arbor_exception: std::runtime_error {
-    arbor_exception(const std::string& what_arg):
-        std::runtime_error(what_arg)
-    {}
+    arbor_exception(const std::string&);
+    std::string where;
 };
 
 // Logic errors

arbor/util/unwind.cpp

 #include <util/unwind.hpp>
 
-#ifdef WITH_UNWIND
-
-#define UNW_LOCAL_ONLY
-#include <libunwind.h>
-#include <cxxabi.h>
-
-#include <memory/util.hpp>
-#include <util/file.hpp>
-
-#include <cxxabi.h>
-#include <cstdint>
-#include <cstdio>
+#include <sstream>
 #include <string>
+#include <iomanip>
 #include <iostream>
 #include <vector>
 
+#ifdef WITH_BACKTRACE
+#define BOOST_STACKTRACE_GNU_SOURCE_NOT_REQUIRED
+#define BOOST_STACKTRACE_USE_ADDR2LINE
+#include <boost/stacktrace.hpp>
+#endif
+
 namespace arb {
 namespace util {
 
-static_assert(sizeof(std::uintptr_t)>=sizeof(unw_word_t),
-        "assumption that libunwind unw_word_t can be stored in std::uintptr_t is not valid");
-
-///  Builds a stack trace when constructed.
-///  The trace can then be printed, or accessed via the stack() member function.
 backtrace::backtrace() {
-    unw_cursor_t cursor;
-    unw_context_t context;
-
-    // initialize cursor to current frame for local unwinding.
-    unw_getcontext(&context);
-    unw_init_local(&cursor, &context);
-
-    while (unw_step(&cursor) > 0) {
-        // find the stack position
-        unw_word_t offset, pc;
-        unw_get_reg(&cursor, UNW_REG_IP, &pc);
-        if (pc == 0) {
-            break;
-        }
-
-        // get the name 
-        char sym[512];
-        if (unw_get_proc_name(&cursor, sym, sizeof(sym), &offset) == 0) {
-            frames_.push_back({std::string(sym), pc});
-        } else {
-            frames_.push_back({std::string("???"), pc});
-        }
+#ifdef WITH_BACKTRACE
+    auto bt = boost::stacktrace::basic_stacktrace{};
+    for (const auto& f: bt) {
+        frames_.push_back(source_location{f.name(), f.source_file(), f.source_line()});
     }
-}
-
-std::string demangle(std::string s) {
-    int status;
-    char* demangled = abi::__cxa_demangle(s.c_str(), nullptr, nullptr, &status);
-
-    // __cxa_demangle only returns a non-empty string if it is passed a valid C++
-    // mangled c++ symbol (i.e. it returns an empty string for normal c symbols)
-    if (status==0) {
-        s = demangled;
-    }
-    std::free(demangled); // don't leak the demangled string
-
-    return s;
+#endif
 }
 
 std::ostream& operator<<(std::ostream& out, const backtrace& trace) {
-    for (auto& f: trace.frames_) {
-        char loc_str[64];
-        snprintf(loc_str, sizeof(loc_str), "0x%lx", f.position);
-        out << loc_str << " " << f.name << "\n";
-        if (f.name=="main") {
-            break;
-        }
+#ifdef WITH_BACKTRACE
+    out << "Backtrace:\n";
+    int ix = 0;
+    for (const auto& f: trace.frames_) {
+        out << std::setw(8) << ix << " " << f.func << " (" << f.file << ":" << f.line << ")\n";
+        ix++;
     }
+#endif
     return out;
 }
 
-#if 0
-// Temporarily deprecated: automatic writing to disk of strack traces
-// needs to be run-time configurable.
-
-void backtrace::print(bool stop_at_main) const {
-    using namespace arb::memory::util;
-
-    auto i = 0;
-    while (file_exists("backtrace_" + std::to_string(i))) {
-        ++i;
-    }
-    auto fname = "backtrace_" + std::to_string(i);
-    auto fid = std::ofstream(fname);
-    for (auto& f: frames_) {
-        char loc_str[64];
-        snprintf(loc_str, sizeof(loc_str),"0x%lx", f.position);
-        fid << loc_str << " " << f.name << "\n";
-        if (stop_at_main && f.name=="main") {
-            break;
-        }
-    }
-    std::cerr << "BACKTRACE: A backtrace was generated and stored in the file " << fname << ".\n";
-    std::cerr << "           View a brief summary of the backtrace by running \"scripts/print_backtrace " << fname << " -b\".\n";
-    std::cerr << "           Run \"scripts/print_backtrace -h\" for more options.\n";
+backtrace& backtrace::pop(std::size_t n) {
+    frames_.erase(frames_.begin(),
+                  frames_.begin() + std::min(n, frames_.size()));
+    return *this;
 }
-#endif
 
-} // namespace util
-} // namespace arb
-
-#else
-
-namespace arb {
-namespace util {
-
-std::ostream& operator<<(std::ostream& out, const backtrace& trace) {
-    return out;
+std::string backtrace::to_string() const {
+    std::stringstream ss;
+    ss << *this;
+    return ss.str();
 }
 
-//void arb::util::backtrace::print(bool) const {}
-
 } // namespace util
 } // namespace arb
-
-#endif
-

arbor/util/unwind.hpp

 #pragma once
 
+#include <iostream>
 #include <cstdint>
 #include <string>
 #include <vector>
@@ -9,27 +10,27 @@ namespace util {
 
 /// Represents a source code location as a function name and address
 struct source_location {
-    std::string name;
-    std::uintptr_t position; // assume that unw_word_t is a unit64_t
+    std::string func;
+    std::string file;
+    std::size_t line;
 };
 
 /// Builds a stack trace when constructed.
-/// The trace can then be printed, or accessed via the stack() member function.
-/// NOTE: if WITH_UNWIND is not defined, the methods are empty
+/// NOTE: if WITH_BACKTRACE is not defined, the methods are empty
 class backtrace {
 public:
     /// the default constructor will build and store the strack trace.
-    backtrace() = default;
+    backtrace();
 
-    /// Creates a new file named backtrace_# where # is a number chosen
-    /// The back trace is printed to the file, and a message printed to
-    /// std::cerr with the backtrace file name and instructions for how
-    /// to post-process it.
-    void print(bool stop_at_main=true) const;
-    const std::vector<source_location>& frames() const { return frames_; }
+    std::vector<source_location>& frames() { return frames_; }
 
     friend std::ostream& operator<<(std::ostream&, const backtrace&);
 
+    // remove the top N=1 frames
+    backtrace& pop(std::size_t n=1);
+
+    std::string to_string() const;
+
 private:
     std::vector<source_location> frames_;
 };

cmake/FindUnwind.cmake

-# Find the libunwind library
-#
-#  Unwind_FOUND       - True if libunwind was found
-#  Unwind_LIBRARIES   - The libraries needed to use libunwind
-#  Unwind_INCLUDE_DIR - Location of unwind.h and libunwind.h
-#
-# The environment and cmake variables Unwind_ROOT and Unwind_ROOT_DIR
-# respectively can be used to help CMake finding the library if it
-# is not installed in any of the usual locations.
-#
-# Registers "Unwind::unwind" as an import library.
-
-if(NOT Unwind_FOUND)
-    set(Unwind_SEARCH_DIR ${Unwind_ROOT_DIR} $ENV{Unwind_ROOT})
-
-    find_path(Unwind_INCLUDE_DIR libunwind.h
-        HINTS ${Unwind_SEARCH_DIR}
-        PATH_SUFFIXES include
-    )
-
-    # libunwind requires that we link agains both libunwind.so/a and a
-    # a target-specific library libunwind-target.so/a.
-    # This code sets the "target" string above in libunwind_arch.
-    if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
-        set(_libunwind_arch "arm")
-    elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
-        set(_libunwind_arch "x86_64")
-    elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$")
-        set(_libunwind_arch "x86")
-    endif()
-
-    find_library(_unwind_library_generic unwind
-        HINTS ${Unwind_SEARCH_DIR}
-        PATH_SUFFIXES lib64 lib
-    )
-
-    find_library(_unwind_library_target unwind-${_libunwind_arch}
-        HINTS ${Unwind_SEARCH_DIR}
-        PATH_SUFFIXES lib64 lib
-    )
-
-    set(Unwind_LIBRARIES ${_unwind_library_generic} ${_unwind_library_target})
-
-    include(FindPackageHandleStandardArgs)
-    find_package_handle_standard_args(Unwind DEFAULT_MSG Unwind_INCLUDE_DIR Unwind_LIBRARIES)
-
-    mark_as_advanced(Unwind_LIBRARIES Unwind_INCLUDE_DIR)
-
-    if(Unwind_FOUND)
-        set(Unwind_INCLUDE_DIRS ${Unwind_INCLUDE_DIR})
-        if(NOT TARGET Unwind::unwind)
-            add_library(Unwind::unwind UNKNOWN IMPORTED)
-            set_target_properties(Unwind::unwind PROPERTIES
-                    IMPORTED_LOCATION "${_unwind_library_generic}"
-                    INTERFACE_LINK_LIBRARIES "${_unwind_library_target}"
-                    INTERFACE_INCLUDE_DIRECTORIES "${Unwind_INCLUDE_DIR}"
-            )
-        endif()
-    endif()
-
-    unset(_unwind_search_dir)
-    unset(_unwind_library_generic)
-    unset(_unwind_library_target)
-    unset(_libunwind_arch)
-endif()

doc/contrib/index.rst

@@ -61,12 +61,12 @@ share your Arbor simulations or publications!
 Filing an issue
 ~~~~~~~~~~~~~~~
 
-If you have found a bug or problem in Arbor, or want to request a
-feature, you can use our `issue
-tracker <https://github.com/arbor-sim/arbor/issues>`__. If you issue is
-not yet filed in the issue tracker, please do so and describe the
-problem, bug or feature as best you can. You can add supporting data,
-code or documents to help make your point.
+If you have found a bug or problem in Arbor, or want to request a feature, you
+can use our `issue tracker <https://github.com/arbor-sim/arbor/issues>`__. If
+you issue is not yet filed in the issue tracker, please do so and describe the
+problem, bug or feature as best you can. You can add supporting data, code or
+documents to help make your point. For bugs in particular, stacktraces (either
+from inside a debugger or by enabling ``ARB_BACKTRACE``) are extremely useful.
 
 .. _contribindex-solveissue:
 

doc/dev/debug.rst

+.. _dev-debug:
+
+Debugging
+=========
+
+Backtraces
+----------
+
+When building Arbor you can enable backtraces in the CMake configure step by
+setting ``ARB_BACKTRACE=ON``. Beware aware that this requires the ``Boost``
+libraries to be installed on your system. This will cause the following
+additions to Arbor's behaviour
+
+1. Failed assertions via ``asb_assert`` will print the corresponding stacktrace.
+2. All exceptions deriving from ``arbor_exception`` and ``arbor_internal_error``
+   will have stacktraces attached in the ``where`` field.
+3. Python exceptions derived from these types will add that same stacktrace
+   information to their message.
+
+Alternatively, you can obtain the same information using a debugger like GDB or
+LLDB.
+
+.. note::
+
+   Since Arbor often uses a buffer of instructions on how to construct a
+   particular object instead of perform the action right away, errors occur not
+   always at the location you might expect.
+
+   Consider this (adapted from ``network_ring.py``)
+
+   .. code-block:: python
+
+      class Recipe(arb.recipe):
+        # [...]
+        def connections_on(self, gid):
+          return [arbor.connection((src, "detector"), "syn-NOT", w, d)]
+
+        def cell_description(self, gid):
+          # [...]
+          decor = (arbor.decor()
+              .place('"synapse_site"', arbor.synapse("expsyn"), "syn"))
+          return arbor.cable_cell(tree, labels, decor)
+
+      rec = Recipe()            # "Ok"
+      sim = arb.simulation(rec) # ERROR here

doc/dev/index.rst

@@ -16,6 +16,7 @@ Here we document internal components of Arbor. These pages can be useful if you'
 
    cable_cell
    cell_groups
+   debug
    matrix_solver
    simd_api
    shared_state

doc/install/build_install.rst

@@ -150,12 +150,19 @@ an additional support library ``arborio``. This library requires
 with NeuroML support enabled.
 See :ref:`install-neuroml` for more information.
 
+Boost
+~~~~~
+
+When ``ARB_BACKTRACE`` is set to ``ON`` during configure we use
+``Boost::stacktrace`` to print stacktraces upon failed assertions and attach
+them to the base exception types ``arbor_exception`` and
+``arbor_internal_error`` as ``where``.
 
 Documentation
 ~~~~~~~~~~~~~~
 
 To build a local copy of the html documentation that you are reading now, you will need to
-install `Sphinx <http://www.sphinx-doc.org/en/master/>`_.
+install ``Sphinx <http://www.sphinx-doc.org/en/master/>`_.
 
 .. _install-downloading:
 

modcc/CMakeLists.txt

@@ -41,7 +41,6 @@ if (ARB_USE_BUNDLED_FMT)
             $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../ext/fmt/include>)
 
     target_compile_definitions(libmodcc PRIVATE FMT_HEADER_ONLY)
-
 else()
     target_include_directories(libmodcc
         PUBLIC

python/pyarb.cpp

 #include <pybind11/pybind11.h>
 #include <pybind11/numpy.h>
 
+#include <sstream>
+
 #include <arbor/spike.hpp>
 #include <arbor/common_types.hpp>
 #include <arbor/arbexcept.hpp>
@@ -44,10 +46,6 @@ PYBIND11_MODULE(_arbor, m) {
     m.doc() = "arbor: multicompartment neural network models.";
     m.attr("__version__") = ARB_VERSION;
 
-    // Translate Arbor errors -> Python exceptions.
-    pybind11::register_exception<arb::file_not_found_error>(m, "FileNotFoundError", PyExc_FileNotFoundError);
-    pybind11::register_exception<arb::zero_thread_requested_error>(m, "ValueError", PyExc_ValueError);
-
     pyarb::register_cable_loader(m);
     pyarb::register_cable_probes(m, global_ptr);
     pyarb::register_cells(m);
@@ -64,6 +62,33 @@ PYBIND11_MODULE(_arbor, m) {
     pyarb::register_simulation(m, global_ptr);
     pyarb::register_single_cell(m);
 
+    // This is the fallback. All specific translators take precedence by being
+    // registered *later*.
+    pybind11::register_exception_translator([](std::exception_ptr p) {
+        try {
+            if (p) std::rethrow_exception(p);
+        }
+        catch (const arb::arbor_exception& e) {
+            std::stringstream msg;
+            msg << e.what()
+                << "\n"
+                << e.where;
+            PyErr_SetString(PyExc_RuntimeError, msg.str().c_str());
+        }
+        catch (const arb::arbor_internal_error& e) {
+            std::stringstream msg;
+            msg << e.what()
+                << "\n"
+                << e.where;
+            PyErr_SetString(PyExc_RuntimeError, msg.str().c_str());
+        }
+    });
+
+    // Translate Arbor errors -> Python exceptions.
+    pybind11::register_exception<arb::file_not_found_error>(m, "FileNotFoundError", PyExc_FileNotFoundError);
+    pybind11::register_exception<arb::zero_thread_requested_error>(m, "ValueError", PyExc_ValueError);
+
+
     #ifdef ARB_MPI_ENABLED
     pyarb::register_mpi(m);
     #endif