backward trace from all threads

CMakeLists.txt

# https://www.boost.org/LICENSE_1_0.txt

cmake_minimum_required(VERSION 3.18)
project(backward_trace_all_threads)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
find_package(Threads)
enable_testing()

add_executable(main main.cc)
target_link_libraries(main Threads::Threads dw)
set_property(TARGET main PROPERTY CXX_STANDARD 11)
target_compile_definitions(main PUBLIC BACKWARD_HAS_DW=1)
add_test(NAME main_test COMMAND main)

main.cc

// https://www.boost.org/LICENSE_1_0.txt

#include <atomic>
#include <cassert>
#include <iostream>
#include <mutex>
#include <thread>
#include <vector>

#include <pthread.h>
#include <sys/signal.h>

// https://github.com/bombela/backward-cpp/
#include "backward.hpp"

// Use only async-signal-safe locking (it's forbidden for a reason; locking of
// any kind in a signal handler is probably a terrible idea but we're crashing
// anyway)
class AtomicOnlyMutex {
public:
  bool try_lock() {
    bool expected{false};
    bool desired{true};
    bool success = locked_.compare_exchange_weak(expected, desired);
    return success;
  }

  void lock() {
    while (!try_lock()) {
      usleep(1000);
    }
  }

  void unlock() { locked_.store(false); }

private:
  std::atomic<bool> locked_{false};
};

namespace backward {
namespace detail {

void *error_addr_from_ucontext(void *ucontext) {
  ucontext_t *uctx = static_cast<ucontext_t *>(ucontext);
#ifdef REG_RIP // x86_64
  return reinterpret_cast<void *>(uctx->uc_mcontext.gregs[REG_RIP]);
#else
  return nullptr;
#endif
}

} // namespace detail
} // namespace backward

class ThreadRegistry {
public:
  // NB: It is assumed that Register() will be called before we might need
  // to worry about tracing
  void Register() { pthreads_.push_back(pthread_self()); }

  // send a signal to every thread which has been registered here
  void Kill(int sig) {
    original_error_ = pthread_self();
    for (pthread_t t : pthreads_) {
      pthread_kill(t, sig);
    }
  }

  void SignalAction(int sig, siginfo_t *info, void *ucontext) {
    auto self = pthread_self();
    bool is_main = pthread_equal(pthreads_[0], self);
    bool is_original_error = pthread_equal(original_error_, self);

    // Sleeping like this *would* be wasteful- but we're crashing anyways.
    // We just need the threads to wait their turn to print traces and die.
    std::unique_lock<AtomicOnlyMutex> lock{mutex_};

    if (is_main) {
      // ensure the main thread is printed last since signals delivered to it
      // will cause us to terminate before the rest of the stacks are printed
      while (traced_ + 1 != pthreads_.size()) {
        lock.unlock();
        usleep(1000);
        lock.lock();
      }
    }

    backward::StackTrace st;

    if (auto error_addr =
            backward::detail::error_addr_from_ucontext(ucontext)) {
      st.load_from(error_addr, 32, ucontext, info->si_addr);
    } else {
      st.load_here(32, ucontext, info->si_addr);
    }

    std::cout << "--------------------------------------------- "
              << "(" << ++traced_ << "/" << pthreads_.size() << ")";
    if (is_original_error) {
      std::cout << " original error here";
    }
    std::cout << std::endl;

    backward::Printer printer;
    printer.address = true;
    printer.print(st, stderr);

    if (is_main) {
      std::cout << "--------------------------------------------- "
                << "forwarding signal and exiting" << std::endl;
      raise(info->si_code);
      _exit(EXIT_FAILURE);
    }
  }

private:
  std::vector<pthread_t> pthreads_;
  std::atomic<pthread_t> original_error_;
  size_t traced_{0};
  AtomicOnlyMutex mutex_;
};

ThreadRegistry thread_registry;

//////////////////////////////////////////////////////////////////////

void Foo(bool segfault) {
  if (segfault) {
    std::cout << "thread " << syscall(SYS_gettid) << " will now segfault"
              << std::endl;
    *reinterpret_cast<int *>(0x42) = 0;
  }
  usleep(1000);
}

void Bar(bool segfault) { Foo(segfault); }

void Baz(bool segfault) { Bar(segfault); }

void Quux(bool segfault) { Baz(segfault); }

int main() {
  // ensure the main thread is the first to register
  thread_registry.Register();

  std::atomic<int> segfaulter{-1};
  std::mutex add_thread_mutex;
  std::vector<std::thread> threads;
  for (int i = 0; i < 10; ++i) {
    threads.emplace_back([&, i] {
      {
        std::unique_lock<std::mutex> lock{add_thread_mutex};
        thread_registry.Register();
      }

      while (true) {
        Quux(segfaulter == i);
      }
    });
  }

  // set a signal handler for SIGSEGV which just
  // kills all threads with a SIGUSR2 to get their stacks
  signal(SIGSEGV, [](int) {
    thread_registry.Kill(SIGUSR2);
    sleep(10);
    raise(SIGSEGV);
  });

  // set a signal handler for SIGUSR2 which prints backtraces for all threads
  // in thread_registry
  stack_t stack;
  constexpr size_t kStackSize = 1024 * 1024 * 8;
  stack.ss_sp = malloc(kStackSize);
  stack.ss_size = kStackSize;
  stack.ss_flags = 0;
  assert(sigaltstack(&stack, nullptr) == 0);

  struct sigaction action = {0};
  action.sa_flags = SA_SIGINFO | SA_NODEFER | SA_ONSTACK;
  sigaddset(&action.sa_mask, SIGUSR2);
  action.sa_sigaction = [](int sig, siginfo_t *info, void *ucontext) {
    thread_registry.SignalAction(sig, info, ucontext);
  };
  assert(sigaction(SIGUSR2, &action, nullptr) == 0);

  // trigger a segfault
  sleep(1);
  segfaulter = rand() % threads.size();
  sleep(1); // wait for it...
}