[Feature] Print stack trace on crash

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6526996a3908824ce1fd31076dc91ad6148cd90b..a3fe5bc2ccd30097420c25a366ad94312a3a149f 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -64,6 +64,7 @@ OPTION(ENABLE_COVERAGE     "Build rspamd with code coverage options [default: OF
 OPTION(ENABLE_FULL_DEBUG   "Build rspamd with all possible debug [default: OFF]" OFF)
 OPTION(ENABLE_UTILS        "Build rspamd internal utils [default: OFF]" OFF)
 OPTION(ENABLE_TORCH        "Install torch7 with Rspamd [default: ON]" ON)
+OPTION(ENABLE_LIBUNWIND    "Use libunwind to print crash traces [default: OFF]" OFF)
 
 INCLUDE(FindArch.cmake)
 TARGET_ARCHITECTURE(ARCH)
@@ -630,6 +631,12 @@ IF(ENABLE_JEMALLOC MATCHES "ON")
        SET(WITH_JEMALLOC "1")
        LIST(APPEND RSPAMD_REQUIRED_LIBRARIES "-lpthread")
 ENDIF()
+
+IF(ENABLE_LIBUNWIND MATCHES "ON")
+       ProcessPackage(LIBUNWIND LIBRARY unwind INCLUDE libunwind.h INCLUDE_SUFFIXES include/libunwind
+                       ROOT ${LIBUNWIND_ROOT_DIR} MODULES libunwind)
+       SET(WITH_LIBUNWIND "1")
+ENDIF()
 ProcessPackage(GTHREAD2 LIBRARY gthread-2.0 INCLUDE glib.h
        INCLUDE_SUFFIXES include/glib include/glib-2.0
        ROOT ${GLIB_ROOT_DIR} MODULES gthread-2.0>=2.28)
@@ -984,6 +991,7 @@ LIST(APPEND CMAKE_REQUIRED_INCLUDES "${LIBSSL_INCLUDE}")
 CHECK_SYMBOL_EXISTS(SSL_set_tlsext_host_name "openssl/ssl.h" HAVE_SSL_TLSEXT_HOSTNAME)
 CHECK_SYMBOL_EXISTS(dirfd "sys/types.h;unistd.h;dirent.h" HAVE_DIRFD)
 CHECK_SYMBOL_EXISTS(fpathconf "sys/types.h;unistd.h" HAVE_FPATHCONF)
+CHECK_SYMBOL_EXISTS(sigaltstack "signal.h" HAVE_SIGALTSTACK)
 
 IF(ENABLE_PCRE2 MATCHES "ON")
        IF(HAVE_PCRE_JIT)

diff --git a/config.h.in b/config.h.in
index 007586b8fa6d51a257a3be82403bf421b05f5cdf..5b22d3df96ddeb4795251af52c2fa0724e9763b2 100644 (file)
--- a/config.h.in
+++ b/config.h.in
@@ -83,6 +83,7 @@
 #cmakedefine HAVE_SETITIMER      1
 #cmakedefine HAVE_SETPROCTITLE   1
 #cmakedefine HAVE_SETSIG         1
+#cmakedefine HAVE_SIGALTSTACK    1
 #cmakedefine HAVE_SIGINFO_H      1
 #cmakedefine HAVE_SOCK_SEQPACKET 1
 #cmakedefine HAVE_SSL_TLSEXT_HOSTNAME 1
@@ -141,6 +142,7 @@
 #cmakedefine WITH_SQLITE         1
 #cmakedefine WITH_SYSTEM_HIREDIS 1
 #cmakedefine WITH_TORCH          1
+#cmakedefine WITH_LIBUNWIND      1
 
 #cmakedefine DISABLE_PTHREAD_MUTEX 1
 

diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c
index c60273dea2882915d67ec5d98fdadc4ae9d0bbd6..a1fa4048c8d1cf0734ecb9548ff14db8507becf4 100644 (file)
--- a/src/libserver/worker_util.c
+++ b/src/libserver/worker_util.c
@@ -44,6 +44,12 @@
 #endif
 #include "zlib.h"
 
+#ifdef WITH_LIBUNWIND
+#define UNW_LOCAL_ONLY 1
+#include <libunwind.h>
+#define UNWIND_BACKTRACE_DEPTH 256
+#endif
+
 static void rspamd_worker_ignore_signal (int signo);
 /**
  * Return worker's control structure by its type
@@ -924,4 +930,106 @@ rspamd_worker_init_monitored (struct rspamd_worker *worker,
        rspamd_monitored_ctx_config (worker->srv->cfg->monitored_ctx,
                        worker->srv->cfg, ev_base, resolver->r,
                        rspamd_worker_monitored_on_change, worker);
+}
+
+#ifdef HAVE_SA_SIGINFO
+
+#ifdef WITH_LIBUNWIND
+static void
+rspamd_print_crash (ucontext_t *uap)
+{
+       unw_cursor_t cursor;
+       unw_word_t ip, off;
+       guint level;
+       gint ret;
+
+       if ((ret = unw_init_local (&cursor, uap)) != 0) {
+               msg_err ("unw_init_local: %d", ret);
+
+               return;
+       }
+
+       level = 0;
+       ret = 0;
+
+       for (;;) {
+               char name[128];
+
+               if (level >= UNWIND_BACKTRACE_DEPTH) {
+                       break;
+               }
+
+               unw_get_reg (&cursor, UNW_REG_IP, &ip);
+               ret = unw_get_proc_name(&cursor, name, sizeof (name), &off);
+
+               if (ret == 0) {
+                       msg_err ("%d: %p: %s()+0x%xl",
+                               level, ip, name, (uintptr_t)off);
+               } else {
+                       msg_err ("%d: %p: <unknown>", level, ip);
+               }
+
+               level++;
+               ret = unw_step (&cursor);
+
+               if (ret <= 0) {
+                       break;
+               }
+       }
+
+       if (ret < 0) {
+               msg_err ("unw_step_ptr: %d", ret);
+       }
+}
+#endif
+
+static void
+rspamd_crash_sig_handler (int sig, siginfo_t *info, void *ctx)
+{
+       struct sigaction sa;
+       ucontext_t *uap = ctx;
+
+       msg_err ("caught fatal signal %d(%s), "
+                        "pid: %P, trace: ",
+                       sig, strsignal (sig), getpid ());
+       (void)uap;
+#ifdef WITH_LIBUNWIND
+       rspamd_print_crash (uap);
+#endif
+
+       /*
+        * Invoke signal with the default handler
+        */
+       sigemptyset (&sa.sa_mask);
+       sa.sa_handler = SIG_DFL;
+       sa.sa_flags = 0;
+       sigaction (sig, &sa, NULL);
+       kill (getpid (), sig);
+}
+#endif
+
+void
+rspamd_set_crash_handler (struct rspamd_main *main)
+{
+#ifdef HAVE_SA_SIGINFO
+       struct sigaction sa;
+
+#ifdef HAVE_SIGALTSTACK
+       stack_t ss;
+
+       /* Allocate special stack, NOT freed at the end so far */
+       ss.ss_size = MAX (SIGSTKSZ, 8192 * 4);
+       ss.ss_sp = g_malloc0 (ss.ss_size);
+       sigaltstack (&ss, NULL);
+#endif
+
+       sigemptyset(&sa.sa_mask);
+       sa.sa_sigaction = &rspamd_crash_sig_handler;
+       sa.sa_flags = SA_RESTART | SA_SIGINFO | SA_ONSTACK;
+       sigaction(SIGSEGV, &sa, NULL);
+       sigaction(SIGBUS, &sa, NULL);
+       sigaction(SIGABRT, &sa, NULL);
+       sigaction(SIGFPE, &sa, NULL);
+       sigaction(SIGSYS, &sa, NULL);
+#endif
 }
\ No newline at end of file

diff --git a/src/libserver/worker_util.h b/src/libserver/worker_util.h
index 2e3fd44580f0b9d3fbf70ec1d4795536a40eaa1b..dbcc8f8a2beb348157f0088f1ae2ee7f26e1a77b 100644 (file)
--- a/src/libserver/worker_util.h
+++ b/src/libserver/worker_util.h
@@ -187,6 +187,11 @@ void rspamd_worker_session_cache_remove (void *cache, void *ptr);
 struct rspamd_worker *rspamd_fork_worker (struct rspamd_main *,
                struct rspamd_worker_conf *, guint idx, struct event_base *ev_base);
 
+/**
+ * Sets crash signals handlers if compiled with libunwind
+ */
+void rspamd_set_crash_handler (struct rspamd_main *);
+
 /**
  * Initialise the main monitoring worker
  * @param worker

diff --git a/src/rspamd.c b/src/rspamd.c
index d7543643e28da11ce226ee270d6ee9e4024c3d7e..7c54505216286b00d516d8fbfafe4a48dc2cf27b 100644 (file)
--- a/src/rspamd.c
+++ b/src/rspamd.c
@@ -1242,6 +1242,7 @@ main (gint argc, gchar **argv, gchar **env)
        }
 
        type = g_quark_from_static_string ("main");
+       rspamd_set_crash_handler (rspamd_main);
 
        /* First set logger to console logger */
        rspamd_main->cfg->log_type = RSPAMD_LOG_CONSOLE;