add backtrace in unix_signal_handler

crash stack backtrace will be directed to syslog

1. make use of glic backtrace in execinfo.h. the old clib_backtrace is removed
2. install SIGABRT in signal handler, but have to remove it when backtrace is
   done. reason is to capture stack trace caused by SIGABRT. vPP ASSERT always
   call os_exit then abort().  we definitely want to know the trace of this
   situation. It is a little tricky to avoid SIGABRT infinite loop
3. always load symbols by calling  clib_elf_main_init () in main(). Otherwise,
   PC addresses instead of symbols will be displayed.

Change-Id: I150e15b94a4620b2ea4f08c73dc3e6ad1856de1e
Signed-off-by: Kingwel Xie <kingwel.xie@ericsson.com>

diff --git a/src/vlib/unix/main.c b/src/vlib/unix/main.c
old mode 100644 (file)
new mode 100755 (executable)
index f812b08..947c664
--- a/src/vlib/unix/main.c
+++ b/src/vlib/unix/main.c
@@ -73,17 +73,33 @@ unix_main_init (vlib_main_t * vm)
 
 VLIB_INIT_FUNCTION (unix_main_init);
 
+static int
+unsetup_signal_handlers (int sig)
+{
+  struct sigaction sa;
+
+  sa.sa_handler = SIG_DFL;
+  sa.sa_flags = 0;
+  sigemptyset (&sa.sa_mask);
+  return sigaction (sig, &sa, 0);
+}
+
+
+/* allocate this buffer from mheap when setting up the signal handler.
+    dangerous to vec_resize it when crashing, mheap itself might have been
+    corruptted already */
+static u8 *syslog_msg = 0;
+
 static void
 unix_signal_handler (int signum, siginfo_t * si, ucontext_t * uc)
 {
   uword fatal = 0;
-  u8 *msg = 0;
 
-  msg = format (msg, "received signal %U, PC %U",
-               format_signal, signum, format_ucontext_pc, uc);
+  syslog_msg = format (syslog_msg, "received signal %U, PC %U",
+                      format_signal, signum, format_ucontext_pc, uc);
 
   if (signum == SIGSEGV)
-    msg = format (msg, ", faulting address %p", si->si_addr);
+    syslog_msg = format (syslog_msg, ", faulting address %p", si->si_addr);
 
   switch (signum)
     {
@@ -103,6 +119,7 @@ unix_signal_handler (int signum, siginfo_t * si, ucontext_t * uc)
     case SIGSEGV:
     case SIGHUP:
     case SIGFPE:
+    case SIGABRT:
       fatal = 1;
       break;
 
@@ -113,17 +130,35 @@ unix_signal_handler (int signum, siginfo_t * si, ucontext_t * uc)
     }
 
   /* Null terminate. */
-  vec_add1 (msg, 0);
+  vec_add1 (syslog_msg, 0);
 
   if (fatal)
     {
-      syslog (LOG_ERR | LOG_DAEMON, "%s", msg);
+      syslog (LOG_ERR | LOG_DAEMON, "%s", syslog_msg);
+
+      /* Address of callers: outer first, inner last. */
+      uword callers[15];
+      uword n_callers = clib_backtrace (callers, ARRAY_LEN (callers), 0);
+      int i;
+      for (i = 0; i < n_callers; i++)
+       {
+         vec_reset_length (syslog_msg);
+
+         syslog_msg =
+           format (syslog_msg, "#%-2d 0x%016lx %U%c", i, callers[i],
+                   format_clib_elf_symbol_with_address, callers[i], 0);
+
+         syslog (LOG_ERR | LOG_DAEMON, "%s", syslog_msg);
+       }
+
+      /* have to remove SIGABRT to avoid recusive - os_exit calling abort() */
+      unsetup_signal_handlers (SIGABRT);
+
       os_exit (1);
     }
   else
-    clib_warning ("%s", msg);
+    clib_warning ("%s", syslog_msg);
 
-  vec_free (msg);
 }
 
 static clib_error_t *
@@ -132,6 +167,9 @@ setup_signal_handlers (unix_main_t * um)
   uword i;
   struct sigaction sa;
 
+  /* give a big enough buffer for msg, most likely it can avoid vec_resize  */
+  vec_alloc (syslog_msg, 2048);
+
   for (i = 1; i < 32; i++)
     {
       memset (&sa, 0, sizeof (sa));
@@ -141,7 +179,6 @@ setup_signal_handlers (unix_main_t * um)
       switch (i)
        {
          /* these signals take the default action */
-       case SIGABRT:
        case SIGKILL:
        case SIGSTOP:
        case SIGUSR1:
@@ -626,6 +663,9 @@ vlib_unix_main (int argc, char *argv[])
     }
   unformat_free (&input);
 
+  /* always load symbols, for signal handler and mheap memory get/put backtrace */
+  clib_elf_main_init (vm->name);
+
   vlib_thread_stack_init (0);
 
   __os_thread_index = 0;

diff --git a/src/vppinfra/backtrace.c b/src/vppinfra/backtrace.c
old mode 100644 (file)
new mode 100755 (executable)
index bbfb792..ca7591c
--- a/src/vppinfra/backtrace.c
+++ b/src/vppinfra/backtrace.c
@@ -219,43 +219,36 @@ backtrace_done:
 #ifndef clib_backtrace_defined
 #define clib_backtrace_defined
 
-typedef struct clib_generic_stack_frame_t
-{
-  struct clib_generic_stack_frame_t *prev;
-  void *return_address;
-} clib_generic_stack_frame_t;
+/* use glibc backtrace for stack trace */
+#include <execinfo.h>
 
-/* This will only work if we have a frame pointer.
-   Without a frame pointer we have to parse the machine code to
-   parse the stack frames. */
 uword
 clib_backtrace (uword * callers, uword max_callers, uword n_frames_to_skip)
 {
-  clib_generic_stack_frame_t *f;
-  uword i;
-
-  f = __builtin_frame_address (0);
-
+  int size;
+  void *array[20];
   /* Also skip current frame. */
   n_frames_to_skip += 1;
 
-  for (i = 0; i < max_callers + n_frames_to_skip; i++)
+  size = clib_min (ARRAY_LEN (array), max_callers + n_frames_to_skip);
+
+  size = backtrace (array, size);
+
+  uword i;
+
+  for (i = 0; i < max_callers + n_frames_to_skip && i < size; i++)
     {
-      f = f->prev;
-      if (!f)
-       goto backtrace_done;
-      if (clib_abs ((void *) f - (void *) f->prev) > (64 * 1024))
-       goto backtrace_done;
       if (i >= n_frames_to_skip)
-       callers[i - n_frames_to_skip] = pointer_to_uword (f->return_address);
+       callers[i - n_frames_to_skip] = pointer_to_uword (array[i]);
     }
 
-backtrace_done:
   if (i < n_frames_to_skip)
     return 0;
   else
     return i - n_frames_to_skip;
 }
+
+
 #endif /* clib_backtrace_defined */
 
 /*