trackrdrd: signal handling with sigaction, added a handler for stack traces

           attempt to make AMQ_Worker destructor more robust
           shutdown call for the monitor thread

trackrdrd/configure.ac

@@ -43,6 +43,7 @@ PKG_PROG_PKG_CONFIG
 
 # Checks for header files.
 AC_HEADER_STDC
+AC_CHECK_HEADERS([execinfo.h])
 
 ## From Varnish configure.ac
 # white lie - we don't actually test it

trackrdrd/src/activemq/amq.cpp

@@ -84,16 +84,25 @@ AMQ_Worker::AMQ_Worker(std::string& qName,
 }
 
 AMQ_Worker::~AMQ_Worker() {
-    if (connection != NULL)
-        connection->close();
-    delete producer;
-    producer = NULL;
-    delete queue;
-    queue = NULL;
-    delete session;
-    session = NULL;
-    delete connection;
-    connection = NULL;
+    if (producer != NULL) {
+	delete producer;
+	producer = NULL;
+    }
+    if (queue != NULL) {
+	delete queue;
+	queue = NULL;
+    }
+    if (session != NULL) {
+	delete session;
+	session = NULL;
+    }
+    if (connection != NULL) {
+#if 0	    
+	connection->close();
+#endif	    
+	delete connection;
+	connection = NULL;
+    }
 }
 
 /* XXX: Timeout */

trackrdrd/src/monitor.c

@@ -38,6 +38,8 @@
 #include "trackrdrd.h"
 #include "vas.h"
 
+static int run;
+
 void
 *MON_StatusThread(void *arg)
 {
@@ -48,12 +50,14 @@ void
     t.tv_nsec = (long)(t.tv_sec - *interval) * 10e9;
     LOG_Log(LOG_INFO, "Monitor thread running every %.2f secs",
             t.tv_sec + ((float) t.tv_nsec * 10e-9));
+    run = 1;
     
-    while (1) {
+    while (run) {
         int err;
         if (nanosleep(&t, NULL) != 0) {
             if (errno == EINTR) {
-                /* XXX: Terminate on TERM, INT, etc. */
+                if (run == 0)
+		    break;
                 LOG_Log0(LOG_INFO, "Monitoring thread interrupted");
                 continue;
             }
@@ -73,6 +77,16 @@ void
             tbl.occ_hi, tbl.seen, tbl.submitted, tbl.sent, tbl.failed,
             tbl.wait_qfull, tbl.data_hi);
     }
+
+    LOG_Log0(LOG_INFO, "Monitoring thread exiting");
+    pthread_exit((void *) NULL);
+}
+
+void
+MON_StatusShutdown(pthread_t monitor)
+{
+    run = 0;
+    AZ(pthread_join(monitor, NULL));
 }
 
 void

trackrdrd/src/signals.h

@@ -30,12 +30,18 @@
  */
 
 
-PARENT(SIGTERM, terminate);
-PARENT(SIGINT, terminate);
-PARENT(SIGUSR1, SIG_IGN);
-PARENT(SIGUSR2, SIG_IGN);
+PARENT(SIGTERM, terminate_action);
+PARENT(SIGINT, terminate_action);
+PARENT(SIGUSR1, ignore_action);
+PARENT(SIGUSR2, ignore_action);
+PARENT(SIGABRT, stacktrace_action);
+PARENT(SIGSEGV, stacktrace_action);
+PARENT(SIGBUS, stacktrace_action);
 
-CHILD(SIGTERM, terminate);
-CHILD(SIGINT, terminate);
-CHILD(SIGUSR1, dump);
-CHILD(SIGUSR2, SIG_IGN);
+CHILD(SIGTERM, terminate_action);
+CHILD(SIGINT, terminate_action);
+CHILD(SIGUSR1, dump_action);
+CHILD(SIGUSR2, ignore_action);
+CHILD(SIGABRT, stacktrace_action);
+CHILD(SIGSEGV, stacktrace_action);
+CHILD(SIGBUS, stacktrace_action);

trackrdrd/src/trackrdrd.c

@@ -50,6 +50,11 @@
 #include <sys/wait.h>
 #include <sys/types.h>
 
+#ifndef HAVE_EXECINFO_H
+#include "compat/execinfo.h"
+#else
+#include <execinfo.h>
+#endif
 #include "compat/daemon.h"
 
 #include "vsb.h"
@@ -70,13 +75,16 @@
 
 #define DEFAULT_CONFIG "/etc/trackrdrd.conf"
 
+/* XXX: should this be configurable ? */
+#define MAX_STACK_DEPTH 100
+
 /* Hack, because we cannot have #ifdef in the macro definition SIGDISP */
 #define _UNDEFINED(SIG) ((#SIG)[0] == 0)
 #define UNDEFINED(SIG) _UNDEFINED(SIG)
 
-#define SIGDISP(SIG, disp)						\
+#define SIGDISP(SIG, action)						\
     do { if (UNDEFINED(SIG)) break;					\
-         if (signal((SIG), (disp)) == SIG_ERR)                          \
+	if (sigaction((SIG), (&action), NULL) != 0)			\
              LOG_Log(LOG_ALERT,						\
                  "Cannot install handler for " #SIG ": %s",		\
                  strerror(errno));					\
@@ -86,6 +94,9 @@ static void child_main(struct VSM_data *vd, int endless);
 
 static volatile sig_atomic_t term;
 
+static struct sigaction terminate_action, dump_action, ignore_action,
+	stacktrace_action, default_action;
+
 /*--------------------------------------------------------------------*/
 
 static void
@@ -250,21 +261,54 @@ OSL_Track(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
 /* Handle for the PID file */
 struct vpf_fh *pfh = NULL;
 
+static void
+stacktrace(int sig)
+{
+    void *buf[MAX_STACK_DEPTH];
+    int depth, i;
+    char **strings;
+
+    depth = backtrace (buf, MAX_STACK_DEPTH);
+    LOG_Log(LOG_ALERT, "Received signal %d (%s), stacktrace follows", sig,
+	strsignal(sig));
+    if (depth == 0) {
+	LOG_Log0(LOG_ERR, "Stacktrace empty");
+	return;
+    }
+    strings = backtrace_symbols(buf, depth);
+    if (strings == NULL) {
+	LOG_Log0(LOG_ERR, "Cannot retrieve symbols for stacktrace");
+	return;
+    }
+    for (i = 0; i < depth; i++)
+	LOG_Log(LOG_ERR, "%p: %s", buf[i], strings[i]);
+    
+    free(strings);
+}
+
 static void
 dump(int sig)
 {
+    (void) sig;
     DATA_Dump();
-    signal(sig, dump);
 }
 
 static void
 terminate(int sig)
 {
-    LOG_Log0(LOG_DEBUG, "Signal handler terminate called");
     (void) sig;
     term = 1;
 }
 
+static void
+stacktrace_abort(int sig)
+{
+    stacktrace(sig);
+    AZ(sigaction(SIGABRT, &default_action, NULL));
+    LOG_Log0(LOG_ALERT, "Aborting");
+    abort();
+}
+
 static void
 parent_shutdown(int status, pid_t child_pid)
 {
@@ -325,8 +369,9 @@ parent_main(pid_t child_pid, struct VSM_data *vd, int endless)
             LOG_Log(LOG_WARNING, "Worker process %d exited with status %d",
                 child_pid, WEXITSTATUS(status));
         if (WIFSIGNALED(status))
-            LOG_Log(LOG_WARNING, "Worker process %d exited due to signal %d",
-                child_pid, WTERMSIG(status));
+            LOG_Log(LOG_WARNING,
+		"Worker process %d exited due to signal %d (%s)",
+                child_pid, WTERMSIG(status), strsignal(WTERMSIG(status)));
 
         if (config.restarts && restarts > config.restarts) {
             LOG_Log(LOG_ERR, "Too many restarts: %d", restarts);
@@ -431,6 +476,7 @@ child_main(struct VSM_data *vd, int endless)
     /* Main loop */
     term = 0;
     /* XXX: Varnish restart? */
+    /* XXX: TERM not noticed until request received */
     while (VSL_Dispatch(vd, OSL_Track, NULL))
         if (term || !endless)
             break;
@@ -438,6 +484,7 @@ child_main(struct VSM_data *vd, int endless)
     WRK_Halt();
     WRK_Shutdown();
     AZ(MQ_GlobalShutdown());
+    MON_StatusShutdown(monitor);
     LOG_Log0(LOG_INFO, "Worker process exiting");
     LOG_Close();
     exit(EXIT_SUCCESS);
@@ -568,6 +615,19 @@ main(int argc, char * const *argv)
 	if (pfh != NULL)
 		VPF_Write(pfh);
 
+	terminate_action.sa_handler = terminate;
+	AZ(sigemptyset(&terminate_action.sa_mask));
+	terminate_action.sa_flags &= ~SA_RESTART;
+
+	dump_action.sa_handler = dump;
+	AZ(sigemptyset(&dump_action.sa_mask));
+	dump_action.sa_flags |= SA_RESTART;
+
+	stacktrace_action.sa_handler = stacktrace_abort;
+
+	ignore_action.sa_handler = SIG_IGN;
+	default_action.sa_handler = SIG_DFL;
+
         if (!D_flag) {
             child_pid = fork();
             switch(child_pid) {

trackrdrd/src/trackrdrd.h

@@ -188,6 +188,7 @@ typedef enum {
 } stats_update_t;
 
 void *MON_StatusThread(void *arg);
+void MON_StatusShutdown(pthread_t monitor);
 void MON_StatsInit(void);
 void MON_StatsUpdate(stats_update_t update);