MQ plugin for Kafka: signal USR2 toggles debug level logging

trackrdrd/src/mq/kafka/README.rst

@@ -243,6 +243,9 @@ Statistic             Description
                       payload.
 ===================== ==========================================================
 
+The log level can be toggled to DEBUG and back by sending signal
+``USR2`` to the process, as described below.
+
 MESSAGE SEND FAILURE AND RECOVERY
 =================================
 
@@ -278,7 +281,20 @@ point in time.
 SIGNALS
 =======
 
-XXX: TuDu -- toggle DEBUG log level
+The message plugin overrides the signal handler of the tracking
+reader's child process for signal ``USR2`` (see signal(7)), so that it
+toggles the DEBUG log level when the process receives the signal.
+
+The initial log level is set by the configuration parameter
+``mq.debug`` when the plugin is initialized, and the level is changed
+from this level to DEBUG, or from DEBUG back to the initial level,
+when ``USR2`` is sent to the process (for example by using
+kill(1)). Log level toggling affects logging for the messaging plugin
+as well as the rdkafka and zookeeper client libraries.
+
+Logging at DEBUG level may be very verbose, so that log files may
+become very large (and partitions may overflow) if DEBUG level is left
+on for a long time.
 
 SEE ALSO
 ========

trackrdrd/src/mq/kafka/mq.c

@@ -37,6 +37,7 @@
 #include <limits.h>
 #include <syslog.h>
 #include <ctype.h>
+#include <signal.h>
 
 #include <zookeeper/zookeeper.h>
 #include <zookeeper/zookeeper_version.h>
@@ -75,11 +76,42 @@ static char topic[LINE_MAX] = "";
 static rd_kafka_topic_conf_t *topic_conf;
 static rd_kafka_conf_t *conf;
 
+static unsigned stats_interval = 0;
+
 static char errmsg[LINE_MAX];
 static char _version[LINE_MAX];
 
 static int loglvl = LOG_INFO;
-static unsigned stats_interval = 0;
+static int saved_lvl = LOG_INFO;
+static int debug_toggle = 0;
+struct sigaction toggle_action;
+
+static void
+toggle_debug(int sig)
+{
+    (void) sig;
+
+    if (debug_toggle) {
+        /* Toggle from debug back to saved level */
+        loglvl = saved_lvl;
+        debug_toggle = 0;
+        MQ_LOG_Log(LOG_INFO, "Debug toggle switched off");
+        zoo_set_debug_level(ZOO_LOG_LEVEL_INFO);
+    }
+    else {
+        saved_lvl = loglvl;
+        loglvl = LOG_DEBUG;
+        debug_toggle = 1;
+        MQ_LOG_Log(LOG_INFO, "Debug toggle switched on");
+        zoo_set_debug_level(ZOO_LOG_LEVEL_DEBUG);
+    }
+    MQ_LOG_SetLevel(loglvl);
+    for (int i = 0; i < nwrk; i++)
+        if (workers[i] != NULL) {
+            CHECK_OBJ(workers[i], KAFKA_WRK_MAGIC);
+            rd_kafka_set_log_level(workers[i]->kafka, loglvl);
+        }
+}
 
 static void
 log_cb(const rd_kafka_t *rk, int level, const char *fac, const char *buf)
@@ -397,6 +429,16 @@ MQ_GlobalInit(unsigned nworkers, const char *config_fname)
         return errmsg;
     }
 
+    toggle_action.sa_handler = toggle_debug;
+    AZ(sigemptyset(&toggle_action.sa_mask));
+    toggle_action.sa_flags |= SA_RESTART;
+    if (sigaction(SIGUSR2, &toggle_action, NULL) != 0) {
+        snprintf(errmsg, LINE_MAX, "Cannot install signal handler for USR2: %s",
+                 strerror(errno));
+        MQ_LOG_Log(LOG_ERR, errmsg);
+        return errmsg;
+    }
+
     if (zoolog[0] != '\0') {
         zoologf = fopen(zoolog, "a");
         if (zoologf == NULL) {