Limit watchdog to highest priority only

The watchdog mechanism currently triggers when any queueing is happening,
regardless of the priority. Strictly speaking it is only the backend
fetches that are critical to get executed, and this prevents the thread
limits to be used as limits on the amount of work the Varnish instance
should handle.

This can be especially important for instances with H/2 enabled, as these
connections will be holding threads for extended periods of time, possibly
triggering the watchdog in benign situations.

This patch limits the watchdog to only trigger for no queue development
on the highest priority queue.

bin/varnishd/cache/cache.h

@@ -220,6 +220,7 @@ enum task_prio {
 	TASK_QUEUE__END
 };
 
+#define TASK_QUEUE_HIGHEST_PRIORITY TASK_QUEUE_BO
 #define TASK_QUEUE_CLIENT(prio) \
 	(prio == TASK_QUEUE_REQ || prio == TASK_QUEUE_STR)
 

bin/varnishd/cache/cache_wrk.c

@@ -600,7 +600,9 @@ pool_herder(void *priv)
 		 * Instead we implement a watchdog and kill the worker if
 		 * nothing has been dequeued for that long.
 		 */
-		if (pp->lqueue == 0) {
+		if (VTAILQ_EMPTY(&pp->queues[TASK_QUEUE_HIGHEST_PRIORITY])) {
+			/* Watchdog only applies to no movement on the
+			 * highest priority queue (TASK_QUEUE_BO) */
 			dq = pp->ndequeued + 1;
 		} else if (dq != pp->ndequeued) {
 			dq = pp->ndequeued;

bin/varnishtest/tests/c00104.vtc

+varnishtest "Test watchdog only active on queue 0"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -cliok "param.set thread_pools 1"
+varnish v1 -cliok "param.set thread_pool_min 5"
+varnish v1 -cliok "param.set thread_pool_max 5"
+varnish v1 -cliok "param.set thread_pool_watchdog 1"
+varnish v1 -cliok "param.set feature +http2"
+
+varnish v1 -vcl+backend {
+} -start
+
+client c1 {
+	txpri
+	delay 2
+} -start
+
+client c2 {
+	txpri
+	delay 2
+} -start
+
+client c3 {
+	txpri
+	delay 2
+} -start
+
+delay 2