pool: Wait when we run out of workers

Since the removal of dry signals, pools will spin when they run out of
threads and increment MAIN.threads_limited at a very high rate. That
spike in CPU consumption will also have detrimental effects on useful
tasks.

This change introduces a 1s delay when the pool is saturated. This
allows to periodically exercise the watchdog check. We could wait until
the watchdog times out but we would also miss potential updates to the
thread_pool_watchdog parameter.

To avoid spurious increments of MAIN.threads_limited we only take task
submissions into account and ignore timeouts of the condvar.

Refs #2942
Refs #3531

bin/varnishd/cache/cache_wrk.c

@@ -579,6 +579,7 @@ pool_herder(void *priv)
 	unsigned wthread_min;
 	uintmax_t dq = (1ULL << 31);
 	vtim_mono dqt = 0;
+	int r = 0;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 
@@ -680,11 +681,15 @@ pool_herder(void *priv)
 		if (pp->lqueue == 0) {
 			if (DO_DEBUG(DBG_VTC_MODE))
 				delay = 0.5;
-			(void)Lck_CondWait(&pp->herder_cond, &pp->mtx,
-				VTIM_real() + delay);
-		} else if (pp->nthr >= cache_param->wthread_max)
+			r = Lck_CondWait(&pp->herder_cond, &pp->mtx,
+			    VTIM_real() + delay);
+		} else if (pp->nthr >= cache_param->wthread_max) {
 			/* XXX: unsafe counters */
-			VSC_C_main->threads_limited++;
+			if (r != ETIMEDOUT)
+				VSC_C_main->threads_limited++;
+			r = Lck_CondWait(&pp->herder_cond, &pp->mtx,
+			    VTIM_real() + 1.0);
+		}
 		Lck_Unlock(&pp->mtx);
 	}
 	return (NULL);