wake up herder from sleeping after destroying, sleep for longer

Previously, after sending a thread to varnish heaven, the herder slept
for wthread_destroy_delay unconditionally. Instead, we now wait on the
cv so we get woken up in case we run dry during the delay.

This change is relevant proportionally to the value of
wthread_destroy_delay if the spread between thread_pool_min and
thread_pool_max is big and varnish is exposed to sudden traffic peaks.

IOW, it will probably be only relevant for high performance setups.

Also, we now sleep for thread_pool_timeout unless a shorter delay is
warranted. This will delay the effect of thread parameter changes for
up to thread_pool_timeout seconds unless the pool runs dry, in which
case they will become effective immediately.

bin/varnishd/cache/cache_wrk.c

@@ -435,6 +435,7 @@ pool_herder(void *priv)
 	struct pool_task *pt;
 	double t_idle;
 	struct worker *wrk;
+	int delay;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 
@@ -447,6 +448,8 @@ pool_herder(void *priv)
 			pool_breed(pp);
 			continue;
 		}
+
+		delay = cache_param->wthread_timeout;
 		assert(pp->nthr >= cache_param->wthread_min);
 
 		if (pp->nthr > cache_param->wthread_min) {
@@ -472,8 +475,10 @@ pool_herder(void *priv)
 					    &wrk->task, list);
 					wrk->task.func = pool_kiss_of_death;
 					AZ(pthread_cond_signal(&wrk->cond));
-				} else
+				} else {
+					delay = wrk->lastused - t_idle;
 					wrk = NULL;
+				}
 			}
 			Lck_Unlock(&pp->mtx);
 
@@ -483,15 +488,15 @@ pool_herder(void *priv)
 				VSC_C_main->threads--;
 				VSC_C_main->threads_destroyed++;
 				Lck_Unlock(&pool_mtx);
-				VTIM_sleep(cache_param->wthread_destroy_delay);
-				continue;
-			}
+				delay = cache_param->wthread_destroy_delay;
+			} else if (delay < cache_param->wthread_destroy_delay)
+				delay = cache_param->wthread_destroy_delay;
 		}
 
 		Lck_Lock(&pp->mtx);
 		if (!pp->dry) {
 			(void)Lck_CondWait(&pp->herder_cond, &pp->mtx,
-				VTIM_real() + 5);
+				VTIM_real() + delay);
 		} else {
 			/* XXX: unsafe counters */
 			VSC_C_main->threads_limited++;

bin/varnishtest/tests/r01490.vtc

@@ -9,6 +9,7 @@ varnish v1 \
 	-arg "-p thread_pool_min=2" \
 	-arg "-p thread_pool_max=3" \
 	-arg "-p thread_pools=1" \
+	-arg "-p thread_pool_timeout=10" \
 	-vcl+backend {}
 varnish v1 -start
 
@@ -21,16 +22,16 @@ logexpect l1 -v v1 -g raw {
 
 varnish v1 -cliok "param.set thread_pool_min 3"
 
-# Herder thread sleeps 5 seconds. Have to wait longer than that.
-delay 6
+# Have to wait longer than thread_pool_timeout
+delay 11
 
 varnish v1 -expect threads == 3
 
 varnish v1 -cliok "param.set thread_pool_min 2"
 varnish v1 -cliok "param.set thread_pool_max 2"
 
-# Herder thread sleeps 5 seconds. Have to wait longer than that.
-delay 6
+# Have to wait longer than thread_pool_timeout
+delay 11
 
 varnish v1 -expect threads == 2