Tell the waiter (if necessary) that we steal a filedescriptor, not

all of them find out on their own.

bin/varnishd/cache/cache_backend_tcp.c

@@ -384,6 +384,8 @@ VBT_Get(struct tcp_pool *tp, double tmo)
 		CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC);
 
 		assert(vbc->in_waiter == VBC_W_INWAIT);
+VSL(SLT_Debug, 0, "------> Steal fd %d", vbc->fd);
+		Wait_Steal(tp->waiter, vbc->waited);
 		vbc->in_waiter = VBC_W_STOLEN;
 		pfd.fd = vbc->fd;
 		pfd.events = POLLIN;

bin/varnishd/waiter/cache_waiter.c

@@ -203,6 +203,36 @@ Wait_Enter(const struct waiter *w, struct waited *wp)
 	return (0);
 }
 
+int
+Wait_Steal(const struct waiter *w, struct waited *wp)
+{
+	ssize_t written;
+	uintptr_t up;
+
+	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
+	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
+	assert(wp->fd > 0);		// stdin never comes here
+	AZ(w->dismantle);
+
+	if (w->impl->pass != NULL) {
+		INCOMPL();
+	}
+
+	assert(w->pipes[1] > 0);
+
+	if (w->impl->evict == NULL)
+		return (0);
+
+	up = (uintptr_t)wp;
+	AZ(up & 1);
+	up |= 1;
+	written = write(w->pipes[1], &up, sizeof up);
+	if (written != sizeof up && (errno == EAGAIN || errno == EWOULDBLOCK))
+		return (-1);
+	assert (written == sizeof up);
+	return (0);
+}
+
 static void
 wait_updidle(struct waiter *w, double now)
 {
@@ -226,6 +256,7 @@ Wait_Handle(struct waiter *w, struct waited *wp, enum wait_event ev, double now)
 	uintptr_t ss[NEV];
 	struct waited *wp2;
 	int i, j, dotimer = 0;
+	int steal;
 
 	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
@@ -253,9 +284,17 @@ Wait_Handle(struct waiter *w, struct waited *wp, enum wait_event ev, double now)
 			AN(w->dismantle);
 			continue;
 		}
+		steal = ss[j] & 1;
+		ss[j] &= ~1;
 		CAST_OBJ_NOTNULL(wp2, (void*)ss[j], WAITED_MAGIC);
 		if (wp2 == w->pipe_w) {
 			dotimer = 1;
+		} else if (steal) {
+			assert(wp2->fd >= 0);
+			VTAILQ_REMOVE(&w->waithead, wp2, list);
+			AN (w->impl->evict);
+			w->impl->evict(w, wp2);
+			w->func(wp2, WAITER_ACTION, now);
 		} else {
 			assert(wp2->fd >= 0);
 			VTAILQ_INSERT_TAIL(&w->waithead, wp2, list);

bin/varnishd/waiter/cache_waiter_kqueue.c

@@ -99,6 +99,17 @@ vwk_inject(const struct waiter *w, struct waited *wp)
 		vwk_kq_sess(vwk, wp, EV_ADD | EV_ONESHOT);
 }
 
+#if 0
+static void
+vwk_evict(const struct waiter *w, struct waited *wp)
+{
+	struct vwk *vwk;
+
+	CAST_OBJ_NOTNULL(vwk, w->priv, VWK_MAGIC);
+	vwk_kq_sess(vwk, wp, EV_DELETE);
+}
+#endif
+
 /*--------------------------------------------------------------------*/
 
 static void
@@ -193,6 +204,7 @@ const struct waiter_impl waiter_kqueue = {
 	.init =		vwk_init,
 	.fini =		vwk_fini,
 	.inject =	vwk_inject,
+	// .evict =	vwk_evict,
 	.size =		sizeof(struct vwk),
 };
 

bin/varnishd/waiter/waiter.h

@@ -58,6 +58,7 @@ typedef void waiter_handle_f(struct waited *, enum wait_event, double now);
 
 /* cache_waiter.c */
 int Wait_Enter(const struct waiter *, struct waited *);
+int Wait_Steal(const struct waiter *, struct waited *);
 struct waiter *Wait_New(waiter_handle_f *, volatile double *timeout);
 void Wait_Destroy(struct waiter **);
 const char *Wait_GetName(void);