Start weaning waiters off sessions.

bin/varnishd/http1/cache_http1_fsm.c

@@ -111,7 +111,13 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req)
 			if (when < now || tmo == 0) {
 				wrk->stats->sess_herd++;
 				SES_ReleaseReq(req);
-				WAIT_Enter(sp);
+				if (VTCP_nonblocking(sp->fd))
+					SES_Close(sp, SC_REM_CLOSE);
+				else if (WAIT_Enter(sp)) {
+					VSC_C_main->sess_pipe_overflow++;
+					SES_Delete(sp, SC_SESS_PIPE_OVERFLOW,
+					    NAN);
+				}
 				return (REQ_FSM_DONE);
 			}
 		} else {

bin/varnishd/waiter/cache_waiter.c

@@ -38,8 +38,6 @@
 
 #include "waiter/waiter.h"
 
-#include "vtcp.h"
-
 static void *waiter_priv;
 
 const char *
@@ -63,31 +61,27 @@ WAIT_Init(void)
 	waiter_priv = waiter->init();
 }
 
-void
+int
 WAIT_Enter(struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	assert(sp->fd >= 0);
 
-	/*
-	* Set nonblocking in the worker-thread, before passing to the
-	* acceptor thread, to reduce syscall density of the latter.
-	*/
-	if (VTCP_nonblocking(sp->fd))
-		SES_Close(sp, SC_REM_CLOSE);
-	waiter->pass(waiter_priv, sp);
+	return (waiter->pass(waiter_priv, sp));
 }
 
-void
+/*
+ * We do not make sp a const, in order to hint that we actually do take
+ * control of it.
+ */
+int __match_proto__()
 WAIT_Write_Session(struct sess *sp, int fd)
 {
 	ssize_t written;
 	written = write(fd, &sp, sizeof sp);
-	if (written != sizeof sp && (errno == EAGAIN || errno == EWOULDBLOCK)) {
-		VSC_C_main->sess_pipe_overflow++;
-		SES_Delete(sp, SC_SESS_PIPE_OVERFLOW, NAN);
-		return;
-	}
+	if (written != sizeof sp && (errno == EAGAIN || errno == EWOULDBLOCK))
+		return (-1);
 	assert (written == sizeof sp);
+	return (0);
 }

bin/varnishd/waiter/cache_waiter_epoll.c

@@ -219,14 +219,14 @@ vwe_timeout_idle_ticker(void *priv)
 
 /*--------------------------------------------------------------------*/
 
-static void
+static int
 vwe_pass(void *priv, struct sess *sp)
 {
 	struct vwe *vwe;
 
 	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
 
-	WAIT_Write_Session(sp, vwe->pipes[1]);
+	return (WAIT_Write_Session(sp, vwe->pipes[1]));
 }
 
 /*--------------------------------------------------------------------*/

bin/varnishd/waiter/cache_waiter_kqueue.c

@@ -215,14 +215,14 @@ vwk_thread(void *priv)
 
 /*--------------------------------------------------------------------*/
 
-static void
+static int
 vwk_pass(void *priv, struct sess *sp)
 {
 	struct vwk *vwk;
 
 	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
 
-	WAIT_Write_Session(sp, vwk->pipes[1]);
+	return (WAIT_Write_Session(sp, vwk->pipes[1]));
 }
 
 /*--------------------------------------------------------------------*/

bin/varnishd/waiter/cache_waiter_poll.c

@@ -192,14 +192,14 @@ vwp_main(void *priv)
 
 /*--------------------------------------------------------------------*/
 
-static void
+static int
 vwp_poll_pass(void *priv, struct sess *sp)
 {
 	struct vwp *vwp;
 
 	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
 
-	WAIT_Write_Session(sp, vwp->pipes[1]);
+	return (WAIT_Write_Session(sp, vwp->pipes[1]));
 }
 
 /*--------------------------------------------------------------------*/

bin/varnishd/waiter/cache_waiter_ports.c

@@ -239,7 +239,7 @@ vws_thread(void *priv)
 
 /*--------------------------------------------------------------------*/
 
-static void
+static int
 vws_pass(void *priv, struct sess *sp)
 {
 	int r;
@@ -247,12 +247,10 @@ vws_pass(void *priv, struct sess *sp)
 
 	CAST_OBJ_NOTNULL(vws, priv, VWS_MAGIC);
 	r = port_send(vws->dport, 0, TRUST_ME(sp));
-	if (r == -1 && errno == EAGAIN) {
-		VSC_C_main->sess_pipe_overflow++;
-		SES_Delete(sp, SC_SESS_PIPE_OVERFLOW, NAN);
-		return;
-	}
+	if (r == -1 && errno == EAGAIN)
+		return (-1);
 	AZ(r);
+	return (0);
 }
 
 /*--------------------------------------------------------------------*/

bin/varnishd/waiter/waiter.h

@@ -31,7 +31,7 @@
 struct sess;
 
 typedef void* waiter_init_f(void);
-typedef void waiter_pass_f(void *priv, struct sess *);
+typedef int waiter_pass_f(void *priv, struct sess *);
 
 #define WAITER_DEFAULT		"platform dependent"
 
@@ -42,10 +42,10 @@ struct waiter {
 };
 
 /* cache_waiter.c */
-void WAIT_Enter(struct sess *sp);
+int WAIT_Enter(struct sess *sp);
 void WAIT_Init(void);
 const char *WAIT_GetName(void);
-void WAIT_Write_Session(struct sess *sp, int fd);
+int WAIT_Write_Session(struct sess *sp, int fd);
 
 /* mgt_waiter.c */
 extern struct waiter const * waiter;