Merge: r4046, r4047, r4183: Change the way we close client sessions.

r4046:
Change the way we close client sessions.

Previously we always used SO_LINGER to make sure that all queued data
got transmitted, no matter under which circumstances we closed the
client connection.

Change this so that SO_LINGER is only activated for orderly connection
closure (ie: "Connection: close" from client or error handling), in
all other cases (usually the client connecting on us, abandon any data
queued for transmission.

This _may_ reduce the tendency of worker threads to get hung up on
network failures a little bit.

r4047:
r4046 forgot to reset SO_LINGER for pipe handling which basically
broke pipehandling.

Fixes #505

r4183:
Disable SO_LINGER when we time out a connection due to sess_timeout,
so that we do not RST connections that have still not transmitted
their
data.

Since we were able to get the writev(2) to detach the socket, we
should
not end up sleeping in the close(2) either.

We still RST the socket for all error conditions.

Ideally I would still like to RST connections that have no outstanding
data after their sess_timeout, in order to avoid the 2*RTT+misc
timeouts delays associated with loosing a TCP socket for a client
that have gone to meet some other IP#.

In particular with load-balancers, this allows the load balancer to
declare the session dead right away, and reuse it for something more
productive.

Unfortunately, this lacks OS support in all presently released
OS'es: you cannot ask if a socket is done transmitting what you
asked it to.

FreeBSD-8.0 will have experimental support for this (FIONWRITE)
and I will revisit it in that context.


git-svn-id: http://www.varnish-cache.org/svn/branches/2.0@4248 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/cache_acceptor.c

@@ -68,7 +68,9 @@ static struct acceptor const *vca_act;
 static pthread_t	vca_thread_acct;
 static struct timeval	tv_sndtimeo;
 static struct timeval	tv_rcvtimeo;
-static struct linger	linger;
+static const struct linger linger = {
+	.l_onoff	=	1,
+};
 
 static unsigned char	need_sndtimeo, need_rcvtimeo, need_linger, need_test;
 

bin/varnishd/cache_acceptor_epoll.c

@@ -118,6 +118,7 @@ vca_main(void *arg)
 				continue;
 			VTAILQ_REMOVE(&sesshead, sp, list);
 			vca_del(sp->fd);
+			TCP_linger(sp->fd, 0);
 			vca_close_session(sp, "timeout");
 			SES_Delete(sp);
 		}

bin/varnishd/cache_acceptor_kqueue.c

@@ -197,6 +197,7 @@ vca_kqueue_main(void *arg)
 			if (sp->t_open > deadline)
 				break;
 			VTAILQ_REMOVE(&sesshead, sp, list);
+			TCP_linger(sp->fd, 0);
 			vca_close_session(sp, "timeout");
 			SES_Delete(sp);
 		}

bin/varnishd/cache_acceptor_poll.c

@@ -146,6 +146,7 @@ vca_main(void *arg)
 				continue;
 			VTAILQ_REMOVE(&sesshead, sp, list);
 			vca_unpoll(fd);
+			TCP_linger(sp->fd, 0);
 			vca_close_session(sp, "timeout");
 			SES_Delete(sp);
 		}

bin/varnishd/cache_acceptor_ports.c

@@ -138,6 +138,7 @@ vca_main(void *arg)
 			VTAILQ_REMOVE(&sesshead, sp, list);
 			if(sp->fd != -1)
 				vca_del(sp->fd);
+			TCP_linger(sp->fd, 0);
 			vca_close_session(sp, "timeout");
 			SES_Delete(sp);
 		}

bin/varnishd/cache_center.c

@@ -240,8 +240,14 @@ cnt_done(struct sess *sp)
 
 	sp->t_req = NAN;
 
-	if (sp->fd >= 0 && sp->doclose != NULL)
+	if (sp->fd >= 0 && sp->doclose != NULL) {
+		/*
+		 * This is an orderly close of the connection; ditch linger 
+		 * before we close, to get queued data transmitted.
+		 */
+		TCP_linger(sp->fd, 0);
 		vca_close_session(sp, sp->doclose);
+	}
 	if (sp->fd < 0) {
 		SES_Charge(sp);
 		VSL_stats->sess_closed++;

bin/varnishd/cache_pipe.c

@@ -102,8 +102,12 @@ PipeSession(struct sess *sp)
 	sp->t_resp = TIM_real();
 
 	memset(fds, 0, sizeof fds);
+
+	TCP_linger(vc->fd, 0);
 	fds[0].fd = vc->fd;
 	fds[0].events = POLLIN | POLLERR;
+
+	TCP_linger(sp->fd, 0);
 	fds[1].fd = sp->fd;
 	fds[1].events = POLLIN | POLLERR;
 

bin/varnishtest/tests/b00029.vtc

+# $Id$
+
+test "Test orderly connection closure"
+
+
+server s1 {
+	rxreq
+	txresp -bodylen 130000
+} -start
+
+varnish v1 -vcl+backend { } -start
+
+client c1 {
+	txreq -hdr "Connection: close"
+	delay 3
+	rxresp
+	expect resp.bodylen == 130000
+} -run

bin/varnishtest/vtc_http.c

@@ -781,7 +781,7 @@ http_process(struct vtclog *vl, const char *spec, int sock, int client)
 	hp->vl = vl;
 	hp->client = client;
 	hp->timeout = 3000;
-	hp->nrxbuf = 64*1024;
+	hp->nrxbuf = 640*1024;
 	hp->vsb = vsb_newauto();
 	hp->rxbuf = malloc(hp->nrxbuf);		/* XXX */
 	AN(hp->rxbuf);

include/libvarnish.h

@@ -65,6 +65,7 @@ void TCP_myname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen);
 int TCP_filter_http(int sock);
 void TCP_blocking(int sock);
 void TCP_nonblocking(int sock);
+void TCP_linger(int sock, int linger);
 #ifdef SOL_SOCKET
 void TCP_name(const struct sockaddr *addr, unsigned l, char *abuf,
     unsigned alen, char *pbuf, unsigned plen);

lib/libvarnish/tcp.c

@@ -222,3 +222,17 @@ TCP_set_read_timeout(int s, double seconds)
 	AZ(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout));
 #endif
 }
+
+/*--------------------------------------------------------------------
+ * Set or reset SO_LINGER flag
+ */
+ 
+void
+TCP_linger(int sock, int linger)
+{
+	struct linger lin;
+
+	memset(&lin, 0, sizeof lin);
+	lin.l_onoff = linger;
+	AZ(setsockopt(sock, SOL_SOCKET, SO_LINGER, &lin, sizeof lin));
+}