Merge branch 'master' into 4.1

bin/varnishd/cache/cache_esi_deliver.c

@@ -228,7 +228,7 @@ VDP_ESI(struct req *req, enum vdp_action act, void **priv,
 	uint32_t icrc = 0;
 	uint8_t tailbuf[8 + 5];
 	const uint8_t *pp;
-	struct ecx *ecx;
+	struct ecx *ecx, *pecx;
 	int retval = 0;
 
 	if (act == VDP_INIT) {
@@ -360,6 +360,13 @@ VDP_ESI(struct req *req, enum vdp_action act, void **priv,
 
 				(void)VDP_bytes(req, VDP_NULL, tailbuf, 13);
 			}
+			if (req->transport->deliver == VED_Deliver) {
+				CAST_OBJ_NOTNULL(pecx, req->transport_priv,
+				    ECX_MAGIC);
+				pecx->crc = crc32_combine(pecx->crc,
+				    ecx->crc, ecx->l_crc);
+				pecx->l_crc += ecx->l_crc;
+			}
 			retval = VDP_bytes(req, VDP_FLUSH, NULL, 0);
 			ecx->state = 99;
 			return (retval);

bin/varnishd/http1/cache_http1_proto.c

@@ -282,15 +282,19 @@ http1_body_status(struct http *hp, struct http_conn *htc)
 
 	htc->content_length = -1;
 
-	if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) {
-		http_Unset(hp, H_Content_Length);
+	cl = http_GetContentLength(hp);
+	if (http_GetHdr(hp, H_Transfer_Encoding, &b)) {
+		if (strcasecmp(b, "chunked"))
+			return (BS_ERROR);
+		if (cl != -1) {
+			/*
+			 * RFC7230 3.3.3 allows more lenient handling
+			 * but we're going to be strict.
+			 */
+			return (BS_ERROR);
+		}
 		return (BS_CHUNKED);
 	}
-
-	if (http_GetHdr(hp, H_Transfer_Encoding, &b))
-		return (BS_ERROR);
-
-	cl = http_GetContentLength(hp);
 	if (cl == -2)
 		return (BS_ERROR);
 	if (cl >= 0) {
@@ -298,6 +302,9 @@ http1_body_status(struct http *hp, struct http_conn *htc)
 		return (cl == 0 ? BS_NONE : BS_LENGTH);
 	}
 
+	if (hp->protover == 11)
+		return (BS_NONE);
+
 	if (http_HdrIs(hp, H_Connection, "keep-alive")) {
 		/*
 		 * Keep alive with neither TE=Chunked or C-Len is impossible.
@@ -306,20 +313,6 @@ http1_body_status(struct http *hp, struct http_conn *htc)
 		return (BS_NONE);
 	}
 
-	if (http_HdrIs(hp, H_Connection, "close")) {
-		/*
-		 * In this case, it is safe to just read what comes.
-		 */
-		return (BS_EOF);
-	}
-
-	if (hp->protover < 11) {
-		/*
-		 * With no Connection header, assume EOF.
-		 */
-		return (BS_EOF);
-	}
-
 	/*
 	 * Fall back to EOF transfer.
 	 */

bin/varnishtest/tests/b00011.vtc

@@ -2,7 +2,7 @@ varnishtest "Check HTTP/1.0 EOF transmission"
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 OK\n"
+	send "HTTP/1.0 200 OK\n"
 	send "Connection: close\n"
 	send "\n"
 	send "Body line 1\n"

bin/varnishtest/tests/b00020.vtc

@@ -4,7 +4,7 @@ feature SO_RCVTIMEO_WORKS
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
+	send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n"
 	delay 1.5
 	# send "Baba\n"
 } -start

bin/varnishtest/tests/b00021.vtc

@@ -4,7 +4,7 @@ feature SO_RCVTIMEO_WORKS
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
+	send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n"
 	delay 4.0
 	send "Baba\n"
 } -start
@@ -26,7 +26,7 @@ client c1 {
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
+	send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n"
 	delay 1.0
 	send "Baba\n"
 	delay 1.0

bin/varnishtest/tests/b00022.vtc

@@ -4,7 +4,7 @@ feature SO_RCVTIMEO_WORKS
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
+	send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n"
 	delay 1.5
 	send "Baba\n"
 } -start
@@ -28,7 +28,7 @@ client c1 {
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
+	send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n"
 	delay 0.5
 	send "Baba\n"
 	delay 0.5

bin/varnishtest/tests/c00013.vtc

@@ -3,7 +3,7 @@ varnishtest "Test parking second request on backend delay"
 server s1 {
 	rxreq
 	expect req.url == "/foo"
-	send "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
+	send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n"
 	delay .2
 	sema r1 sync 2
 	delay .2

bin/varnishtest/tests/c00070.vtc

@@ -28,9 +28,9 @@ varnish v1 -vcl+backend {
 } -start
 
 logexpect l1 -v v1 -d 1 -g vxid -q "Error ~ 'overflow'" {
-        expect 0 *      Begin
-        expect * =      Error	"workspace_client overflow"
-        expect * =      End
+	expect 0 *	Begin
+	expect * =	Error	"workspace_client overflow"
+	expect * =	End
 } -start
 
 client c1 {

bin/varnishtest/tests/e00007.vtc

@@ -16,7 +16,7 @@ varnishtest "ESI spanning storage bits"
 server s1 {
 	rxreq
 	expect req.url == "/foo/bar"
-	send "HTTP/1.1 200 OK\n"
+	send "HTTP/1.0 200 OK\n"
 	send "Connection: close\n"
 	send "\n"
 	send {

bin/varnishtest/tests/e00014.vtc

@@ -3,7 +3,7 @@ varnishtest "Check <esi: detector"
 server s1 {
 	rxreq
 	expect req.url == "/foo"
-	send "HTTP/1.1 200 OK\n"
+	send "HTTP/1.0 200 OK\n"
 	send "Connection: close\n"
 	send "\n"
 	send {           <a>   <esi/>                          }

bin/varnishtest/tests/g00002.vtc

@@ -4,7 +4,7 @@ server s1 {
 	rxreq
 	expect req.http.accept-encoding == "gzip"
 	expect req.url == "/foo"
-	txresp -nolen -gziplen 4100
+	txresp -proto HTTP/1.0 -nolen -gziplen 4100
 	accept
 	rxreq
 	expect req.url == "/bar"

bin/varnishtest/tests/m00016.vtc

-varnishtest "Test std.real2time, std.time2integer and std.time2real"
+varnishtest "Test real2integer, real2time, time2integer and time2real in std"
 
 server s1 {
 	rxreq
@@ -10,8 +10,18 @@ varnish v1 -vcl+backend {
 
 	sub vcl_deliver {
 		set resp.http.x-foo = std.integer(req.http.foo, 0);
-		set resp.http.x-bar = std.time2integer(std.real2time(std.real(resp.http.x-foo, 0.0)));
-		set resp.http.x-baz = std.time2real(std.real2time(std.real(resp.http.x-foo, 0.0)));
+		set resp.http.x-bar = std.time2integer(std.real2time(
+		    std.real(resp.http.x-foo, 0.0), now), 1);
+
+		set resp.http.x-baz = std.time2real(std.real2time(
+		    std.real(resp.http.x-foo, 0.0), now), 1.0);
+
+		set resp.http.x-qux = std.real2integer(
+		    std.real(req.http.foo, 2.0), 2);
+
+		# Representation of 9e99, which is larger than what fits in the
+		# 128bit integers on $exotic_platform.
+		set resp.http.x-int-fallback = std.real2integer(9000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.000, 2);
 	}
 } -start
 
@@ -20,4 +30,15 @@ client c1 {
 	rxresp
 	expect resp.http.x-foo == resp.http.x-bar
 	expect resp.http.x-baz == 1140618699.000
+	expect resp.http.x-qux == 1140618699
+	expect resp.http.x-int-fallback == 2
+} -run
+
+# check we get the fallback if the conversion fails
+client c2 {
+	txreq -hdr "foo: -9e99+1"
+	rxresp
+	expect resp.http.x-bar == 0
+	expect resp.http.x-baz == 0.000
+	expect resp.http.x-qux == 2
 } -run

bin/varnishtest/tests/r00733.vtc

@@ -2,7 +2,7 @@ varnishtest "HTTP/1.1 Backend sends no length hint"
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 OK\n"
+	send "HTTP/1.0 200 OK\n"
 	send "\n"
 	send "12345"
 } -start

bin/varnishtest/tests/r01532.vtc

@@ -9,7 +9,7 @@ varnish v1 -vcl+backend {
 	import ${vmod_std};
 
 	sub vcl_deliver {
-		set resp.http.x-foo = std.real2time(1140618699.00);
+		set resp.http.x-foo = std.real2time(1140618699.00, now);
 	}
 } -start
 

bin/varnishtest/tests/r01729.vtc

 varnishtest "C-L/T-E:chunked conflict"
 
 server s1 {
-	rxreqhdrs
-	expect req.http.content-length == <undef>
-	expect req.http.transfer-encoding == chunked
-	rxreqbody
-	expect req.url == /1
+	rxreq
 	expect req.bodylen == 20
 
-	send "HTTP/1.1 200 OK"
-	send "Content-Length: 31\r\n"
-	send "Transfer-Encoding: chunked\r\n"
-	send "\r\n"
-	send "14\r\n"
-	send "0123456789"
-	send "0123456789"
-	send "0\r\n"
-	send "\r\n"
-
-	rxreqhdrs
-	expect req.http.content-length == 20
-	expect req.http.transfer-encoding == <undef>
-	expect req.url == /2
-	expect req.bodylen == 20
-
-	send "HTTP/1.1 200 OK"
+	send "HTTP/1.1 200 OK\r\n"
 	send "Content-Length: 31\r\n"
 	send "Transfer-Encoding: chunked\r\n"
 	send "\r\n"
@@ -36,14 +16,7 @@ server s1 {
 
 } -start
 
-varnish v1 -vcl+backend {
-	import ${vmod_std};
-	sub vcl_recv {
-		if (req.url == "/2") {
-			std.cache_req_body(1KB);
-		}
-	}
-} -start
+varnish v1 -vcl+backend { } -start
 
 client c1 {
 
@@ -56,12 +29,14 @@ client c1 {
 	send "0123456789"
 	send "0\r\n"
 	send "\r\n"
+
 	rxresp
-	expect resp.bodylen == 20
+	expect resp.status == 400
+} -run
 
+client c1 {
 
 	send "PUT /2 HTTP/1.1\r\n"
-	send "Content-Length: 31\r\n"
 	send "Transfer-Encoding: chunked\r\n"
 	send "\r\n"
 	send "14\r\n"
@@ -69,7 +44,8 @@ client c1 {
 	send "0123456789"
 	send "0\r\n"
 	send "\r\n"
+
 	rxresp
-	expect resp.bodylen == 20
+	expect resp.status == 503
 
 } -run

bin/varnishtest/tests/r01781.vtc

+varnishtest "#1781 gzip checksum with multilevel esi"
+
+server s1 {
+	rxreq
+	txresp -body {<esi:include src="/1"/>Baz}
+
+	rxreq
+	txresp -body {<esi:include src="/2"/>Bar}
+
+	rxreq
+	txresp -body {Foo}
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_gzip = true;
+		set beresp.do_esi = true;
+	}
+} -start
+
+client c1 {
+	txreq -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == "gzip"
+	gunzip
+	expect resp.body == "FooBarBaz"
+} -run

bin/varnishtest/tests/r01783.vtc

+varnishtest "POST with no body"
+
+server s1 {
+	rxreq
+	txresp -hdr "foo: 1"
+	rxreq
+	txresp -hdr "foo: 2"
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+client c1 {
+	txreq -req "POST" -nolen
+	rxresp
+	expect resp.http.foo == 1
+	txreq -req "POST" -nolen
+	rxresp
+	expect resp.http.foo == 2
+} -run

doc/sphinx/whats-new/changes.rst

@@ -105,6 +105,18 @@ TCP to fetch resources. More details in the :ref:`ref-writing-a-director`
 documentation.
 
 
+Backend connection timeout
+==========================
+
+Backend connections will now be closed by Varnish after `backend_idle_timeout`
+seconds of inactivity.
+
+Previously they were kept around forever and the backend servers would close
+the connection without Varnish noticing it. On the next traffic spike needing
+these extra backend connections, the request would fail, perhaps multiple
+times, before a working backend connection was found/created.
+
+
 Surrogate keys
 ==============
 
@@ -117,7 +129,7 @@ A new `req_top` identifier is available in VCL, which is a reference to
 `req` in the top-level ESI request.
 
 This is useful to pass data back and forth between the main ESI request
-and any ESI subrequests it lead to.
+and any ESI sub-requests it leads to.
 
 
 Other noteworthy small changes
@@ -125,7 +137,5 @@ Other noteworthy small changes
 
 * Varnish will now use the ``stale-while-revalidate`` defined in RFC5861
   to set object grace time.
-* Varnish will now discard remaining/older open backend connections when
-  a failing connection is found.
 * -smalloc storage is now recommended over -sfile on Linux systems.
 

doc/sphinx/whats-new/upgrade-4.0.rst

@@ -132,7 +132,7 @@ Remember to import the std module if you're not doing so already.
 client.port, and server.port replaced by respectively std.port(client.ip) and std.port(server.ip)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-`client.ip` and `server.ip` are now proper datatypes, which renders
+`client.ip` and `server.ip` are now proper data types, which renders
 as an IP address by default. You need to use the `std.port()`
 function to get the port number.
 

doc/sphinx/whats-new/upgrading.rst

@@ -63,12 +63,12 @@ replaced as the active VCL.
 
 The following parameters have been retired:
 
-* `group` (security rewamp)
-* `group_cc` (security rewamp)
-* `listen_address` (security rewamp)
+* `group` (security revamp)
+* `group_cc` (security revamp)
+* `listen_address` (security revamp)
 * `pool_vbc`
 * `timeout_req` - merged with `timeout_idle`.
-* `user` (security rewamp)
+* `user` (security revamp)
 
 Minor changes of default values on `workspace_session` and `vsl_mask`.
 

lib/libvmod_std/vmod.vcc

@@ -109,7 +109,7 @@ $Function VOID collect(HEADER hdr)
 Description
 	Collapses multiple *hdr* headers into one long header.
 
-	Care should be taken when collapsing headers.  In particular
+	Care should be taken when collapsing headers. In particular
 	collapsing Set-Cookie will lead to unexpected results on the
 	browser side.
 Example
@@ -128,7 +128,7 @@ Example
 $Function INT integer(STRING s, INT fallback)
 
 Description
-	Converts the string *s* to an integer.	If conversion fails,
+	Converts the string *s* to an integer. If conversion fails,
 	*fallback* will be returned.
 Example
 	| if (std.integer(req.http.foo, 0) > 5) {
@@ -139,7 +139,7 @@ $Function IP ip(STRING s, IP fallback)
 
 Description
 	Converts the string *s* to the first IP number returned by
-	the system library function getaddrinfo(3).  If conversion
+	the system library function getaddrinfo(3). If conversion
 	fails, *fallback* will be returned.
 Example
 	| if (std.ip(req.http.X-forwarded-for, "0.0.0.0") ~ my_acl) {
@@ -149,33 +149,44 @@ Example
 $Function REAL real(STRING s, REAL fallback)
 
 Description
-	Converts the string *s* to a real.  If conversion fails,
+	Converts the string *s* to a real. If conversion fails,
 	*fallback* will be returned.
 Example
 	| if (std.real(req.http.foo, 0.0) > 5.5) {
 	|	...
 	| }
 
-$Function TIME real2time(REAL r)
+$Function INT real2integer(REAL r, INT fallback)
 
 Description
-	Converts the real *r* to a time.
+	Converts the real *r* to an integer. If conversion fails,
+	*fallback* will be returned.
+Example
+	set req.http.integer = std.real2integer(1140618699.00, 0);
+
+$Function TIME real2time(REAL r, TIME fallback)
+
+Description
+	Converts the real *r* to a time. If conversion fails,
+	*fallback* will be returned.
 Example
-	set req.http.time = std.real2time(1140618699.00);
+	set req.http.time = std.real2time(1140618699.00, now);
 
-$Function INT time2integer(TIME t)
+$Function INT time2integer(TIME t, INT fallback)
 
 Description
-	Converts the time *t* to a integer.
+	Converts the time *t* to a integer. If conversion fails,
+	*fallback* will be returned.
 Example
-	set req.http.int = std.time2integer(now);
+	set req.http.int = std.time2integer(now, 0);
 
-$Function REAL time2real(TIME t)
+$Function REAL time2real(TIME t, REAL fallback)
 
 Description
-	Converts the time *t* to a real.
+	Converts the time *t* to a real. If conversion fails,
+	*fallback* will be returned.
 Example
-	set req.http.real = std.time2real(now);
+	set req.http.real = std.time2real(now, 1.0);
 
 $Function BOOL healthy(BACKEND be)
 
@@ -241,7 +252,7 @@ Example
 $Function TIME time(STRING s, TIME fallback)
 
 Description
-	Converts the string *s* to a time.  If conversion fails,
+	Converts the string *s* to a time. If conversion fails,
 	*fallback* will be returned.
 
 	Supported formats:

lib/libvmod_std/vmod_std_conversions.c

@@ -142,7 +142,7 @@ vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d)
 	p = WS_Alloc(ctx->ws, vsa_suckaddr_len);
 	if (p == NULL) {
 		VSLb(ctx->vsl, SLT_VCL_Error,
-		     "vmod std.ip(): insufficient workspace");
+		    "vmod std.ip(): insufficient workspace");
 		return d;
 	}
 	r = NULL;
@@ -187,27 +187,51 @@ vmod_real(VRT_CTX, VCL_STRING p, VCL_REAL d)
 	return (r);
 }
 
+VCL_INT __match_proto__(td_std_real2integer)
+vmod_real2integer(VRT_CTX, VCL_REAL r, VCL_INT i)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	if (!isfinite(r))
+		return (i);
+	r = round(r);
+	if (r > LONG_MAX || r < LONG_MIN)
+		return(i);
+	return ((long)r);
+}
+
 VCL_TIME __match_proto__(td_std_real2time)
-vmod_real2time(VRT_CTX, VCL_REAL r)
+vmod_real2time(VRT_CTX, VCL_REAL r, VCL_TIME t)
 {
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 
+	if (!isfinite(r))
+		return (t);
+
 	return (r);
 }
 
 VCL_INT __match_proto__(td_std_time2integer)
-vmod_time2integer(VRT_CTX, VCL_TIME t)
+vmod_time2integer(VRT_CTX, VCL_TIME t, VCL_INT i)
 {
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 
-	return ((long)floor(t));
+	if (!isfinite(t))
+		return (i);
+	t = round(t);
+	if (t > LONG_MAX || t < LONG_MIN)
+		return(i);
+	return ((long)t);
 }
 
 VCL_REAL __match_proto__(td_std_time2real)
-vmod_time2real(VRT_CTX, VCL_TIME t)
+vmod_time2real(VRT_CTX, VCL_TIME t, VCL_REAL r)
 {
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 
+	if (!isfinite(t))
+		return (r);
+
 	return (t);
 }