Nils patch for retrying partial fetches, with minor changes by me.

Fixes #1638
Fixes #1641

Ported to 4.0 by Martin

Conflicts:
	bin/varnishd/cache/cache_fetch.c

bin/varnishd/cache/cache_fetch.c

@@ -223,6 +223,7 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 
 /*--------------------------------------------------------------------
  * Start a new VSL transaction and try again
+ * Prepare the busyobj and fetch processors
  */
 
 static enum fetch_step
@@ -233,8 +234,27 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
+	assert(bo->state == BOS_REQ_DONE);
+
 	VSLb_ts_busyobj(bo, "Retry", W_TIM_real(wrk));
 
+	/* VDI_CloseFd must have been called before */
+	AZ(bo->vbc);
+	bo->should_close = 0;
+
+	/* reset other bo attributes - See VBO_GetBusyObj */
+	bo->storage_hint = NULL;
+	bo->do_esi = 0;
+	bo->do_gzip = 0;
+	bo->do_gunzip = 0;
+	bo->do_stream = 1;
+	bo->uncacheable = 0;
+	bo->abandon = 0;
+
+	/* reset fetch processors */
+	bo->failed = 0;
+	bo->vfp_nxt = 0;
+
 	// XXX: BereqEnd + BereqAcct ?
 	wid = VXID_Get(&wrk->vxid_pool);
 	VSLb(bo->vsl, SLT_Link, "bereq %u retry", wid);
@@ -380,6 +400,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 		bo->retries++;
 		if (bo->retries <= cache_param->max_retries)
 			return (F_STP_RETRY);
+
 		VSLb(bo->vsl, SLT_VCL_Error,
 		    "Too many retries, delivering 503");
 		return (F_STP_ERROR);
@@ -538,6 +559,11 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		VFP_Fetch_Body(bo, est);
 	}
 
+	if (bo->failed) {
+		AN(bo->vbc);
+		VDI_CloseFd(&bo->vbc, &bo->acct);
+	}
+
 	if (bo->failed && !bo->do_stream) {
 		assert(bo->state < BOS_STREAM);
 		if (bo->fetch_obj != NULL) {
@@ -690,6 +716,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	now = W_TIM_real(wrk);
 	VSLb_ts_busyobj(bo, "Error", now);
 
+	AZ(bo->vbc);		/* VDI_CloseFd must have been called before */
 	AN(bo->fetch_objcore->flags & OC_F_BUSY);
 
 	AZ(bo->synth_body);
@@ -716,8 +743,10 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	if (wrk->handling == VCL_RET_RETRY) {
 		VSB_delete(bo->synth_body);
 		bo->synth_body = NULL;
+
 		if (bo->retries++ < cache_param->max_retries)
 			return (F_STP_RETRY);
+
 		return (F_STP_FAIL);
 	}
 

bin/varnishtest/tests/r01638.vtc

+varnishtest "Test retry of straight insufficient bytes pass-fetch : do_stream = false"
+
+server s1 {
+	rxreq
+	txresp -hdr "Content-Length: 10000" -nolen -bodylen 5000
+}
+
+server s2 {
+	rxreq
+	txresp -bodylen 5
+}
+
+server s1 -start
+server s2 -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		return (pass);
+	}
+	sub vcl_backend_fetch {
+		set bereq.between_bytes_timeout = 10s;
+		if (bereq.retries == 0) {
+			set bereq.backend = s1;
+		} else {
+			set bereq.backend = s2;
+		}
+	}
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
+	sub vcl_backend_error {
+		if (bereq.retries == 0) {
+			return (retry);
+		} else {
+			return (deliver);
+		}
+	}
+} -start
+
+client c1 {
+	timeout 10
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 5
+} -run
+
+server s1 -wait
+server s2 -wait

bin/varnishtest/tests/r01641.vtc

+varnishtest "Test retry of straight insufficient bytes pass-fetch : do_esi = true"
+
+server s1 {
+	rxreq
+	txresp -hdr "Content-Length: 10000" -nolen -bodylen 5000
+}
+
+server s2 {
+	rxreq
+	txresp -bodylen 5
+}
+
+server s1 -start
+server s2 -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		return (pass);
+	}
+	sub vcl_backend_fetch {
+		set bereq.between_bytes_timeout = 10s;
+		if (bereq.retries == 0) {
+			set bereq.backend = s1;
+		} else {
+			set bereq.backend = s2;
+		}
+	}
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+	sub vcl_backend_error {
+		if (bereq.retries == 0) {
+			return (retry);
+		} else {
+			return (deliver);
+		}
+	}
+} -start
+
+client c1 {
+	timeout 10
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 5
+} -run
+
+server s1 -wait
+server s2 -wait