Add a "hole-through" path that does the trivial bit of streaming.

Now for all the hard bits...

bin/varnishd/cache.h

@@ -312,6 +312,9 @@ struct worker {
 	unsigned		do_close;
 	char			*h_content_length;
 
+	/* Stream state */
+	ssize_t			stream_next;
+
 	/* ESI stuff */
 	struct vep_state	*vep;
 	int			gzip_resp;
@@ -858,6 +861,7 @@ void RES_BuildHttp(struct sess *sp);
 void RES_WriteObj(struct sess *sp);
 void RES_StreamStart(struct sess *sp);
 void RES_StreamEnd(struct sess *sp);
+void RES_StreamPoll(const struct sess *sp);
 
 /* cache_vary.c */
 struct vsb *VRY_Create(const struct sess *sp, const struct http *hp);

bin/varnishd/cache_fetch.c

@@ -102,6 +102,8 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		st->len += w;
 		sp->obj->len += w;
 		bytes -= w;
+		if (sp->wrk->do_stream)
+			RES_StreamPoll(sp);
 	}
 	return (1);
 }

bin/varnishd/cache_gzip.c

@@ -477,6 +477,8 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		assert(i == VGZ_OK || i == VGZ_END);
 		sp->obj->len += dl;
+		if (sp->wrk->do_stream)
+			RES_StreamPoll(sp);
 	}
 	if (i == Z_OK || i == Z_STREAM_END)
 		return (1);
@@ -547,6 +549,8 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
 		sp->obj->len += dl;
+		if (sp->wrk->do_stream)
+			RES_StreamPoll(sp);
 	}
 	return (1);
 }
@@ -623,6 +627,8 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		VGZ_Ibuf(vg, st->ptr + st->len, w);
 		st->len += w;
 		sp->obj->len += w;
+		if (sp->wrk->do_stream)
+			RES_StreamPoll(sp);
 
 		while (!VGZ_IbufEmpty(vg)) {
 			VGZ_Obuf(vg, ibuf, sizeof ibuf);

bin/varnishd/cache_response.c

@@ -270,7 +270,7 @@ res_WriteGunzipObj(struct sess *sp)
 	}
 	if (obufl) {
 		(void)WRW_Write(sp->wrk, obuf, obufl);
-		WRW_Flush(sp->wrk);
+		(void)WRW_Flush(sp->wrk);
 	}
 	VGZ_Destroy(&vg);
 	assert(u == sp->obj->len);
@@ -413,19 +413,42 @@ RES_StreamStart(struct sess *sp)
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
 		WRW_Chunked(sp->wrk);
+
+	sp->wrk->stream_next = 0;
+}
+
+void
+RES_StreamPoll(const struct sess *sp)
+{
+	struct storage *st;
+	ssize_t l, l2;
+	void *ptr;
+
+	if (sp->obj->len == sp->wrk->stream_next)
+		return;
+	assert(sp->obj->len > sp->wrk->stream_next);
+	l = 0;
+	VTAILQ_FOREACH(st, &sp->obj->store, list) {
+		if (st->len + l <= sp->wrk->stream_next) {
+			l += st->len;
+			continue;
+		}
+		l2 = st->len + l - sp->wrk->stream_next;
+		ptr = st->ptr + (sp->wrk->stream_next - l);
+		(void)WRW_Write(sp->wrk, ptr, l2);
+		l += st->len;
+		sp->wrk->stream_next += l2;
+	}
+	(void)WRW_Flush(sp->wrk);
 }
 
 void
 RES_StreamEnd(struct sess *sp)
 {
-	ssize_t low, high;
 
 	if (sp->wrk->res_mode & RES_GUNZIP) {
+		INCOMPL();
 		res_WriteGunzipObj(sp);
-	} else {
-		low = 0;
-		high = sp->obj->len - 1;
-		res_WriteDirObj(sp, low, high);
 	}
 	if (sp->wrk->res_mode & RES_CHUNKED &&
 	    !(sp->wrk->res_mode & RES_ESI_CHILD))

bin/varnishtest/tests/README

@@ -15,10 +15,14 @@ Naming scheme
 
 		[id]%05d.vtc
 
-	id ~ [a] --> varnishtester(1) tests
+	id ~ [a] --> varnishtest(1) tests
 	id ~ [b] --> Basic functionality tests
 	id ~ [c] --> Complex functionality tests
 	id ~ [e] --> ESI tests
+	id ~ [g] --> GZIP tests
+	id ~ [m] --> VMOD tests
+	id ~ [p] --> Persistent tests
 	id ~ [r] --> Regression tests, same number as ticket
 	id ~ [s] --> Slow tests, expiry, grace etc.
+	id ~ [t] --> sTreaming tests
 	id ~ [v] --> VCL tests: execute VRT functions

bin/varnishtest/tests/t00000.vtc

+# $Id$
+
+test "Ticket #873"
+
+server s1 {
+	rxreq 
+	txresp -nolen -hdr "Transfer-encoding: chunked"
+	chunked "<1>------------------------<1>\n"
+	sema r1 sync 2
+	chunked "<2>------------------------<2>\n"
+	sema r2 sync 2
+	chunked "<3>------------------------<3>\n"
+	sema r1 sync 2
+	chunked "<4>------------------------<4>\n"
+	sema r2 sync 2
+	chunkedlen 0
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_stream = true;
+	}
+} -start 
+
+varnish v1 -cliok "param.set diag_bitmap 1"
+
+client c1 {
+	txreq  -hdr "foo: /foo"
+	rxresp -no_obj
+
+	rxchunk
+	expect resp.chunklen == 31
+	sema r1 sync 2
+
+	rxchunk
+	expect resp.chunklen == 31
+	sema r2 sync 2
+
+	rxchunk
+	expect resp.chunklen == 31
+	sema r1 sync 2
+
+	rxchunk
+	expect resp.chunklen == 31
+	sema r2 sync 2
+
+	rxchunk
+	expect resp.chunklen == 0
+	expect resp.bodylen == 124
+} -run