Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache

bin/varnishd/cache.h

@@ -312,6 +312,9 @@ struct worker {
 	unsigned		do_close;
 	char			*h_content_length;
 
+	/* Stream state */
+	ssize_t			stream_next;
+
 	/* ESI stuff */
 	struct vep_state	*vep;
 	int			gzip_resp;
@@ -858,6 +861,7 @@ void RES_BuildHttp(struct sess *sp);
 void RES_WriteObj(struct sess *sp);
 void RES_StreamStart(struct sess *sp);
 void RES_StreamEnd(struct sess *sp);
+void RES_StreamPoll(const struct sess *sp);
 
 /* cache_vary.c */
 struct vsb *VRY_Create(const struct sess *sp, const struct http *hp);

bin/varnishd/cache_fetch.c

@@ -102,6 +102,8 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		st->len += w;
 		sp->obj->len += w;
 		bytes -= w;
+		if (sp->wrk->do_stream)
+			RES_StreamPoll(sp);
 	}
 	return (1);
 }

bin/varnishd/cache_gzip.c

@@ -477,6 +477,8 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		assert(i == VGZ_OK || i == VGZ_END);
 		sp->obj->len += dl;
+		if (sp->wrk->do_stream)
+			RES_StreamPoll(sp);
 	}
 	if (i == Z_OK || i == Z_STREAM_END)
 		return (1);
@@ -547,6 +549,8 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
 		sp->obj->len += dl;
+		if (sp->wrk->do_stream)
+			RES_StreamPoll(sp);
 	}
 	return (1);
 }
@@ -623,6 +627,8 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		VGZ_Ibuf(vg, st->ptr + st->len, w);
 		st->len += w;
 		sp->obj->len += w;
+		if (sp->wrk->do_stream)
+			RES_StreamPoll(sp);
 
 		while (!VGZ_IbufEmpty(vg)) {
 			VGZ_Obuf(vg, ibuf, sizeof ibuf);
@@ -634,9 +640,9 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			}
 		}
 	}
-	if (i == Z_STREAM_END)
+	if (i == VGZ_OK || i == VGZ_END)
 		return (1);
-	WSP(sp, SLT_FetchError, "Incomplete Gzip data (not STREAM_END)");
+	WSP(sp, SLT_FetchError, "Gunzip trouble (%d)", i);
 	return (-1);
 }
 

bin/varnishd/cache_response.c

@@ -270,7 +270,7 @@ res_WriteGunzipObj(struct sess *sp)
 	}
 	if (obufl) {
 		(void)WRW_Write(sp->wrk, obuf, obufl);
-		WRW_Flush(sp->wrk);
+		(void)WRW_Flush(sp->wrk);
 	}
 	VGZ_Destroy(&vg);
 	assert(u == sp->obj->len);
@@ -413,19 +413,42 @@ RES_StreamStart(struct sess *sp)
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
 		WRW_Chunked(sp->wrk);
+
+	sp->wrk->stream_next = 0;
+}
+
+void
+RES_StreamPoll(const struct sess *sp)
+{
+	struct storage *st;
+	ssize_t l, l2;
+	void *ptr;
+
+	if (sp->obj->len == sp->wrk->stream_next)
+		return;
+	assert(sp->obj->len > sp->wrk->stream_next);
+	l = 0;
+	VTAILQ_FOREACH(st, &sp->obj->store, list) {
+		if (st->len + l <= sp->wrk->stream_next) {
+			l += st->len;
+			continue;
+		}
+		l2 = st->len + l - sp->wrk->stream_next;
+		ptr = st->ptr + (sp->wrk->stream_next - l);
+		(void)WRW_Write(sp->wrk, ptr, l2);
+		l += st->len;
+		sp->wrk->stream_next += l2;
+	}
+	(void)WRW_Flush(sp->wrk);
 }
 
 void
 RES_StreamEnd(struct sess *sp)
 {
-	ssize_t low, high;
 
 	if (sp->wrk->res_mode & RES_GUNZIP) {
+		INCOMPL();
 		res_WriteGunzipObj(sp);
-	} else {
-		low = 0;
-		high = sp->obj->len - 1;
-		res_WriteDirObj(sp, low, high);
 	}
 	if (sp->wrk->res_mode & RES_CHUNKED &&
 	    !(sp->wrk->res_mode & RES_ESI_CHILD))

bin/varnishd/rfc2616.c

@@ -170,7 +170,7 @@ RFC2616_Ttl(const struct sess *sp)
 	} 
 
 	/* calculated TTL, Our time, Date, Expires, max-age, age */
-	WSP(sp, SLT_TTL, "%u RFC %g %g %g %g %u %u", sp->xid,
+	WSP(sp, SLT_TTL, "%u RFC %g %.0f %.0f %.0f %u %u", sp->xid,
 	    ttl, sp->wrk->entered, h_date, h_expires, max_age, age);
 
 	return (ttl);

bin/varnishncsa/varnishncsa.c

@@ -219,6 +219,7 @@ clean_logline(struct logline *lp)
 	freez(lp->df_m);
 	freez(lp->df_s);
 	freez(lp->df_u);
+	freez(lp->df_ttfb);
 #undef freez
 	memset(lp, 0, sizeof *lp);
 }
@@ -415,17 +416,23 @@ collect_client(struct logline *lp, enum vsl_tag tag, unsigned spec,
 	case SLT_RxHeader:
 		if (!lp->active)
 			break;
-		if (isprefix(ptr, "user-agent:", end, &next))
+		if (isprefix(ptr, "user-agent:", end, &next)) {
+			free(lp->df_User_agent);
 			lp->df_User_agent = trimline(next, end);
-		else if (isprefix(ptr, "referer:", end, &next))
+		} else if (isprefix(ptr, "referer:", end, &next)) {
+			free(lp->df_Referer);
 			lp->df_Referer = trimline(next, end);
-		else if (isprefix(ptr, "authorization:", end, &next) &&
-		    isprefix(next, "basic", end, &next))
+		} else if (isprefix(ptr, "authorization:", end, &next) &&
+			   isprefix(next, "basic", end, &next)) {
+			free(lp->df_u);
 			lp->df_u = trimline(next, end);
-		else if (isprefix(ptr, "x-forwarded-for:", end, &next))
+		} else if (isprefix(ptr, "x-forwarded-for:", end, &next)) {
+			free(lp->df_X_Forwarded_For);
 			lp->df_X_Forwarded_For = trimline(next, end);
-		else if (isprefix(ptr, "host:", end, &next))
+		} else if (isprefix(ptr, "host:", end, &next)) {
+			free(lp->df_Host);
 			lp->df_Host = trimline(next, end);
+		}
 		break;
 
 	case SLT_VCL_call:
@@ -471,7 +478,7 @@ collect_client(struct logline *lp, enum vsl_tag tag, unsigned spec,
 		char ttfb[64];
 		if (!lp->active)
 			break;
-		if (sscanf(ptr, "%*u %*u.%*u %ld.%*u %*u.%*u %s", &l, ttfb) != 2) {
+		if (lp->df_ttfb != NULL || sscanf(ptr, "%*u %*u.%*u %ld.%*u %*u.%*u %s", &l, ttfb) != 2) {
 			clean_logline(lp);
 			break;
 		}

bin/varnishtest/tests/README

@@ -15,10 +15,14 @@ Naming scheme
 
 		[id]%05d.vtc
 
-	id ~ [a] --> varnishtester(1) tests
+	id ~ [a] --> varnishtest(1) tests
 	id ~ [b] --> Basic functionality tests
 	id ~ [c] --> Complex functionality tests
 	id ~ [e] --> ESI tests
+	id ~ [g] --> GZIP tests
+	id ~ [m] --> VMOD tests
+	id ~ [p] --> Persistent tests
 	id ~ [r] --> Regression tests, same number as ticket
 	id ~ [s] --> Slow tests, expiry, grace etc.
+	id ~ [t] --> sTreaming tests
 	id ~ [v] --> VCL tests: execute VRT functions

bin/varnishtest/tests/t00000.vtc

+# $Id$
+
+test "Ticket #873"
+
+server s1 {
+	rxreq 
+	txresp -nolen -hdr "Transfer-encoding: chunked"
+	chunked "<1>------------------------<1>\n"
+	sema r1 sync 2
+	chunked "<2>------------------------<2>\n"
+	sema r2 sync 2
+	chunked "<3>------------------------<3>\n"
+	sema r1 sync 2
+	chunked "<4>------------------------<4>\n"
+	sema r2 sync 2
+	chunkedlen 0
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_stream = true;
+	}
+} -start 
+
+varnish v1 -cliok "param.set diag_bitmap 1"
+
+client c1 {
+	txreq  -hdr "foo: /foo"
+	rxresp -no_obj
+
+	rxchunk
+	expect resp.chunklen == 31
+	sema r1 sync 2
+
+	rxchunk
+	expect resp.chunklen == 31
+	sema r2 sync 2
+
+	rxchunk
+	expect resp.chunklen == 31
+	sema r1 sync 2
+
+	rxchunk
+	expect resp.chunklen == 31
+	sema r2 sync 2
+
+	rxchunk
+	expect resp.chunklen == 0
+	expect resp.bodylen == 124
+} -run

bin/varnishtest/vtc_http.c

@@ -69,6 +69,7 @@ struct http {
 	char			*body;
 	unsigned		bodyl;
 	char			bodylen[20];
+	char			chunklen[20];
 
 	char			*req[MAX_HDR];
 	char			*resp[MAX_HDR];
@@ -191,6 +192,8 @@ cmd_var_resolve(struct http *hp, char *spec)
 		return(hp->resp[1]);
 	if (!strcmp(spec, "resp.msg"))
 		return(hp->resp[2]);
+	if (!strcmp(spec, "resp.chunklen"))
+		return(hp->chunklen);
 	if (!strcmp(spec, "resp.bodylen"))
 		return(hp->bodylen);
 	if (!memcmp(spec, "req.http.", 9)) {
@@ -371,6 +374,47 @@ http_rxchar(struct http *hp, int n)
 	assert(i > 0);
 }
 
+static int
+http_rxchunk(struct http *hp)
+{
+	char *q;
+	int l, i;
+
+	l = hp->prxbuf;
+	do
+		http_rxchar(hp, 1);
+	while (hp->rxbuf[hp->prxbuf - 1] != '\n');
+	vtc_dump(hp->vl, 4, "len", hp->rxbuf + l, -1);
+	i = strtoul(hp->rxbuf + l, &q, 16);
+	bprintf(hp->chunklen, "%d", i);
+	if ((q == hp->rxbuf + l) ||
+		(*q != '\0' && !vct_islws(*q))) {
+		vtc_log(hp->vl, 0, "chunked fail %02x @ %d",
+		    *q, q - (hp->rxbuf + l));
+	}
+	assert(q != hp->rxbuf + l);
+	assert(*q == '\0' || vct_islws(*q));
+	hp->prxbuf = l;
+	if (i > 0) {
+		http_rxchar(hp, i);
+		vtc_dump(hp->vl, 4, "chunk",
+		    hp->rxbuf + l, i);
+	}
+	l = hp->prxbuf;
+	http_rxchar(hp, 2);
+	if(!vct_iscrlf(hp->rxbuf[l]))
+		vtc_log(hp->vl, 0,
+		    "Wrong chunk tail[0] = %02x",
+		    hp->rxbuf[l] & 0xff);
+	if(!vct_iscrlf(hp->rxbuf[l + 1]))
+		vtc_log(hp->vl, 0,
+		    "Wrong chunk tail[1] = %02x",
+		    hp->rxbuf[l + 1] & 0xff);
+	hp->prxbuf = l;
+	hp->rxbuf[l] = '\0';
+	return (i);
+}
+
 /**********************************************************************
  * Swallow a HTTP message body
  */
@@ -378,15 +422,13 @@ http_rxchar(struct http *hp, int n)
 static void
 http_swallow_body(struct http *hp, char * const *hh, int body)
 {
-	char *p, *q;
+	char *p;
 	int i, l, ll;
 
-
 	ll = 0;
 	p = http_find_header(hh, "content-length");
 	if (p != NULL) {
 		l = strtoul(p, NULL, 0);
-		hp->body = hp->rxbuf + hp->prxbuf;
 		http_rxchar(hp, l);
 		vtc_dump(hp->vl, 4, "body", hp->body, l);
 		hp->bodyl = l;
@@ -395,44 +437,10 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 	}
 	p = http_find_header(hh, "transfer-encoding");
 	if (p != NULL && !strcmp(p, "chunked")) {
-		hp->body = hp->rxbuf + hp->prxbuf;
-		while (1) {
-			l = hp->prxbuf;
-			do
-				http_rxchar(hp, 1);
-			while (hp->rxbuf[hp->prxbuf - 1] != '\n');
-			vtc_dump(hp->vl, 4, "len", hp->rxbuf + l, -1);
-			i = strtoul(hp->rxbuf + l, &q, 16);
-			if ((q == hp->rxbuf + l) ||
-				(*q != '\0' && !vct_islws(*q))) {
-				vtc_log(hp->vl, 0, "chunked fail %02x @ %d",
-				    *q, q - (hp->rxbuf + l));
-			}
-			assert(q != hp->rxbuf + l);
-			assert(*q == '\0' || vct_islws(*q));
-			hp->prxbuf = l;
-			if (i > 0) {
-				ll += i;
-				http_rxchar(hp, i);
-				vtc_dump(hp->vl, 4, "chunk",
-				    hp->rxbuf + l, i);
-			}
-			l = hp->prxbuf;
-			http_rxchar(hp, 2);
-			if(!vct_iscrlf(hp->rxbuf[l]))
-				vtc_log(hp->vl, 0,
-				    "Wrong chunk tail[0] = %02x",
-				    hp->rxbuf[l] & 0xff);
-			if(!vct_iscrlf(hp->rxbuf[l + 1]))
-				vtc_log(hp->vl, 0,
-				    "Wrong chunk tail[1] = %02x",
-				    hp->rxbuf[l + 1] & 0xff);
-			hp->prxbuf = l;
-			hp->rxbuf[l] = '\0';
-			if (i == 0)
-				break;
-		}
+		while (http_rxchunk(hp) != 0)
+			continue;
 		vtc_dump(hp->vl, 4, "body", hp->body, ll);
+		ll = hp->rxbuf + hp->prxbuf - hp->body;
 		hp->bodyl = ll;
 		sprintf(hp->bodylen, "%d", ll);
 		return;
@@ -505,8 +513,9 @@ cmd_http_rxresp(CMD_ARGS)
 			    "Unknown http rxresp spec: %s\n", *av);
 	http_rxhdr(hp);
 	http_splitheader(hp, 0);
+	hp->body = hp->rxbuf + hp->prxbuf;
 	if (!has_obj)
-		;
+		return;
 	else if (!strcmp(hp->resp[1], "200"))
 		http_swallow_body(hp, hp->resp, 1);
 	else
@@ -787,6 +796,26 @@ cmd_http_rxbody(CMD_ARGS)
 	vtc_log(hp->vl, 4, "bodylen = %s", hp->bodylen);
 }
 
+static void
+cmd_http_rxchunk(CMD_ARGS)
+{
+	struct http *hp;
+	int ll, i;
+
+	(void)cmd;
+	(void)vl;
+	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
+	ONLY_CLIENT(hp, av);
+
+	i = http_rxchunk(hp);
+	if (i == 0) {
+		ll = hp->rxbuf + hp->prxbuf - hp->body;
+		hp->bodyl = ll;
+		sprintf(hp->bodylen, "%d", ll);
+		vtc_log(hp->vl, 4, "bodylen = %s", hp->bodylen);
+	}
+}
+
 /**********************************************************************
  * Transmit a request
  */
@@ -1043,6 +1072,7 @@ static const struct cmds http_cmds[] = {
 
 	{ "rxreq",		cmd_http_rxreq },
 	{ "rxhdrs",		cmd_http_rxhdrs },
+	{ "rxchunk",		cmd_http_rxchunk },
 	{ "rxbody",		cmd_http_rxbody },
 
 	{ "txresp",		cmd_http_txresp },