Flesh out the gzip and gunzip fetch processors.

Now you can get varnish to gzip or gunzip objects as they are
received from the backend by setting

	sub vcl_fetch {
		beresp.do_gzip = true;
		beresp.do_guzip = true;
	}

They will only do something if the object is/isn't gzip'ed already
(according to the beresp.http.Content-Encoding header) so the above
silly-ish example will compress al uncompressed objects and decompress
all compressed objects.

NB: Lots of errorchecks not implemented yet.



git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@5715 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/cache_center.c

@@ -585,6 +585,22 @@ cnt_fetch(struct sess *sp)
 		AZ(sp->objcore);
 	}
 
+	AZ(sp->wrk->vfp);
+	/* XXX: precedence, also: do_esi */
+
+	if (sp->wrk->do_gunzip &&
+	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
+		http_Unset(sp->wrk->beresp, H_Content_Encoding);
+		sp->wrk->vfp = &vfp_gunzip;
+	}
+
+	if (sp->wrk->do_gzip &&
+	    !http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
+		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->beresp,
+		    "Content-Encoding: %s", "gzip");
+		sp->wrk->vfp = &vfp_gzip;
+	}
+
 	l = http_EstimateWS(sp->wrk->beresp,
 	    sp->pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 
@@ -642,12 +658,6 @@ cnt_fetch(struct sess *sp)
 	else
 		sp->obj->last_modified = sp->wrk->entered;
 
-	AZ(sp->wrk->vfp);
-	/* XXX: precedence, also: do_esi */
-	if (sp->wrk->do_gunzip)
-		sp->wrk->vfp = &vfp_gunzip;
-	else if (sp->wrk->do_gzip)
-		sp->wrk->vfp = &vfp_gzip;
 	i = FetchBody(sp);
 	sp->wrk->vfp = NULL;
 	AZ(sp->wrk->wfd);

bin/varnishd/cache_fetch.c

@@ -60,7 +60,7 @@ static unsigned fetchfrag;
  * 'estimate' is the estimate of the number of bytes we expect to receive,
  * as seen on the socket, or zero if unknown.
  */
-static void
+static void __match_proto__()
 vfp_nop_begin(struct sess *sp, size_t estimate)
 {
 
@@ -84,7 +84,7 @@ vfp_nop_begin(struct sess *sp, size_t estimate)
  * Return 1 when 'bytes' have been processed.
  */
 
-static int
+static int __match_proto__()
 vfp_nop_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
 	ssize_t l, w;
@@ -130,7 +130,7 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
  * Return 0 for OK
  */
 
-static int
+static int __match_proto__()
 vfp_nop_end(struct sess *sp)
 {
 	struct storage *st;

bin/varnishd/cache_gzip.c

@@ -41,6 +41,7 @@
 SVNID("$Id$")
 
 #include "cache.h"
+#include "stevedore.h"
 
 #include "zlib.h"
 
@@ -52,7 +53,7 @@ struct vgz {
 	char			*tmp_snapshot;
 
 	struct ws		*buf;
-	unsigned		bufsiz;
+	size_t			bufsiz;
 
 	z_stream		vz;
 };
@@ -173,31 +174,86 @@ VGZ_Destroy(struct vgz **vg)
  * A VFP for gunzip'ing an object as we receive it from the backend
  */
 
-static void
+static void __match_proto__()
 vfp_gunzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
 	sp->wrk->vfp_private = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
 }
 
-static int
+static int __match_proto__()
 vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
-	struct vgz *vgz;
+	struct vgz *vg;
+	struct storage *st;
+	ssize_t l, w;
+	int i = -100;
 
-	CAST_OBJ_NOTNULL(vgz, sp->wrk->vfp_private, VGZ_MAGIC);
-	(void)htc;
-	(void)bytes;
+	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	AZ(vg->vz.avail_in);
+	while (bytes > 0 || vg->vz.avail_in > 0) {
+		if (sp->wrk->storage == NULL)
+			sp->wrk->storage = STV_alloc(sp,
+			    params->fetch_chunksize * 1024LL);
+		if (sp->wrk->storage == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		st = sp->wrk->storage;
+
+		vg->vz.next_out = st->ptr + st->len;
+		vg->vz.avail_out = st->space - st->len;
+
+		if (vg->vz.avail_in == 0 && bytes > 0) {
+			l = vg->bufsiz;
+			if (l > bytes)
+				l = bytes;
+			w = HTC_Read(htc, vg->buf->f, l);
+			if (w <= 0)
+				return (w);
+			vg->vz.next_in = (void*)vg->buf->f;
+			vg->vz.avail_in = w;
+			bytes -= w;
+		}
+
+		i = inflate(&vg->vz, 0);
+		assert(i == Z_OK || i == Z_STREAM_END);
+		st->len = st->space - vg->vz.avail_out;
+		if (st->len == st->space) {
+			VTAILQ_INSERT_TAIL(&sp->obj->store,
+			    sp->wrk->storage, list);
+			sp->obj->len += st->len;
+			sp->wrk->storage = NULL;
+		}
+	}
+	if (i == Z_STREAM_END)
+		return (1);
 	return (-1);
 }
 
-static int
+static int __match_proto__()
 vfp_gunzip_end(struct sess *sp)
 {
-	struct vgz *vgz;
+	struct vgz *vg;
+	struct storage *st;
 
-	CAST_OBJ_NOTNULL(vgz, sp->wrk->vfp_private, VGZ_MAGIC);
-	return (-1);
+	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	VGZ_Destroy(&vg);
+
+	st = sp->wrk->storage;
+	sp->wrk->storage = NULL;
+	if (st == NULL)
+		return (0);
+
+	if (st->len == 0) {
+		STV_free(st);
+		return (0);
+	}
+	if (st->len < st->space)
+		STV_trim(st, st->len);
+	sp->obj->len += st->len;
+	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+	return (0);
 }
 
 struct vfp vfp_gunzip = {
@@ -213,27 +269,98 @@ struct vfp vfp_gunzip = {
  * A VFP for gzip'ing an object as we receive it from the backend
  */
 
-static void
+static void __match_proto__()
 vfp_gzip_begin(struct sess *sp, size_t estimate)
 {
-	(void)sp;
+	struct vgz *vg;
 	(void)estimate;
+
+	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
+	/* XXX: hack */
+	memset(&vg->vz, 0, sizeof vg->vz);
+	assert(Z_OK == deflateInit2(&vg->vz,
+	    0,
+	    Z_DEFLATED,
+	    31,
+	    9,
+	    Z_DEFAULT_STRATEGY));
+
+	sp->wrk->vfp_private = vg;
 }
 
-static int
+static int __match_proto__()
 vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
-	(void)sp;
-	(void)htc;
-	(void)bytes;
+	struct vgz *vg;
+	struct storage *st;
+	ssize_t l, w;
+	int i = -100;
+
+	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	AZ(vg->vz.avail_in);
+	while (bytes > 0 || vg->vz.avail_in > 0) {
+		if (sp->wrk->storage == NULL)
+			sp->wrk->storage = STV_alloc(sp,
+			    params->fetch_chunksize * 1024LL);
+		if (sp->wrk->storage == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		st = sp->wrk->storage;
+
+		vg->vz.next_out = st->ptr + st->len;
+		vg->vz.avail_out = st->space - st->len;
+
+		if (vg->vz.avail_in == 0 && bytes > 0) {
+			l = vg->bufsiz;
+			if (l > bytes)
+				l = bytes;
+			w = HTC_Read(htc, vg->buf->f, l);
+			if (w <= 0)
+				return (w);
+			vg->vz.next_in = (void*)vg->buf->f;
+			vg->vz.avail_in = w;
+			bytes -= w;
+		}
+
+		i = deflate(&vg->vz, bytes == 0 ? Z_FINISH : 0);
+		assert(i == Z_OK || i == Z_STREAM_END);
+		st->len = st->space - vg->vz.avail_out;
+		if (st->len == st->space) {
+			VTAILQ_INSERT_TAIL(&sp->obj->store,
+			    sp->wrk->storage, list);
+			sp->obj->len += st->len;
+			sp->wrk->storage = NULL;
+		}
+	}
+	if (i == Z_STREAM_END)
+		return (1);
 	return (-1);
 }
 
-static int
+static int __match_proto__()
 vfp_gzip_end(struct sess *sp)
 {
-	(void)sp;
-	return (-1);
+	struct vgz *vg;
+	struct storage *st;
+
+	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	VGZ_Destroy(&vg);
+
+	st = sp->wrk->storage;
+	sp->wrk->storage = NULL;
+	if (st == NULL)
+		return (0);
+
+	if (st->len == 0) {
+		STV_free(st);
+		return (0);
+	}
+	if (st->len < st->space)
+		STV_trim(st, st->len);
+	sp->obj->len += st->len;
+	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+	return (0);
 }
 
 struct vfp vfp_gzip = {
@@ -241,4 +368,3 @@ struct vfp vfp_gzip = {
         .bytes  =       vfp_gzip_bytes,
         .end    =       vfp_gzip_end,
 };
-

bin/varnishd/cache_response.c

@@ -236,6 +236,7 @@ RES_BuildHttp(struct sess *sp)
 /*--------------------------------------------------------------------
  * We have a gzip'ed object and need to ungzip it for a client which
  * does not understand gzip.
+ * XXX: handle invalid gzip data better (how ?)
  */
 
 static void

bin/varnishtest/tests/g00003.vtc

+# $Id$
+
+test "test gunzip on fetch"
+
+server s1 {
+	rxreq
+	expect req.url == "/foo"
+	expect req.http.accept-encoding == "gzip"
+	txresp -gziplen 41
+
+	rxreq
+	expect req.url == "/bar"
+	expect req.http.accept-encoding == "gzip"
+	txresp -bodylen 42
+
+	rxreq
+	expect req.url == "/foobar"
+	expect req.http.accept-encoding == "gzip"
+	txresp -bodylen 43
+} -start
+
+varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend {
+
+	sub vcl_fetch {
+		set beresp.do_gunzip = true;
+		if (req.url == "/foobar") {
+			set beresp.do_gzip = true;
+		}
+	}
+} -start
+
+client c1 {
+	txreq -url /foo -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.bodylen == 41
+
+	txreq -url /bar -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.bodylen == 42
+
+	txreq -url /foobar -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == "gzip"
+	gunzip
+	expect resp.bodylen == 43
+
+	txreq -url /foobar 
+	rxresp
+	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.bodylen == 43
+} -run
+