Yah! Enable gzip-file stitching and pass the first trivial

ESI+GZIP testcase

bin/varnishd/cache.h

@@ -98,6 +98,7 @@ struct ban;
 struct SHA256Context;
 struct vsc_lck;
 struct waitinglist;
+struct vef_priv;
 
 struct lock { void *priv; };		// Opaque
 
@@ -280,7 +281,7 @@ struct worker {
 	struct storage		*storage;
 	struct vfp		*vfp;
 	struct vgz		*vgz_rx;
-	struct vgz		*vgz_esi;
+	struct vef_priv		*vef_priv;
 	unsigned		do_esi;
 	unsigned		do_gzip;
 	unsigned		is_gzip;

bin/varnishd/cache_esi.h

@@ -27,6 +27,7 @@
  *
  */
 
+#define	VEC_GZ	(0x21)
 #define	VEC_V1	(0x40 + 1)
 #define	VEC_V2	(0x40 + 2)
 #define	VEC_V8	(0x40 + 8)

bin/varnishd/cache_esi_deliver.c

@@ -174,8 +174,10 @@ ESI_Deliver(struct sess *sp)
 	struct storage *st;
 	uint8_t *p, *e, *q, *r;
 	unsigned off;
-	ssize_t l, l_crc;
-	uint32_t crc, crc_ref;
+	ssize_t l, l_icrc, l_crc;
+	uint32_t crc, icrc;
+	uint8_t tailbuf[8 + 5];
+	int dogzip;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	st = sp->obj->esidata;
@@ -183,6 +185,14 @@ ESI_Deliver(struct sess *sp)
 	p = st->ptr;
 	e = st->ptr + st->len;
 
+	if (*p == VEC_GZ) {
+		p++;	
+		dogzip = 1;
+		crc = crc32(0L, Z_NULL, 0);
+		l_crc = 0;
+	} else 
+		dogzip = 0;
+
 	st = VTAILQ_FIRST(&sp->obj->store);
 	off = 0;
 
@@ -192,15 +202,19 @@ ESI_Deliver(struct sess *sp)
 		case VEC_V2:
 		case VEC_V8:
 			l = ved_decode_len(&p);
-			assert(*p == VEC_C1 || *p == VEC_C2 || *p == VEC_C8);
-			l_crc = ved_decode_len(&p);
-			crc = vbe32dec(p);
-			p += 4;
+			if (dogzip) {
+				assert(*p == VEC_C1 || *p == VEC_C2 ||
+				    *p == VEC_C8);
+				l_icrc = ved_decode_len(&p);
+				icrc = vbe32dec(p);
+				p += 4;
+			}
 			q = (void*)strchr((const char*)p, '\0');
 			assert (q > p);
-			crc_ref = crc32(0L, Z_NULL, 0);
-			crc_ref = crc32(crc_ref, st->ptr + off, l);
-			xxxassert(crc_ref == crc);
+			if (dogzip) {
+				crc = crc32_combine(crc, icrc, l_icrc);
+				l_crc += l_icrc;
+			}
 			ved_sendchunk(sp, p, q - p, st->ptr + off, l);
 			off += l;
 			p = q + 1;
@@ -229,6 +243,22 @@ ESI_Deliver(struct sess *sp)
 			INCOMPL();
 		}
 	}
+	if (dogzip) {
+		/* Emit a gzip literal block with finish bit set */
+		tailbuf[0] = 0x01;
+		tailbuf[1] = 0x00;
+		tailbuf[2] = 0x00;
+		tailbuf[3] = 0xff;
+		tailbuf[4] = 0xff;
+
+		/* Emit CRC32 */
+		vle32enc(tailbuf + 5, crc);
+
+		/* MOD(2^32) length */
+		vle32enc(tailbuf + 9, l_crc);
+
+		ved_sendchunk(sp, "d\r\n", 3, tailbuf, 13);
+	}
 	(void)WRW_Flush(sp->wrk);
 }
 

bin/varnishd/cache_esi_fetch.c

@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * VEP Varnish Esi Parsing
+ * VEF Varnish Esi Fetching
  */
 
 #include "config.h"
@@ -94,16 +94,6 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
 	return (1);
 }
 
-/*---------------------------------------------------------------------
- * We receive a ungzip'ed object, and want to store it gzip'ed.
- */
-
-static int __match_proto__()
-vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
-{
-	return (vfp_esi_bytes_uu(sp, htc, bytes));
-}
-
 /*---------------------------------------------------------------------
  * We receive a gzip'ed object, and want to store it ungzip'ed.
  */
@@ -159,6 +149,123 @@ vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
 	return (1);
 }
 
+/*---------------------------------------------------------------------
+ */
+
+struct vef_priv {
+	unsigned		magic;
+#define VEF_MAGIC		0xf104b51f
+	struct vgz		*vgz;
+
+	char			*bufp;
+	ssize_t			tot;
+	int			error;
+};
+
+/*---------------------------------------------------------------------
+ * We receive a ungzip'ed object, and want to store it gzip'ed.
+ */
+
+#include "vend.h"
+
+static ssize_t
+vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
+{
+	struct vef_priv *vef;
+	struct storage *st;
+	size_t dl;
+	const void *dp;
+	int i;
+	char *p;
+
+printf("ZCB(%jd, %d)\n", l, flg);
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vef = sp->wrk->vef_priv;
+	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+	assert(l >= 0);
+
+	if (vef->error) {
+		vef->tot += l;
+		return (vef->tot);
+	}
+
+	VGZ_Ibuf(vef->vgz, vef->bufp, l);
+	do {
+		if (sp->wrk->storage == NULL) {
+			l = params->fetch_chunksize * 1024LL;
+			sp->wrk->storage = STV_alloc(sp, l);
+		}
+		if (sp->wrk->storage == NULL) {
+			vef->error = ENOMEM;
+			vef->tot += l;
+			return (vef->tot);
+		}
+
+		st = sp->wrk->storage;
+		VGZ_Obuf(vef->vgz, st->ptr + st->len, st->space - st->len);
+		p = (void*)(st->ptr + st->len);
+		i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
+printf("GZI = %d %jd\n", i, dl);
+		vef->tot += dl;
+		st->len += dl;
+		if (st->len == st->space) {
+			VTAILQ_INSERT_TAIL(&sp->obj->store,
+			    sp->wrk->storage, list);
+			sp->wrk->storage = NULL;
+			st = NULL;
+		}
+	} while (!VGZ_IbufEmpty(vef->vgz));
+	vef->bufp += l;
+if (flg == VGZ_FINISH)
+	assert(i == 1);			/* XXX */
+else
+	assert(i == 0);			/* XXX */
+printf("ZCB = %jd\n", vef->tot);
+fflush(stdout);
+usleep(100);
+	return (vef->tot);
+}
+
+static int __match_proto__()
+vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	ssize_t l, w;
+	char ibuf[1024 * params->gzip_stack_buffer];
+	struct vef_priv *vef;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vef = sp->wrk->vef_priv;
+	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+
+	while (bytes > 0) {
+		if (sp->wrk->storage == NULL) {
+			l = params->fetch_chunksize * 1024LL;
+			sp->wrk->storage = STV_alloc(sp, l);
+		}
+		if (sp->wrk->storage == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		l = sizeof ibuf;
+		if (l > bytes)
+			l = bytes;
+		w = HTC_Read(htc, ibuf, l);
+		if (w <= 0) {
+printf("RT %jd\n", w);
+			return (w);
+}
+		bytes -= w;
+		vef->bufp = ibuf;
+		VEP_parse(sp, ibuf, l);
+		if (vef->error) {
+			// errno = vef->error;
+			return (-1);
+		}
+	}
+	return (1);
+}
+
 /*---------------------------------------------------------------------
  * We receive a gzip'ed object, and want to store it gzip'ed.
  */
@@ -175,14 +282,26 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 static void __match_proto__()
 vfp_esi_begin(struct sess *sp, size_t estimate)
 {
+	struct vef_priv *vef;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	/* XXX: snapshot WS's ? We'll need the space */
 
-	VEP_Init(sp, NULL);
-
-	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
+	if (sp->wrk->is_gzip && sp->wrk->do_gunzip) {
 		sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
+		VEP_Init(sp, NULL);
+	} else if (sp->wrk->is_gunzip && sp->wrk->do_gzip) {
+		VEP_Init(sp, vfp_vep_callback);
+		vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
+		AN(vef);
+		memset(vef, 0, sizeof *vef);
+		vef->magic = VEF_MAGIC;
+		vef->vgz = VGZ_NewGzip(sp, sp->ws);
+		AZ(sp->wrk->vef_priv);
+		sp->wrk->vef_priv = vef;
+	} else {
+		VEP_Init(sp, NULL);
+	}
 
 	(void)estimate;
 }
@@ -193,6 +312,7 @@ vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+printf("BYTES = %jd\n", bytes);
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
 		i = vfp_esi_bytes_gu(sp, htc, bytes);
 	else if (sp->wrk->is_gunzip && sp->wrk->do_gzip)
@@ -201,6 +321,7 @@ vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 		i = vfp_esi_bytes_gg(sp, htc, bytes);
 	else
 		i = vfp_esi_bytes_uu(sp, htc, bytes);
+printf("BYTES = %d\n", i);
 	return (i);
 }
 
@@ -209,8 +330,10 @@ vfp_esi_end(struct sess *sp)
 {
 	struct storage *st;
 	struct vsb *vsb;
+	struct vef_priv *vef;
 	ssize_t l;
 
+printf("END\n");
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
 		VGZ_Destroy(&sp->wrk->vgz_rx);
@@ -230,6 +353,16 @@ vfp_esi_end(struct sess *sp)
 
 	st = sp->wrk->storage;
 	sp->wrk->storage = NULL;
+
+	if (sp->wrk->vef_priv != NULL) {
+		vef = sp->wrk->vef_priv;
+		sp->wrk->vef_priv = NULL;
+		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+		XXXAZ(vef->error);
+printf("TOT %jd\n", vef->tot);
+sp->obj->len = vef->tot;
+	}
+
 	if (st == NULL)
 		return (0);
 

bin/varnishd/cache_esi_parse.c

@@ -64,6 +64,7 @@ struct vep_state {
 	struct vsb		*vsb;
 
 	const struct sess	*sp;
+	int			dogzip;
 	vep_callback_t		*cb;
 
 	/* Internal Counter for default call-back function */
@@ -285,9 +286,12 @@ vep_emit_verbatim(const struct vep_state *vep, ssize_t l, ssize_t l_crc)
 		Debug("---> VERBATIM(%jd)\n", (intmax_t)l);
 	}
 	vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V8);
-	vep_emit_len(vep, l_crc, VEC_C1, VEC_C2, VEC_C8);
-	vbe32enc(buf, vep->crc);
-	vsb_bcat(vep->vsb, buf, sizeof buf);
+	if (vep->dogzip) {
+		vep_emit_len(vep, l_crc, VEC_C1, VEC_C2, VEC_C8);
+		vbe32enc(buf, vep->crc);
+		vsb_bcat(vep->vsb, buf, sizeof buf);
+	}
+	/* Emit Chunked header */
 	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
 } 
 
@@ -992,14 +996,19 @@ VEP_Init(const struct sess *sp, vep_callback_t *cb)
 	memset(vep, 0, sizeof *vep);
 	vep->magic = VEP_MAGIC;
 	vep->sp = sp;
-	if (cb != NULL)
+	vep->vsb = vsb_newauto();
+	AN(vep->vsb);
+
+	if (cb != NULL) {
+		vep->dogzip = 1;
+		/* XXX */
+		vsb_printf(vep->vsb, "%c", VEC_GZ);
 		vep->cb = cb;
-	else
+	} else {
 		vep->cb = vep_default_cb;
+	}
 
 	vep->state = VEP_START;
-	vep->vsb = vsb_newauto();
-	AN(vep->vsb);
 	vep->crc = crc32(0L, Z_NULL, 0);
 	vep->crcp = crc32(0L, Z_NULL, 0);
 }
@@ -1020,8 +1029,9 @@ VEP_Finish(const struct sess *sp)
 	if (vep->o_pending)
 		vep_mark_common(vep, vep->ver_p, vep->last_mark);
 	if (vep->o_wait > 0) {
-		lcb = vep->cb(vep->sp, 0, VGZ_FINISH);
+		lcb = vep->cb(vep->sp, 0, VGZ_ALIGN);
 		vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
+		(void)vep->cb(vep->sp, 0, VGZ_FINISH);
 	}
 
 	sp->wrk->vep = NULL;

bin/varnishd/cache_fetch.c

@@ -204,8 +204,8 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 
 	i = sp->wrk->vfp->bytes(sp, htc, cl);
 	if (i <= 0) {
-		WSP(sp, SLT_FetchError, "straight read_error: %d (%s)",
-		    errno, strerror(errno));
+		WSP(sp, SLT_FetchError, "straight read_error: %d %d (%s)",
+		    i, errno, strerror(errno));
 		return (-1);
 	}
 	return (0);

bin/varnishtest/tests/e00021.vtc

+# $Id$
+
+test "ESI ability to stitch gzip files together"
+
+
+server s1 {
+	rxreq 
+	txresp -body {
+		<esi:remove>
+		This is a test: Unseen University
+		<esi:include src="trick question">
+		<!--esi XXX -->
+		</esi:remove>
+		<esX>This is a test: Hello world
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		set req.esi = true;
+	}
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+		set beresp.do_gzip = true;
+	}
+} -start 
+
+varnish v1 -cliok "param.set esi_syntax 4"
+varnish v1 -cliok "param.set http_gzip_support true"
+
+client c1 {
+	txreq  -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == gzip
+	gunzip
+	expect resp.status == 200
+	expect resp.bodylen == 40
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 2