Add the encoder object.

README.rst

@@ -70,7 +70,14 @@ algorithm for responses fetched from backends.
 
 XXX ...
 
-# $Object encoder(STRING name, BYTES buffer=32k, INT quality=11, INT lgwin=22)
+.. _vmod_brotli.encoder:
+
+new xencoder = brotli.encoder(STRING name, BYTES bufffer)
+---------------------------------------------------------
+
+::
+
+   new xencoder = brotli.encoder(STRING name, BYTES bufffer=32768)
 
 XXX ...
 

src/tests/encoder.vtc

+# looks like -*- vcl -*-
+
+varnishtest "brotli compression with custom parameter settings"
+
+# Test vectors from github.com/google/brotli/tests/testdata
+
+server s1 {
+	rxreq
+	txresp -body {The quick brown fox jumps over the lazy dog}
+	rxreq
+	txresp -body {Xyzzy}
+} -start
+
+varnish v1 -arg "-p vsl_mask=+VfpAcct" -vcl+backend {
+	import ${vmod_brotli};
+
+	sub vcl_init {
+		new mybr = brotli.encoder("br1k", 1k);
+	}
+
+	sub vcl_backend_response {
+		set beresp.filters = "br1k";
+		set beresp.uncacheable = true;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.http.Content-Encoding == "br"
+	expect resp.http.Vary == "Accept-Encoding"
+	expect resp.bodylen == 47
+	expect resp.body == "The quick brown fox jumps over the lazy dog"
+
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.http.Content-Encoding == "br"
+	expect resp.http.Vary == "Accept-Encoding"
+	expect resp.bodylen == 9
+	expect resp.body == {Xyzzy}
+} -run
+
+varnish v1 -vcl+backend {
+	import ${vmod_brotli};
+
+	sub vcl_init {
+		new mybr = brotli.encoder("br1m", 1m);
+	}
+
+	sub vcl_backend_response {
+		set beresp.filters = "br1m";
+		set beresp.uncacheable = true;
+	}
+}
+
+# Tests object finalization and DISCARD event.
+varnish v1 -cli "vcl.discard vcl1"
+varnish v1 -cli "vcl.list"
+
+server s1 -wait
+server s1 -start
+
+client c1 -run
+
+varnish v1 -errvcl {vfp brotli failure: new mybr: filter name must be non-empty} {
+	import ${vmod_brotli};
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new mybr = brotli.encoder("");
+	}
+}
+
+varnish v1 -errvcl {vfp brotli failure: new mybr: filter name br already in use by another VFP} {
+	import ${vmod_brotli};
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new mybr = brotli.encoder("br");
+	}
+}
+
+varnish v1 -errvcl {vfp brotli failure: new mybr: filter name unbr already in use by another VFP} {
+	import ${vmod_brotli};
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new mybr = brotli.encoder("unbr");
+	}
+}
+
+varnish v1 -errvcl {vfp brotli failure: new mybr: filter name esi already in use by another VFP} {
+	import ${vmod_brotli};
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new mybr = brotli.encoder("esi");
+	}
+}
+
+varnish v1 -errvcl {vfp brotli failure: new mybr: filter name esi_gzip already in use by another VFP} {
+	import ${vmod_brotli};
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new mybr = brotli.encoder("esi_gzip");
+	}
+}
+
+varnish v1 -errvcl {vfp brotli failure: new mybr: filter name gunzip already in use by another VFP} {
+	import ${vmod_brotli};
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new mybr = brotli.encoder("gunzip");
+	}
+}
+
+varnish v1 -errvcl {vfp brotli failure: new mybr: filter name gzip already in use by another VFP} {
+	import ${vmod_brotli};
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new mybr = brotli.encoder("gzip");
+	}
+}
+
+varnish v1 -errvcl {vfp brotli failure: new mybr: filter name testgunzip already in use by another VFP} {
+	import ${vmod_brotli};
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new mybr = brotli.encoder("testgunzip");
+	}
+}
+
+varnish v1 -errvcl {vfp brotli failure: new mybr: buffer size must be > 0b} {
+	import ${vmod_brotli};
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new mybr = brotli.encoder("empty", 0b);
+	}
+}
+
+# Happens to be the same as the default encoder.
+varnish v1 -vcl+backend {
+	import ${vmod_brotli};
+
+	sub vcl_init {
+		new mybr = brotli.encoder("br_default", 32k);
+	}
+
+	sub vcl_backend_response {
+		set beresp.filters = "br_default";
+		set beresp.uncacheable = true;
+	}
+}
+
+server s1 -wait
+server s1 -start
+
+client c1 -run
+
+logexpect l1 -v v1 -d 1 -g vxid -q "VfpAcct" {
+	expect 0 * Begin	bereq
+	expect * = VfpAcct	{^br1k \d+ 47$}
+	expect * = End
+
+	expect 0 * Begin	bereq
+	expect * = VfpAcct	{^br1k \d+ 9$}
+	expect * = End
+
+	expect 0 * Begin	bereq
+	expect * = VfpAcct	{^br1m \d+ 47$}
+	expect * = End
+
+	expect 0 * Begin	bereq
+	expect * = VfpAcct	{^br1m \d+ 9$}
+	expect * = End
+
+	expect 0 * Begin	bereq
+	expect * = VfpAcct	{^br_default \d+ 47$}
+	expect * = End
+
+	expect 0 * Begin	bereq
+	expect * = VfpAcct	{^br_default \d+ 9$}
+	expect * = End
+} -run

src/vfp_brotli.c

@@ -29,7 +29,7 @@
  */
 
 /* for strdup() */
-//#define _POSIX_C_SOURCE 200809L
+#define _POSIX_C_SOURCE 200809L
 
 #include "config.h"
 
@@ -43,8 +43,11 @@
 
 #include "vcc_if.h"
 
-/* XXX make this configurable; cf. varnishd default gzip_buffer */
-#define BUFFER_SZ (32 * 1024 * 1024)
+#define VFAIL(ctx, fmt, ...) \
+        VRT_fail((ctx), "vfp brotli failure: " fmt, __VA_ARGS__)
+
+/* cf. varnishd default gzip_buffer */
+#define DEFAULT_BUFSZ (32 * 1024 * 1024)
 
 struct vbr_stream {
 	const uint8_t		*next_in;
@@ -70,6 +73,28 @@ struct vbr {
 	enum vbr_which		which;
 };
 
+struct vbr_settings {
+	unsigned	magic;
+#define VBR_SETTINGS_MAGIC 0xa61992aa
+	VCL_BYTES	bufsz;
+	enum vbr_which	which;
+};
+
+struct vmod_brotli_encoder {
+	unsigned	magic;
+#define VMOD_BROTLI_ENCODER_MAGIC 0x1490a42a
+	struct vfp	*vfp;
+};
+
+struct custom_vfp_entry {
+	unsigned	magic;
+#define CUSTOM_VFP_MAGIC 0xfc88cb98
+	VSLIST_ENTRY(custom_vfp_entry) list;
+	struct vfp	*vfp;
+};
+
+VSLIST_HEAD(custom_vfp_head, custom_vfp_entry);
+
 static struct vbr *
 newVBR(enum vbr_which which)
 {
@@ -112,15 +137,14 @@ destroy(struct vbr **vp)
 }
 
 static int
-getBuf(struct vbr *vbr)
+getBuf(struct vbr *vbr, const struct vbr_settings *settings)
 {
 	CHECK_OBJ_NOTNULL(vbr, VBR_MAGIC);
 	AZ(vbr->bufsz);
 	AZ(vbr->buflen);
 	AZ(vbr->buf);
 
-	/* XXX make this configurable */
-	vbr->bufsz = BUFFER_SZ;
+	vbr->bufsz = settings->bufsz;
 	vbr->buf = malloc(vbr->bufsz);
 	if (vbr->buf == NULL) {
 		vbr->bufsz = 0;
@@ -218,9 +242,12 @@ static enum vfp_status v_matchproto_(vfp_init_f)
 vfp_br_init(struct vfp_ctx *ctx, struct vfp_entry *ent)
 {
 	struct vbr *vbr;
+	const struct vbr_settings *settings;
 
 	CHECK_OBJ_NOTNULL(ctx, VFP_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ent, VFP_ENTRY_MAGIC);
+	AN(ent->vfp);
+	CAST_OBJ_NOTNULL(settings, ent->vfp->priv1, VBR_SETTINGS_MAGIC);
 
 	/*
 	 * Ignore partial responses to range requests (as Varnish does for
@@ -229,7 +256,7 @@ vfp_br_init(struct vfp_ctx *ctx, struct vfp_entry *ent)
 	if (http_GetStatus(ctx->resp) == 206)
 	 	return (VFP_NULL);
 
-	if (ent->vfp == &vfp_br) {
+	if (settings->which == ENC) {
 		if (http_GetHdr(ctx->resp, H_Content_Encoding, NULL))
 			return (VFP_NULL);
 		vbr = newVBR(ENC);
@@ -242,7 +269,7 @@ vfp_br_init(struct vfp_ctx *ctx, struct vfp_entry *ent)
 	if (vbr == NULL)
 		return (VFP_ERROR);
 	ent->priv1 = vbr;
-	if (getBuf(vbr))
+	if (getBuf(vbr, settings))
 		return (VFP_ERROR);
 	setInputBuf(vbr, vbr->buf, 0);
 	AZ(vbr->buflen);
@@ -251,7 +278,7 @@ vfp_br_init(struct vfp_ctx *ctx, struct vfp_entry *ent)
 	http_Unset(ctx->resp, H_Content_Length);
 	RFC2616_Weaken_Etag(ctx->resp);
 
-	if (ent->vfp == &vfp_br) {
+	if (settings->which == ENC) {
 		http_SetHeader(ctx->resp, "Content-Encoding: br");
 		RFC2616_Vary_AE(ctx->resp);
 	}
@@ -378,11 +405,24 @@ vfp_unbr_pull(struct vfp_ctx *ctx, struct vfp_entry *ent, void *ptr,
 	return (VFP_END);
 }
 
+static const struct vbr_settings default_encoder = {
+	.magic = VBR_SETTINGS_MAGIC,
+	.bufsz = DEFAULT_BUFSZ,
+	.which = ENC,
+};
+
+static const struct vbr_settings default_decoder = {
+	.magic = VBR_SETTINGS_MAGIC,
+	.bufsz = DEFAULT_BUFSZ,
+	.which = DEC,
+};
+
 static const struct vfp vfp_br = {
 	.name = "br",
 	.init = vfp_br_init,
 	.pull = vfp_br_pull,
 	.fini = vfp_br_fini,
+	.priv1 = &default_encoder,
 };
 
 static const struct vfp vfp_unbr = {
@@ -390,17 +430,41 @@ static const struct vfp vfp_unbr = {
 	.init = vfp_br_init,
 	.pull = vfp_unbr_pull,
 	.fini = vfp_br_fini,
+	.priv1 = &default_decoder,
 };
 
 /* Event function */
 
+static struct custom_vfp_head *
+init_priv_vcl(struct vmod_priv *priv)
+{
+	struct custom_vfp_head *vfph;
+
+	AN(priv);
+	if (priv->priv == NULL) {
+		vfph = malloc(sizeof(*vfph));
+		AN(vfph);
+		priv->priv = vfph;
+		VSLIST_INIT(vfph);
+	}
+	else
+		vfph = priv->priv;
+	return (vfph);
+}
+
 int
 vmod_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
 {
+	struct custom_vfp_head *vfph;
+	struct custom_vfp_entry *vfpe;
+	struct vbr_settings *settings;
+
 	ASSERT_CLI();
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(priv);
 
+	vfph = init_priv_vcl(priv);
+
 	switch(e) {
 	case VCL_EVENT_LOAD:
 		VRT_AddVFP(ctx, &vfp_br);
@@ -409,6 +473,23 @@ vmod_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
 	case VCL_EVENT_DISCARD:
 		VRT_RemoveVFP(ctx, &vfp_br);
 		VRT_RemoveVFP(ctx, &vfp_unbr);
+		while (!VSLIST_EMPTY(vfph)) {
+			vfpe = VSLIST_FIRST(vfph);
+			CHECK_OBJ_NOTNULL(vfpe, CUSTOM_VFP_MAGIC);
+			if (vfpe->vfp != NULL) {
+				if (vfpe->vfp->priv1 != NULL) {
+					CAST_OBJ(settings,
+						 TRUST_ME(vfpe->vfp->priv1),
+						 VBR_SETTINGS_MAGIC);
+					FREE_OBJ(settings);
+				}
+				VRT_RemoveVFP(ctx, vfpe->vfp);
+				free(vfpe->vfp);
+			}
+			VSLIST_REMOVE_HEAD(vfph, list);
+			FREE_OBJ(vfpe);
+		}
+		free(vfph);
 		return (0);
 	case VCL_EVENT_WARM:
 	case VCL_EVENT_COLD:
@@ -419,6 +500,115 @@ vmod_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
 	NEEDLESS(return (0));
 }
 
+/* Object encoder */
+
+VCL_VOID
+vmod_encoder__init(VRT_CTX, struct vmod_brotli_encoder **encp,
+		   const char *vcl_name, struct vmod_priv *priv,
+		   VCL_STRING filter_name, VCL_BYTES bufsz)
+{
+	struct vmod_brotli_encoder *enc;
+	struct vfp *vfp;
+	struct vbr_settings *settings;
+	struct custom_vfp_head *vfph;
+	struct custom_vfp_entry *vfpe;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AN(encp);
+	AZ(*encp);
+	AN(vcl_name);
+	AN(priv);
+	AN(filter_name);
+	assert(bufsz >= 0);
+
+	if (*filter_name == '\0') {
+		VFAIL(ctx, "new %s: filter name must be non-empty", vcl_name);
+		return;
+	}
+	if (strcmp(filter_name, "br") == 0 || strcmp(filter_name, "unbr") == 0
+	    || strcmp(filter_name, "esi") == 0
+	    || strcmp(filter_name, "esi_gzip") == 0
+	    || strcmp(filter_name, "gunzip") == 0
+	    || strcmp(filter_name, "gzip") == 0
+	    || strcmp(filter_name, "testgunzip") == 0) {
+		VFAIL(ctx,
+		      "new %s: filter name %s already in use by another VFP",
+		      vcl_name, filter_name);
+		return;
+	}
+	if (bufsz == 0) {
+		VFAIL(ctx, "new %s: buffer size must be > 0b", vcl_name);
+		return;
+	}
+
+	errno = 0;
+	ALLOC_OBJ(enc, VMOD_BROTLI_ENCODER_MAGIC);
+	if (enc == NULL) {
+		VFAIL(ctx, "new %s: cannot allocate space for object: %s",
+		      vcl_name, strerror(errno));
+		return;
+	}
+
+	errno = 0;
+	vfp = malloc(sizeof(*vfp));
+	if (vfp == NULL) {
+		VFAIL(ctx, "new %s: cannot allocate space for VFP: %s",
+		      vcl_name, strerror(errno));
+		return;
+	}
+
+	errno = 0;
+	ALLOC_OBJ(settings, VBR_SETTINGS_MAGIC);
+	if (vfp == NULL) {
+		VFAIL(ctx, "new %s: cannot allocate space for settings: %s",
+		      vcl_name, strerror(errno));
+		return;
+	}
+
+	errno = 0;
+	ALLOC_OBJ(vfpe, CUSTOM_VFP_MAGIC);
+	if (vfpe == NULL) {
+		VFAIL(ctx, "new %s: cannot allocate space VFP entry: %s",
+		      vcl_name, strerror(errno));
+		return;
+	}
+
+	settings->bufsz = bufsz;
+	settings->which = ENC;
+
+	vfp->name = strdup(filter_name);
+	vfp->init = vfp_br_init;
+	vfp->pull = vfp_br_pull;
+	vfp->fini = vfp_br_fini;
+	vfp->priv1 = settings;
+	enc->vfp = vfp;
+	VRT_AddVFP(ctx, vfp);
+
+	vfph = init_priv_vcl(priv);
+	vfpe->vfp = vfp;
+	VSLIST_INSERT_HEAD(vfph, vfpe, list);
+}
+
+/*
+ * The settings and custom VFP objects and PRIV_VCL list entry are freed
+ * on the DISCARD event, because we need a VRT_CTX to call
+ * VRT_RemoveVFP().
+ */
+VCL_VOID
+vmod_encoder__fini(struct vmod_brotli_encoder **encp)
+{
+	struct vmod_brotli_encoder *enc;
+
+	if (encp == NULL || *encp == NULL)
+		return;
+	CHECK_OBJ(*encp, VMOD_BROTLI_ENCODER_MAGIC);
+	enc = *encp;
+	*encp = NULL;
+	FREE_OBJ(enc);
+}
+
+/* Version functions */
+
 static VCL_STRING
 brotli_version(VRT_CTX, uint32_t version)
 {

src/vfp_brotli.vcc

@@ -66,7 +66,7 @@ algorithm for responses fetched from backends.
 
 XXX ...
 
-# $Object encoder(STRING name, BYTES buffer=32k, INT quality=11, INT lgwin=22)
+$Object encoder(PRIV_VCL, STRING name, BYTES bufffer=32768)
 
 XXX ...