Commit 74d180d3 by Geoff Simmons

Add the encoder object.

parent 113f2318
......@@ -70,7 +70,14 @@ algorithm for responses fetched from backends.
XXX ...
# $Object encoder(STRING name, BYTES buffer=32k, INT quality=11, INT lgwin=22)
.. _vmod_brotli.encoder:
new xencoder = brotli.encoder(STRING name, BYTES bufffer)
---------------------------------------------------------
::
new xencoder = brotli.encoder(STRING name, BYTES bufffer=32768)
XXX ...
......
# looks like -*- vcl -*-
varnishtest "brotli compression with custom parameter settings"
# Test vectors from github.com/google/brotli/tests/testdata
server s1 {
rxreq
txresp -body {The quick brown fox jumps over the lazy dog}
rxreq
txresp -body {Xyzzy}
} -start
varnish v1 -arg "-p vsl_mask=+VfpAcct" -vcl+backend {
import ${vmod_brotli};
sub vcl_init {
new mybr = brotli.encoder("br1k", 1k);
}
sub vcl_backend_response {
set beresp.filters = "br1k";
set beresp.uncacheable = true;
}
} -start
client c1 {
txreq
rxresp
expect resp.status == 200
expect resp.http.Content-Encoding == "br"
expect resp.http.Vary == "Accept-Encoding"
expect resp.bodylen == 47
expect resp.body == " The quick brown fox jumps over the lazy dog"
txreq
rxresp
expect resp.status == 200
expect resp.http.Content-Encoding == "br"
expect resp.http.Vary == "Accept-Encoding"
expect resp.bodylen == 9
expect resp.body == { Xyzzy}
} -run
varnish v1 -vcl+backend {
import ${vmod_brotli};
sub vcl_init {
new mybr = brotli.encoder("br1m", 1m);
}
sub vcl_backend_response {
set beresp.filters = "br1m";
set beresp.uncacheable = true;
}
}
# Tests object finalization and DISCARD event.
varnish v1 -cli "vcl.discard vcl1"
varnish v1 -cli "vcl.list"
server s1 -wait
server s1 -start
client c1 -run
varnish v1 -errvcl {vfp brotli failure: new mybr: filter name must be non-empty} {
import ${vmod_brotli};
backend b { .host = "${bad_ip}"; }
sub vcl_init {
new mybr = brotli.encoder("");
}
}
varnish v1 -errvcl {vfp brotli failure: new mybr: filter name br already in use by another VFP} {
import ${vmod_brotli};
backend b { .host = "${bad_ip}"; }
sub vcl_init {
new mybr = brotli.encoder("br");
}
}
varnish v1 -errvcl {vfp brotli failure: new mybr: filter name unbr already in use by another VFP} {
import ${vmod_brotli};
backend b { .host = "${bad_ip}"; }
sub vcl_init {
new mybr = brotli.encoder("unbr");
}
}
varnish v1 -errvcl {vfp brotli failure: new mybr: filter name esi already in use by another VFP} {
import ${vmod_brotli};
backend b { .host = "${bad_ip}"; }
sub vcl_init {
new mybr = brotli.encoder("esi");
}
}
varnish v1 -errvcl {vfp brotli failure: new mybr: filter name esi_gzip already in use by another VFP} {
import ${vmod_brotli};
backend b { .host = "${bad_ip}"; }
sub vcl_init {
new mybr = brotli.encoder("esi_gzip");
}
}
varnish v1 -errvcl {vfp brotli failure: new mybr: filter name gunzip already in use by another VFP} {
import ${vmod_brotli};
backend b { .host = "${bad_ip}"; }
sub vcl_init {
new mybr = brotli.encoder("gunzip");
}
}
varnish v1 -errvcl {vfp brotli failure: new mybr: filter name gzip already in use by another VFP} {
import ${vmod_brotli};
backend b { .host = "${bad_ip}"; }
sub vcl_init {
new mybr = brotli.encoder("gzip");
}
}
varnish v1 -errvcl {vfp brotli failure: new mybr: filter name testgunzip already in use by another VFP} {
import ${vmod_brotli};
backend b { .host = "${bad_ip}"; }
sub vcl_init {
new mybr = brotli.encoder("testgunzip");
}
}
varnish v1 -errvcl {vfp brotli failure: new mybr: buffer size must be > 0b} {
import ${vmod_brotli};
backend b { .host = "${bad_ip}"; }
sub vcl_init {
new mybr = brotli.encoder("empty", 0b);
}
}
# Happens to be the same as the default encoder.
varnish v1 -vcl+backend {
import ${vmod_brotli};
sub vcl_init {
new mybr = brotli.encoder("br_default", 32k);
}
sub vcl_backend_response {
set beresp.filters = "br_default";
set beresp.uncacheable = true;
}
}
server s1 -wait
server s1 -start
client c1 -run
logexpect l1 -v v1 -d 1 -g vxid -q "VfpAcct" {
expect 0 * Begin bereq
expect * = VfpAcct {^br1k \d+ 47$}
expect * = End
expect 0 * Begin bereq
expect * = VfpAcct {^br1k \d+ 9$}
expect * = End
expect 0 * Begin bereq
expect * = VfpAcct {^br1m \d+ 47$}
expect * = End
expect 0 * Begin bereq
expect * = VfpAcct {^br1m \d+ 9$}
expect * = End
expect 0 * Begin bereq
expect * = VfpAcct {^br_default \d+ 47$}
expect * = End
expect 0 * Begin bereq
expect * = VfpAcct {^br_default \d+ 9$}
expect * = End
} -run
......@@ -29,7 +29,7 @@
*/
/* for strdup() */
//#define _POSIX_C_SOURCE 200809L
#define _POSIX_C_SOURCE 200809L
#include "config.h"
......@@ -43,8 +43,11 @@
#include "vcc_if.h"
/* XXX make this configurable; cf. varnishd default gzip_buffer */
#define BUFFER_SZ (32 * 1024 * 1024)
#define VFAIL(ctx, fmt, ...) \
VRT_fail((ctx), "vfp brotli failure: " fmt, __VA_ARGS__)
/* cf. varnishd default gzip_buffer */
#define DEFAULT_BUFSZ (32 * 1024 * 1024)
struct vbr_stream {
const uint8_t *next_in;
......@@ -70,6 +73,28 @@ struct vbr {
enum vbr_which which;
};
struct vbr_settings {
unsigned magic;
#define VBR_SETTINGS_MAGIC 0xa61992aa
VCL_BYTES bufsz;
enum vbr_which which;
};
struct vmod_brotli_encoder {
unsigned magic;
#define VMOD_BROTLI_ENCODER_MAGIC 0x1490a42a
struct vfp *vfp;
};
struct custom_vfp_entry {
unsigned magic;
#define CUSTOM_VFP_MAGIC 0xfc88cb98
VSLIST_ENTRY(custom_vfp_entry) list;
struct vfp *vfp;
};
VSLIST_HEAD(custom_vfp_head, custom_vfp_entry);
static struct vbr *
newVBR(enum vbr_which which)
{
......@@ -112,15 +137,14 @@ destroy(struct vbr **vp)
}
static int
getBuf(struct vbr *vbr)
getBuf(struct vbr *vbr, const struct vbr_settings *settings)
{
CHECK_OBJ_NOTNULL(vbr, VBR_MAGIC);
AZ(vbr->bufsz);
AZ(vbr->buflen);
AZ(vbr->buf);
/* XXX make this configurable */
vbr->bufsz = BUFFER_SZ;
vbr->bufsz = settings->bufsz;
vbr->buf = malloc(vbr->bufsz);
if (vbr->buf == NULL) {
vbr->bufsz = 0;
......@@ -218,9 +242,12 @@ static enum vfp_status v_matchproto_(vfp_init_f)
vfp_br_init(struct vfp_ctx *ctx, struct vfp_entry *ent)
{
struct vbr *vbr;
const struct vbr_settings *settings;
CHECK_OBJ_NOTNULL(ctx, VFP_CTX_MAGIC);
CHECK_OBJ_NOTNULL(ent, VFP_ENTRY_MAGIC);
AN(ent->vfp);
CAST_OBJ_NOTNULL(settings, ent->vfp->priv1, VBR_SETTINGS_MAGIC);
/*
* Ignore partial responses to range requests (as Varnish does for
......@@ -229,7 +256,7 @@ vfp_br_init(struct vfp_ctx *ctx, struct vfp_entry *ent)
if (http_GetStatus(ctx->resp) == 206)
return (VFP_NULL);
if (ent->vfp == &vfp_br) {
if (settings->which == ENC) {
if (http_GetHdr(ctx->resp, H_Content_Encoding, NULL))
return (VFP_NULL);
vbr = newVBR(ENC);
......@@ -242,7 +269,7 @@ vfp_br_init(struct vfp_ctx *ctx, struct vfp_entry *ent)
if (vbr == NULL)
return (VFP_ERROR);
ent->priv1 = vbr;
if (getBuf(vbr))
if (getBuf(vbr, settings))
return (VFP_ERROR);
setInputBuf(vbr, vbr->buf, 0);
AZ(vbr->buflen);
......@@ -251,7 +278,7 @@ vfp_br_init(struct vfp_ctx *ctx, struct vfp_entry *ent)
http_Unset(ctx->resp, H_Content_Length);
RFC2616_Weaken_Etag(ctx->resp);
if (ent->vfp == &vfp_br) {
if (settings->which == ENC) {
http_SetHeader(ctx->resp, "Content-Encoding: br");
RFC2616_Vary_AE(ctx->resp);
}
......@@ -378,11 +405,24 @@ vfp_unbr_pull(struct vfp_ctx *ctx, struct vfp_entry *ent, void *ptr,
return (VFP_END);
}
static const struct vbr_settings default_encoder = {
.magic = VBR_SETTINGS_MAGIC,
.bufsz = DEFAULT_BUFSZ,
.which = ENC,
};
static const struct vbr_settings default_decoder = {
.magic = VBR_SETTINGS_MAGIC,
.bufsz = DEFAULT_BUFSZ,
.which = DEC,
};
static const struct vfp vfp_br = {
.name = "br",
.init = vfp_br_init,
.pull = vfp_br_pull,
.fini = vfp_br_fini,
.priv1 = &default_encoder,
};
static const struct vfp vfp_unbr = {
......@@ -390,17 +430,41 @@ static const struct vfp vfp_unbr = {
.init = vfp_br_init,
.pull = vfp_unbr_pull,
.fini = vfp_br_fini,
.priv1 = &default_decoder,
};
/* Event function */
static struct custom_vfp_head *
init_priv_vcl(struct vmod_priv *priv)
{
struct custom_vfp_head *vfph;
AN(priv);
if (priv->priv == NULL) {
vfph = malloc(sizeof(*vfph));
AN(vfph);
priv->priv = vfph;
VSLIST_INIT(vfph);
}
else
vfph = priv->priv;
return (vfph);
}
int
vmod_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
{
struct custom_vfp_head *vfph;
struct custom_vfp_entry *vfpe;
struct vbr_settings *settings;
ASSERT_CLI();
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
AN(priv);
vfph = init_priv_vcl(priv);
switch(e) {
case VCL_EVENT_LOAD:
VRT_AddVFP(ctx, &vfp_br);
......@@ -409,6 +473,23 @@ vmod_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
case VCL_EVENT_DISCARD:
VRT_RemoveVFP(ctx, &vfp_br);
VRT_RemoveVFP(ctx, &vfp_unbr);
while (!VSLIST_EMPTY(vfph)) {
vfpe = VSLIST_FIRST(vfph);
CHECK_OBJ_NOTNULL(vfpe, CUSTOM_VFP_MAGIC);
if (vfpe->vfp != NULL) {
if (vfpe->vfp->priv1 != NULL) {
CAST_OBJ(settings,
TRUST_ME(vfpe->vfp->priv1),
VBR_SETTINGS_MAGIC);
FREE_OBJ(settings);
}
VRT_RemoveVFP(ctx, vfpe->vfp);
free(vfpe->vfp);
}
VSLIST_REMOVE_HEAD(vfph, list);
FREE_OBJ(vfpe);
}
free(vfph);
return (0);
case VCL_EVENT_WARM:
case VCL_EVENT_COLD:
......@@ -419,6 +500,115 @@ vmod_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
NEEDLESS(return (0));
}
/* Object encoder */
VCL_VOID
vmod_encoder__init(VRT_CTX, struct vmod_brotli_encoder **encp,
const char *vcl_name, struct vmod_priv *priv,
VCL_STRING filter_name, VCL_BYTES bufsz)
{
struct vmod_brotli_encoder *enc;
struct vfp *vfp;
struct vbr_settings *settings;
struct custom_vfp_head *vfph;
struct custom_vfp_entry *vfpe;
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
AN(encp);
AZ(*encp);
AN(vcl_name);
AN(priv);
AN(filter_name);
assert(bufsz >= 0);
if (*filter_name == '\0') {
VFAIL(ctx, "new %s: filter name must be non-empty", vcl_name);
return;
}
if (strcmp(filter_name, "br") == 0 || strcmp(filter_name, "unbr") == 0
|| strcmp(filter_name, "esi") == 0
|| strcmp(filter_name, "esi_gzip") == 0
|| strcmp(filter_name, "gunzip") == 0
|| strcmp(filter_name, "gzip") == 0
|| strcmp(filter_name, "testgunzip") == 0) {
VFAIL(ctx,
"new %s: filter name %s already in use by another VFP",
vcl_name, filter_name);
return;
}
if (bufsz == 0) {
VFAIL(ctx, "new %s: buffer size must be > 0b", vcl_name);
return;
}
errno = 0;
ALLOC_OBJ(enc, VMOD_BROTLI_ENCODER_MAGIC);
if (enc == NULL) {
VFAIL(ctx, "new %s: cannot allocate space for object: %s",
vcl_name, strerror(errno));
return;
}
errno = 0;
vfp = malloc(sizeof(*vfp));
if (vfp == NULL) {
VFAIL(ctx, "new %s: cannot allocate space for VFP: %s",
vcl_name, strerror(errno));
return;
}
errno = 0;
ALLOC_OBJ(settings, VBR_SETTINGS_MAGIC);
if (vfp == NULL) {
VFAIL(ctx, "new %s: cannot allocate space for settings: %s",
vcl_name, strerror(errno));
return;
}
errno = 0;
ALLOC_OBJ(vfpe, CUSTOM_VFP_MAGIC);
if (vfpe == NULL) {
VFAIL(ctx, "new %s: cannot allocate space VFP entry: %s",
vcl_name, strerror(errno));
return;
}
settings->bufsz = bufsz;
settings->which = ENC;
vfp->name = strdup(filter_name);
vfp->init = vfp_br_init;
vfp->pull = vfp_br_pull;
vfp->fini = vfp_br_fini;
vfp->priv1 = settings;
enc->vfp = vfp;
VRT_AddVFP(ctx, vfp);
vfph = init_priv_vcl(priv);
vfpe->vfp = vfp;
VSLIST_INSERT_HEAD(vfph, vfpe, list);
}
/*
* The settings and custom VFP objects and PRIV_VCL list entry are freed
* on the DISCARD event, because we need a VRT_CTX to call
* VRT_RemoveVFP().
*/
VCL_VOID
vmod_encoder__fini(struct vmod_brotli_encoder **encp)
{
struct vmod_brotli_encoder *enc;
if (encp == NULL || *encp == NULL)
return;
CHECK_OBJ(*encp, VMOD_BROTLI_ENCODER_MAGIC);
enc = *encp;
*encp = NULL;
FREE_OBJ(enc);
}
/* Version functions */
static VCL_STRING
brotli_version(VRT_CTX, uint32_t version)
{
......
......@@ -66,7 +66,7 @@ algorithm for responses fetched from backends.
XXX ...
# $Object encoder(STRING name, BYTES buffer=32k, INT quality=11, INT lgwin=22)
$Object encoder(PRIV_VCL, STRING name, BYTES bufffer=32768)
XXX ...
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment