Skip to content

V
varnishevent3
Commit 5a2161e5 authored by Poul-Henning Kamp's avatar Poul-Henning Kamp
Browse files
Options

Rework ESI storage allocation. 

Previously we stored the esi-metadata in the object workspace, but since we
are trying to make that a snug fit and we cannot preestimate how much space
ESI parsing will need, this no longer works.

Instead parse the ESI metadata into the workers workspace and when
done, allocate a storage object and move it all into that.

Beware that this may increase the memory cost for ESI objects by the stevedores
granularity.

Fixes #578



git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@4351 d4fa192b-c00b-0410-8231-f00ffab90ce4
parent 197f6b02
Hide whitespace changes
Inline Side-by-side
Showing with 191 additions and 43 deletions
bin/varnishd/cache.h
View file @ 5a2161e5
......@@ -89,7 +89,7 @@ struct object;
struct objhead;
struct objcore;
struct workreq;
struct esi_bit;
struct esidata;
struct vrt_backend;
struct cli_proto;
struct ban;
......@@ -342,7 +342,7 @@ struct object {
VTAILQ_HEAD(, storage) store;
VTAILQ_HEAD(, esi_bit) esibits;
struct esidata *esidata;
double last_use;
......
bin/varnishd/cache_esi.c
View file @ 5a2161e5
......@@ -54,8 +54,7 @@ SVNID("$Id$")
#include "vrt.h"
#include "vcl.h"
#include "cache.h"
#define NDEFELEM 10
#include "stevedore.h"
/*--------------------------------------------------------------------*/
......@@ -65,11 +64,8 @@ struct esi_bit {
txt verbatim;
txt host;
txt include;
int free_this;
};
VTAILQ_HEAD(esibithead, esi_bit);
struct esi_ptr {
const char *p;
const char *e;
......@@ -87,14 +83,24 @@ struct esi_work {
txt t;
struct esi_bit *eb;
struct esi_bit *ebl; /* list of */
int neb;
int remflg; /* inside <esi:remove> </esi:remove> */
int incmt; /* inside <!--esi ... --> comment */
unsigned space; /* ... needed */
VTAILQ_HEAD(, esi_bit) esibits;
};
struct esidata {
unsigned magic;
#define ESIDATA_MAGIC 0x7255277f
VTAILQ_HEAD(, esi_bit) esibits;
struct storage *storage;
};
/*--------------------------------------------------------------------
* Move the parse-pointer forward.
* Move an esi_ptr one char forward
*/
static void
......@@ -118,6 +124,10 @@ Nep(struct esi_ptr *ep)
return;
}
/*--------------------------------------------------------------------
* Consume one input character.
*/
static void
N(struct esi_work *ew)
{
......@@ -209,18 +219,12 @@ static void
esi_addbit(struct esi_work *ew, const char *verbatim, unsigned len)
{
if (ew->neb == 0) {
ew->ebl = calloc(NDEFELEM, sizeof(struct esi_bit));
XXXAN(ew->ebl);
ew->neb = NDEFELEM;
ew->ebl->free_this = 1;
}
ew->eb = ew->ebl;
ew->ebl++;
ew->neb--;
ew->space += sizeof(*ew->eb);
ew->eb = (void*)WS_Alloc(ew->sp->wrk->ws, sizeof *ew->eb);
AN(ew->eb);
memset(ew->eb, 0, sizeof *ew->eb);
VTAILQ_INSERT_TAIL(&ew->sp->obj->esibits, ew->eb, list);
VTAILQ_INSERT_TAIL(&ew->esibits, ew->eb, list);
if (verbatim != NULL) {
ew->eb->verbatim.b = TRUST_ME(verbatim);
if (len > 0)
......@@ -231,8 +235,10 @@ esi_addbit(struct esi_work *ew, const char *verbatim, unsigned len)
Tlen(ew->eb->verbatim),
Tlen(ew->eb->verbatim),
ew->eb->verbatim.b);
} else
ew->eb->verbatim.b = ew->eb->verbatim.e = (void*)ew->eb;
} else {
AN(ew->s.p);
ew->eb->verbatim.b = ew->eb->verbatim.e = TRUST_ME(ew->s.p);
}
}
/*--------------------------------------------------------------------*/
......@@ -381,12 +387,12 @@ esi_handle_include(struct esi_work *ew)
if ( val.b != val.e ) {
s = Tlen(val) + 1;
c = WS_Alloc(ws, s);
c = WS_Alloc(ew->sp->wrk->ws, s);
XXXAN(c);
memcpy(c, val.b, Tlen(val));
val.b = c;
val.e = val.b + s;
*val.e = '\0';
val.e[-1] = '\0';
}
if (Tlen(val) > 7 && !memcmp(val.b, "http://", 7)) {
......@@ -428,7 +434,7 @@ esi_handle_include(struct esi_work *ew)
if (q != NULL)
tag.e = q + 1;
u = WS_Reserve(ws, 0);
u = WS_Reserve(ew->sp->wrk->ws, 0);
v = snprintf(ws->f, u - 1, "%.*s%.*s",
pdiff(tag.b, tag.e), tag.b,
pdiff(val.b, val.e), val.b);
......@@ -436,8 +442,12 @@ esi_handle_include(struct esi_work *ew)
xxxassert(v < u);
eb->include.b = ws->f;
eb->include.e = ws->f + v;
WS_Release(ws, v);
WS_Release(ew->sp->wrk->ws, v);
}
if (eb->include.b != NULL)
ew->space += Tlen(eb->include);
if (eb->host.b != NULL)
ew->space += Tlen(eb->host);
}
}
......@@ -654,6 +664,11 @@ void
ESI_Parse(struct sess *sp)
{
struct esi_work *ew, eww[1];
struct esi_bit *eb;
struct esidata *ed;
struct storage *st;
unsigned u;
char *hack;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
......@@ -680,13 +695,19 @@ ESI_Parse(struct sess *sp)
if (!contain_esi(sp->obj))
return;
/* XXX: debugging hack */
hack = sp->wrk->ws->f;
VSL_stats->esi_parse++;
/* XXX: only if GET ? */
ew = eww;
memset(eww, 0, sizeof eww);
VTAILQ_INIT(&ew->esibits);
ew->sp = sp;
ew->off = 1;
ew->space += sizeof(struct esidata);
ew->p.st = VTAILQ_FIRST(&sp->obj->store);
AN(ew->p.st);
ew->p.p = (char *)ew->p.st->ptr;
......@@ -751,6 +772,48 @@ ESI_Parse(struct sess *sp)
if (ew->incmt)
esi_error(ew, ew->t.e, -1,
"ESI 1.0 unterminated <!--esi comment");
st = STV_alloc(sp, ew->space, sp->obj->objcore);
AN(st);
assert(st->space >= ew->space);
ed = (void*)st->ptr;
st->len += sizeof(*ed);
memset(ed, 0, sizeof *ed);
ed->magic = ESIDATA_MAGIC;
ed->storage = st;
VTAILQ_INIT(&ed->esibits);
while (!VTAILQ_EMPTY(&ew->esibits)) {
eb = VTAILQ_FIRST(&ew->esibits);
VTAILQ_REMOVE(&ew->esibits, eb, list);
memcpy(st->ptr + st->len, eb, sizeof *eb);
eb = (void*)(st->ptr + st->len);
st->len += sizeof(*eb);
if (eb->include.b != NULL) {
u = Tlen(eb->include);
memcpy(st->ptr + st->len, eb->include.b, u);
eb->include.b = (void*)(st->ptr + st->len);
eb->include.e = eb->include.b + u;
st->len += u;
}
if (eb->host.b != NULL) {
u = Tlen(eb->host);
memcpy(st->ptr + st->len, eb->host.b, u);
eb->host.b = (void*)(st->ptr + st->len);
eb->host.e = eb->host.b + u;
st->len += u;
}
VTAILQ_INSERT_TAIL(&ed->esibits, eb, list);
}
assert(st->len < st->space);
assert(st->len == ew->space);
sp->obj->esidata = ed;
memset(hack, 0xaa, sp->wrk->ws->f - hack);
}
/*--------------------------------------------------------------------*/
......@@ -763,11 +826,14 @@ ESI_Deliver(struct sess *sp)
struct worker *w;
char *ws_wm;
struct http http_save;
struct esidata *ed;
w = sp->wrk;
WRW_Reserve(w, &sp->fd);
http_save.magic = 0;
VTAILQ_FOREACH(eb, &sp->obj->esibits, list) {
ed = sp->obj->esidata;
CHECK_OBJ_NOTNULL(ed, ESIDATA_MAGIC);
VTAILQ_FOREACH(eb, &ed->esibits, list) {
if (Tlen(eb->verbatim)) {
if (sp->http->protover >= 1.1)
(void)WRW_Write(w, eb->chunk_length, -1);
......@@ -857,16 +923,10 @@ ESI_Deliver(struct sess *sp)
void
ESI_Destroy(struct object *o)
{
struct esi_bit *eb;
struct esidata *ed;
/*
* Delete esi_bits from behind and free(3) the ones that want to be.
*/
while (!VTAILQ_EMPTY(&o->esibits)) {
eb = VTAILQ_LAST(&o->esibits, esibithead);
VTAILQ_REMOVE(&o->esibits, eb, list);
if (eb->free_this)
free(eb);
}
ed = o->esidata;
o->esidata = NULL;
CHECK_OBJ_NOTNULL(ed, ESIDATA_MAGIC);
STV_free(ed->storage);
}
bin/varnishd/cache_hash.c
View file @ 5a2161e5
......@@ -701,7 +701,8 @@ HSH_Deref(struct worker *w, struct object **oo)
DSL(0x40, SLT_Debug, 0, "Object %u workspace min free %u",
o->xid, WS_Free(o->ws_o));
ESI_Destroy(o);
if (o->esidata != NULL)
ESI_Destroy(o);
if (o->objcore != NULL && o->objcore->smp_seg != NULL) {
SMP_FreeObj(o);
} else {
......
bin/varnishd/cache_response.c
View file @ 5a2161e5
......@@ -139,7 +139,7 @@ RES_BuildHttp(struct sess *sp)
HTTPH_A_DELIVER);
/* Only HTTP 1.1 can do Chunked encoding */
if (!sp->disable_esi && !VTAILQ_EMPTY(&sp->obj->esibits)) {
if (!sp->disable_esi && sp->obj->esidata != NULL) {
http_Unset(sp->wrk->resp, H_Content_Length);
if(sp->http->protover >= 1.1)
http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Transfer-Encoding: chunked");
......@@ -178,7 +178,7 @@ RES_WriteObj(struct sess *sp)
if (sp->disable_esi || !sp->esis)
sp->acct_req.hdrbytes += http_Write(sp->wrk, sp->wrk->resp, 1);
if (!sp->disable_esi && sp->wantbody && !VTAILQ_EMPTY(&sp->obj->esibits)) {
if (!sp->disable_esi && sp->wantbody && sp->obj->esidata != NULL) {
if (WRW_FlushRelease(sp->wrk)) {
vca_close_session(sp, "remote closed");
return;
......
bin/varnishd/stevedore.c
View file @ 5a2161e5
......@@ -103,7 +103,6 @@ STV_InitObj(struct sess *sp, struct object *o, unsigned wsl)
o->grace = NAN;
o->entered = NAN;
VTAILQ_INIT(&o->store);
VTAILQ_INIT(&o->esibits);
sp->wrk->stats.n_object++;
}
......
bin/varnishtest/tests/e00017.vtc 0 → 100644
View file @ 5a2161e5
# $Id: e00003.vtc 3742 2009-02-11 07:53:47Z tfheen $
test "Aggressive use of ESI include"
server s1 {
rxreq
txresp -body {
<html>
Before include
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/00"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/01"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/02"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/03"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/04"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/05"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/06"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/07"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/08"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/09"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/10"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/11"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/12"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/13"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/14"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/15"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/16"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/17"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/18"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/19"/>
After include
}
rxreq
txresp -body { Included file 00 }
rxreq
txresp -body { Included file 01 }
rxreq
txresp -body { Included file 02 }
rxreq
txresp -body { Included file 03 }
rxreq
txresp -body { Included file 04 }
rxreq
txresp -body { Included file 05 }
rxreq
txresp -body { Included file 06 }
rxreq
txresp -body { Included file 07 }
rxreq
txresp -body { Included file 08 }
rxreq
txresp -body { Included file 09 }
rxreq
txresp -body { Included file 10 }
rxreq
txresp -body { Included file 11 }
rxreq
txresp -body { Included file 12 }
rxreq
txresp -body { Included file 13 }
rxreq
txresp -body { Included file 14 }
rxreq
txresp -body { Included file 15 }
rxreq
txresp -body { Included file 16 }
rxreq
txresp -body { Included file 17 }
rxreq
txresp -body { Included file 18 }
rxreq
txresp -body { Included file 19 }
} -start
varnish v1 -vcl+backend {
sub vcl_fetch {
esi;
}
} -start
client c1 {
txreq
rxresp
expect resp.status == 200
}
client c1 -run
varnish v1 -expect esi_errors == 0
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment