Skip to content

U
unique-xids
Commit 70f943a4 authored by Poul-Henning Kamp's avatar Poul-Henning Kamp
Browse files
Options

Replay of SVN r5772 

Futher modelling of VGZ functions
parent 13a49a4d
Show whitespace changes
Inline Side-by-side
Showing with 82 additions and 59 deletions
bin/varnishd/cache.h
View file @ 70f943a4
...@@ -625,10 +625,11 @@ void Fetch_Init(void); ...@@ -625,10 +625,11 @@ void Fetch_Init(void);
/* cache_gzip.c */ /* cache_gzip.c */
struct vgz; struct vgz;
struct vgz *VGZ_NewUnzip(const struct sess *sp, struct ws *tmp, struct vgz *VGZ_NewUngzip(const struct sess *sp, struct ws *tmp);
struct ws *buf_ws, void *buf, ssize_t bufl); void VGZ_Ibuf(struct vgz *, const void *, size_t len);
int VGZ_Feed(struct vgz *, const void *, size_t len); void VGZ_Obuf(struct vgz *, const void *, size_t len);
int VGZ_Produce(struct vgz *, const void **, size_t *len); int VGZ_Gzip(struct vgz *, const void **, size_t *len, int flag);
int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
void VGZ_Destroy(struct vgz **); void VGZ_Destroy(struct vgz **);
/* cache_http.c */ /* cache_http.c */
......
bin/varnishd/cache_gzip.c
View file @ 70f943a4
...@@ -81,9 +81,7 @@ struct vgz { ...@@ -81,9 +81,7 @@ struct vgz {
struct ws *tmp; struct ws *tmp;
char *tmp_snapshot; char *tmp_snapshot;
struct ws *buf_ws; void *before;
void *buf;
size_t bufsiz;
z_stream vz; z_stream vz;
}; };
...@@ -114,7 +112,7 @@ vgz_free(voidpf opaque, voidpf address) ...@@ -114,7 +112,7 @@ vgz_free(voidpf opaque, voidpf address)
*/ */
static struct vgz * static struct vgz *
vgz_alloc_vgz(struct ws *ws, struct ws *buf_ws, void *buf, ssize_t bufl) vgz_alloc_vgz(struct ws *ws)
{ {
char *s; char *s;
struct vgz *vg; struct vgz *vg;
...@@ -132,29 +130,16 @@ vgz_alloc_vgz(struct ws *ws, struct ws *buf_ws, void *buf, ssize_t bufl) ...@@ -132,29 +130,16 @@ vgz_alloc_vgz(struct ws *ws, struct ws *buf_ws, void *buf, ssize_t bufl)
vg->vz.zfree = vgz_free; vg->vz.zfree = vgz_free;
vg->vz.opaque = vg; vg->vz.opaque = vg;
assert(buf_ws == NULL || buf == NULL);
if (buf_ws != NULL) {
WS_Assert(buf_ws);
vg->buf_ws = buf_ws;
vg->bufsiz = WS_Reserve(buf_ws, 0);
vg->buf = buf_ws->f;
} else {
assert(bufl > 0);
vg->buf = buf;
vg->bufsiz = bufl;
}
return (vg); return (vg);
} }
struct vgz * struct vgz *
VGZ_NewUnzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws, VGZ_NewUngzip(const struct sess *sp, struct ws *tmp)
void *buf, ssize_t bufl)
{ {
struct vgz *vg; struct vgz *vg;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
vg = vgz_alloc_vgz(tmp, buf_ws, buf, bufl); vg = vgz_alloc_vgz(tmp);
/* /*
* Max memory usage according to zonf.h: * Max memory usage according to zonf.h:
...@@ -167,14 +152,13 @@ VGZ_NewUnzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws, ...@@ -167,14 +152,13 @@ VGZ_NewUnzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws,
} }
static struct vgz * static struct vgz *
VGZ_NewGzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws, VGZ_NewGzip(const struct sess *sp, struct ws *tmp)
void *buf, ssize_t bufl)
{ {
struct vgz *vg; struct vgz *vg;
int i; int i;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
vg = vgz_alloc_vgz(tmp, buf_ws, buf, bufl); vg = vgz_alloc_vgz(tmp);
/* /*
* From zconf.h: * From zconf.h:
...@@ -202,8 +186,8 @@ VGZ_NewGzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws, ...@@ -202,8 +186,8 @@ VGZ_NewGzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws,
/*--------------------------------------------------------------------*/ /*--------------------------------------------------------------------*/
int void
VGZ_Feed(struct vgz *vg, const void *ptr, size_t len) VGZ_Ibuf(struct vgz *vg, const void *ptr, size_t len)
{ {
CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
...@@ -211,14 +195,24 @@ VGZ_Feed(struct vgz *vg, const void *ptr, size_t len) ...@@ -211,14 +195,24 @@ VGZ_Feed(struct vgz *vg, const void *ptr, size_t len)
AZ(vg->vz.avail_in); AZ(vg->vz.avail_in);
vg->vz.next_in = TRUST_ME(ptr); vg->vz.next_in = TRUST_ME(ptr);
vg->vz.avail_in = len; vg->vz.avail_in = len;
}
return (0); /*--------------------------------------------------------------------*/
void
VGZ_Obuf(struct vgz *vg, const void *ptr, size_t len)
{
CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
vg->vz.next_out = TRUST_ME(ptr);
vg->vz.avail_out = len;
} }
/*--------------------------------------------------------------------*/ /*--------------------------------------------------------------------*/
int int
VGZ_Produce(struct vgz *vg, const void **pptr, size_t *plen) VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen)
{ {
int i; int i;
...@@ -226,13 +220,41 @@ VGZ_Produce(struct vgz *vg, const void **pptr, size_t *plen) ...@@ -226,13 +220,41 @@ VGZ_Produce(struct vgz *vg, const void **pptr, size_t *plen)
*pptr = NULL; *pptr = NULL;
*plen = 0; *plen = 0;
vg->vz.next_out = vg->buf; AN(vg->vz.next_out);
vg->vz.avail_out = vg->bufsiz; AN(vg->vz.avail_out);
vg->before = vg->vz.next_out;
i = inflate(&vg->vz, 0); i = inflate(&vg->vz, 0);
if (i == Z_OK || i == Z_STREAM_END) { if (i == Z_OK || i == Z_STREAM_END) {
*pptr = vg->buf; *pptr = vg->before;
*plen = vg->bufsiz - vg->vz.avail_out; *plen = (const uint8_t *)vg->vz.next_out - (const uint8_t*)vg->before;
}
if (i == Z_OK)
return (0);
if (i == Z_STREAM_END)
return (1);
if (i == Z_BUF_ERROR)
return (2);
return (-1);
}
/*--------------------------------------------------------------------*/
int
VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, int flags)
{
int i;
CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
*pptr = NULL;
*plen = 0;
AN(vg->vz.next_out);
AN(vg->vz.avail_out);
vg->before = vg->vz.next_out;
i = deflate(&vg->vz, flags);
if (i == Z_OK || i == Z_STREAM_END) {
*pptr = vg->before;
*plen = (const uint8_t *)vg->vz.next_out - (const uint8_t*)vg->before;
} }
if (i == Z_OK) if (i == Z_OK)
return (0); return (0);
...@@ -250,8 +272,6 @@ VGZ_Destroy(struct vgz **vg) ...@@ -250,8 +272,6 @@ VGZ_Destroy(struct vgz **vg)
{ {
CHECK_OBJ_NOTNULL(*vg, VGZ_MAGIC); CHECK_OBJ_NOTNULL(*vg, VGZ_MAGIC);
if ((*vg)->buf_ws != NULL)
WS_Release((*vg)->buf_ws, 0);
WS_Reset((*vg)->tmp, (*vg)->tmp_snapshot); WS_Reset((*vg)->tmp, (*vg)->tmp_snapshot);
*vg = NULL; *vg = NULL;
} }
...@@ -266,7 +286,7 @@ static void __match_proto__() ...@@ -266,7 +286,7 @@ static void __match_proto__()
vfp_gunzip_begin(struct sess *sp, size_t estimate) vfp_gunzip_begin(struct sess *sp, size_t estimate)
{ {
(void)estimate; (void)estimate;
sp->wrk->vfp_private = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws, NULL, 0); sp->wrk->vfp_private = VGZ_NewUngzip(sp, sp->ws);
} }
static int __match_proto__() static int __match_proto__()
...@@ -276,6 +296,9 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes) ...@@ -276,6 +296,9 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
struct storage *st; struct storage *st;
ssize_t l, w; ssize_t l, w;
int i = -100; int i = -100;
uint8_t ibuf[64*1024]; /* XXX size ? */
size_t dl;
const void *dp;
CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC); CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
AZ(vg->vz.avail_in); AZ(vg->vz.avail_in);
...@@ -289,24 +312,22 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes) ...@@ -289,24 +312,22 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
} }
st = sp->wrk->storage; st = sp->wrk->storage;
vg->vz.next_out = st->ptr + st->len; VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
vg->vz.avail_out = st->space - st->len;
if (vg->vz.avail_in == 0 && bytes > 0) { if (vg->vz.avail_in == 0 && bytes > 0) {
l = vg->bufsiz; l = sizeof ibuf;
if (l > bytes) if (l > bytes)
l = bytes; l = bytes;
w = HTC_Read(htc, vg->buf, l); w = HTC_Read(htc, ibuf, l);
if (w <= 0) if (w <= 0)
return (w); return (w);
vg->vz.next_in = vg->buf; VGZ_Ibuf(vg, ibuf, w);
vg->vz.avail_in = w;
bytes -= w; bytes -= w;
} }
i = inflate(&vg->vz, 0); i = VGZ_Gunzip(vg, &dp, &dl);
assert(i == Z_OK || i == Z_STREAM_END); assert(i == Z_OK || i == Z_STREAM_END);
st->len = st->space - vg->vz.avail_out; st->len += dl;
if (st->len == st->space) { if (st->len == st->space) {
VTAILQ_INSERT_TAIL(&sp->obj->store, VTAILQ_INSERT_TAIL(&sp->obj->store,
sp->wrk->storage, list); sp->wrk->storage, list);
...@@ -363,7 +384,7 @@ vfp_gzip_begin(struct sess *sp, size_t estimate) ...@@ -363,7 +384,7 @@ vfp_gzip_begin(struct sess *sp, size_t estimate)
struct vgz *vg; struct vgz *vg;
(void)estimate; (void)estimate;
vg = VGZ_NewGzip(sp, sp->ws, sp->wrk->ws, NULL, 0); vg = VGZ_NewGzip(sp, sp->ws);
sp->wrk->vfp_private = vg; sp->wrk->vfp_private = vg;
} }
...@@ -374,6 +395,9 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes) ...@@ -374,6 +395,9 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
struct storage *st; struct storage *st;
ssize_t l, w; ssize_t l, w;
int i = -100; int i = -100;
uint8_t ibuf[64*1024]; /* XXX size ? */
size_t dl;
const void *dp;
CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC); CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
AZ(vg->vz.avail_in); AZ(vg->vz.avail_in);
...@@ -387,24 +411,22 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes) ...@@ -387,24 +411,22 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
} }
st = sp->wrk->storage; st = sp->wrk->storage;
vg->vz.next_out = st->ptr + st->len; VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
vg->vz.avail_out = st->space - st->len;
if (vg->vz.avail_in == 0 && bytes > 0) { if (vg->vz.avail_in == 0 && bytes > 0) {
l = vg->bufsiz; l = sizeof ibuf;
if (l > bytes) if (l > bytes)
l = bytes; l = bytes;
w = HTC_Read(htc, vg->buf, l); w = HTC_Read(htc, ibuf, l);
if (w <= 0) if (w <= 0)
return (w); return (w);
vg->vz.next_in = vg->buf; VGZ_Ibuf(vg, ibuf, w);
vg->vz.avail_in = w;
bytes -= w; bytes -= w;
} }
i = deflate(&vg->vz, bytes == 0 ? Z_FINISH : 0); i = VGZ_Gzip(vg, &dp, &dl, bytes == 0 ? Z_FINISH : 0);
assert(i == Z_OK || i == Z_STREAM_END); assert(i == Z_OK || i == Z_STREAM_END);
st->len = st->space - vg->vz.avail_out; st->len = st->space - dl;
if (st->len == st->space) { if (st->len == st->space) {
VTAILQ_INSERT_TAIL(&sp->obj->store, VTAILQ_INSERT_TAIL(&sp->obj->store,
sp->wrk->storage, list); sp->wrk->storage, list);
......
bin/varnishd/cache_response.c
View file @ 70f943a4
...@@ -253,8 +253,7 @@ res_WriteGunzipObj(struct sess *sp) ...@@ -253,8 +253,7 @@ res_WriteGunzipObj(struct sess *sp)
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
vg = VGZ_NewUnzip(sp, sp->wrk->ws, NULL, obuf, sizeof obuf); vg = VGZ_NewUngzip(sp, sp->wrk->ws);
AN(vg);
VTAILQ_FOREACH(st, &sp->obj->store, list) { VTAILQ_FOREACH(st, &sp->obj->store, list) {
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
...@@ -264,9 +263,10 @@ res_WriteGunzipObj(struct sess *sp) ...@@ -264,9 +263,10 @@ res_WriteGunzipObj(struct sess *sp)
sp->acct_tmp.bodybytes += st->len; /* XXX ? */ sp->acct_tmp.bodybytes += st->len; /* XXX ? */
VSC_main->n_objwrite++; VSC_main->n_objwrite++;
VGZ_Feed(vg, st->ptr, st->len); VGZ_Ibuf(vg, st->ptr, st->len);
do { do {
i = VGZ_Produce(vg, &dp, &dl); VGZ_Obuf(vg, obuf, sizeof obuf);
i = VGZ_Gunzip(vg, &dp, &dl);
if (dl != 0) { if (dl != 0) {
if (sp->wrk->res_mode & RES_CHUNKED) { if (sp->wrk->res_mode & RES_CHUNKED) {
bprintf(lenbuf, "%x\r\n", (unsigned)dl); bprintf(lenbuf, "%x\r\n", (unsigned)dl);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment