Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
V
varnish-cache
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Commits
Open sidebar
varnishcache
varnish-cache
Commits
35febe95
Commit
35febe95
authored
Jan 21, 2011
by
Poul-Henning Kamp
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Yah! Enable gzip-file stitching and pass the first trivial
ESI+GZIP testcase
parent
8a79da2a
Changes
7
Show whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
250 additions
and
34 deletions
+250
-34
cache.h
bin/varnishd/cache.h
+2
-1
cache_esi.h
bin/varnishd/cache_esi.h
+1
-0
cache_esi_deliver.c
bin/varnishd/cache_esi_deliver.c
+39
-9
cache_esi_fetch.c
bin/varnishd/cache_esi_fetch.c
+147
-14
cache_esi_parse.c
bin/varnishd/cache_esi_parse.c
+18
-8
cache_fetch.c
bin/varnishd/cache_fetch.c
+2
-2
e00021.vtc
bin/varnishtest/tests/e00021.vtc
+41
-0
No files found.
bin/varnishd/cache.h
View file @
35febe95
...
...
@@ -98,6 +98,7 @@ struct ban;
struct
SHA256Context
;
struct
vsc_lck
;
struct
waitinglist
;
struct
vef_priv
;
struct
lock
{
void
*
priv
;
};
// Opaque
...
...
@@ -280,7 +281,7 @@ struct worker {
struct
storage
*
storage
;
struct
vfp
*
vfp
;
struct
vgz
*
vgz_rx
;
struct
v
gz
*
vgz_esi
;
struct
v
ef_priv
*
vef_priv
;
unsigned
do_esi
;
unsigned
do_gzip
;
unsigned
is_gzip
;
...
...
bin/varnishd/cache_esi.h
View file @
35febe95
...
...
@@ -27,6 +27,7 @@
*
*/
#define VEC_GZ (0x21)
#define VEC_V1 (0x40 + 1)
#define VEC_V2 (0x40 + 2)
#define VEC_V8 (0x40 + 8)
...
...
bin/varnishd/cache_esi_deliver.c
View file @
35febe95
...
...
@@ -174,8 +174,10 @@ ESI_Deliver(struct sess *sp)
struct
storage
*
st
;
uint8_t
*
p
,
*
e
,
*
q
,
*
r
;
unsigned
off
;
ssize_t
l
,
l_crc
;
uint32_t
crc
,
crc_ref
;
ssize_t
l
,
l_icrc
,
l_crc
;
uint32_t
crc
,
icrc
;
uint8_t
tailbuf
[
8
+
5
];
int
dogzip
;
CHECK_OBJ_NOTNULL
(
sp
,
SESS_MAGIC
);
st
=
sp
->
obj
->
esidata
;
...
...
@@ -183,6 +185,14 @@ ESI_Deliver(struct sess *sp)
p
=
st
->
ptr
;
e
=
st
->
ptr
+
st
->
len
;
if
(
*
p
==
VEC_GZ
)
{
p
++
;
dogzip
=
1
;
crc
=
crc32
(
0L
,
Z_NULL
,
0
);
l_crc
=
0
;
}
else
dogzip
=
0
;
st
=
VTAILQ_FIRST
(
&
sp
->
obj
->
store
);
off
=
0
;
...
...
@@ -192,15 +202,19 @@ ESI_Deliver(struct sess *sp)
case
VEC_V2
:
case
VEC_V8
:
l
=
ved_decode_len
(
&
p
);
assert
(
*
p
==
VEC_C1
||
*
p
==
VEC_C2
||
*
p
==
VEC_C8
);
l_crc
=
ved_decode_len
(
&
p
);
crc
=
vbe32dec
(
p
);
if
(
dogzip
)
{
assert
(
*
p
==
VEC_C1
||
*
p
==
VEC_C2
||
*
p
==
VEC_C8
);
l_icrc
=
ved_decode_len
(
&
p
);
icrc
=
vbe32dec
(
p
);
p
+=
4
;
}
q
=
(
void
*
)
strchr
((
const
char
*
)
p
,
'\0'
);
assert
(
q
>
p
);
crc_ref
=
crc32
(
0L
,
Z_NULL
,
0
);
crc_ref
=
crc32
(
crc_ref
,
st
->
ptr
+
off
,
l
);
xxxassert
(
crc_ref
==
crc
);
if
(
dogzip
)
{
crc
=
crc32_combine
(
crc
,
icrc
,
l_icrc
);
l_crc
+=
l_icrc
;
}
ved_sendchunk
(
sp
,
p
,
q
-
p
,
st
->
ptr
+
off
,
l
);
off
+=
l
;
p
=
q
+
1
;
...
...
@@ -229,6 +243,22 @@ ESI_Deliver(struct sess *sp)
INCOMPL
();
}
}
if
(
dogzip
)
{
/* Emit a gzip literal block with finish bit set */
tailbuf
[
0
]
=
0x01
;
tailbuf
[
1
]
=
0x00
;
tailbuf
[
2
]
=
0x00
;
tailbuf
[
3
]
=
0xff
;
tailbuf
[
4
]
=
0xff
;
/* Emit CRC32 */
vle32enc
(
tailbuf
+
5
,
crc
);
/* MOD(2^32) length */
vle32enc
(
tailbuf
+
9
,
l_crc
);
ved_sendchunk
(
sp
,
"d
\r\n
"
,
3
,
tailbuf
,
13
);
}
(
void
)
WRW_Flush
(
sp
->
wrk
);
}
bin/varnishd/cache_esi_fetch.c
View file @
35febe95
...
...
@@ -25,7 +25,7 @@
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* VE
P Varnish Esi Pars
ing
* VE
F Varnish Esi Fetch
ing
*/
#include "config.h"
...
...
@@ -94,16 +94,6 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
return
(
1
);
}
/*---------------------------------------------------------------------
* We receive a ungzip'ed object, and want to store it gzip'ed.
*/
static
int
__match_proto__
()
vfp_esi_bytes_ug
(
struct
sess
*
sp
,
struct
http_conn
*
htc
,
size_t
bytes
)
{
return
(
vfp_esi_bytes_uu
(
sp
,
htc
,
bytes
));
}
/*---------------------------------------------------------------------
* We receive a gzip'ed object, and want to store it ungzip'ed.
*/
...
...
@@ -159,6 +149,123 @@ vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
return
(
1
);
}
/*---------------------------------------------------------------------
*/
struct
vef_priv
{
unsigned
magic
;
#define VEF_MAGIC 0xf104b51f
struct
vgz
*
vgz
;
char
*
bufp
;
ssize_t
tot
;
int
error
;
};
/*---------------------------------------------------------------------
* We receive a ungzip'ed object, and want to store it gzip'ed.
*/
#include "vend.h"
static
ssize_t
vfp_vep_callback
(
const
struct
sess
*
sp
,
ssize_t
l
,
enum
vgz_flag
flg
)
{
struct
vef_priv
*
vef
;
struct
storage
*
st
;
size_t
dl
;
const
void
*
dp
;
int
i
;
char
*
p
;
printf
(
"ZCB(%jd, %d)
\n
"
,
l
,
flg
);
CHECK_OBJ_NOTNULL
(
sp
,
SESS_MAGIC
);
vef
=
sp
->
wrk
->
vef_priv
;
CHECK_OBJ_NOTNULL
(
vef
,
VEF_MAGIC
);
assert
(
l
>=
0
);
if
(
vef
->
error
)
{
vef
->
tot
+=
l
;
return
(
vef
->
tot
);
}
VGZ_Ibuf
(
vef
->
vgz
,
vef
->
bufp
,
l
);
do
{
if
(
sp
->
wrk
->
storage
==
NULL
)
{
l
=
params
->
fetch_chunksize
*
1024LL
;
sp
->
wrk
->
storage
=
STV_alloc
(
sp
,
l
);
}
if
(
sp
->
wrk
->
storage
==
NULL
)
{
vef
->
error
=
ENOMEM
;
vef
->
tot
+=
l
;
return
(
vef
->
tot
);
}
st
=
sp
->
wrk
->
storage
;
VGZ_Obuf
(
vef
->
vgz
,
st
->
ptr
+
st
->
len
,
st
->
space
-
st
->
len
);
p
=
(
void
*
)(
st
->
ptr
+
st
->
len
);
i
=
VGZ_Gzip
(
vef
->
vgz
,
&
dp
,
&
dl
,
flg
);
printf
(
"GZI = %d %jd
\n
"
,
i
,
dl
);
vef
->
tot
+=
dl
;
st
->
len
+=
dl
;
if
(
st
->
len
==
st
->
space
)
{
VTAILQ_INSERT_TAIL
(
&
sp
->
obj
->
store
,
sp
->
wrk
->
storage
,
list
);
sp
->
wrk
->
storage
=
NULL
;
st
=
NULL
;
}
}
while
(
!
VGZ_IbufEmpty
(
vef
->
vgz
));
vef
->
bufp
+=
l
;
if
(
flg
==
VGZ_FINISH
)
assert
(
i
==
1
);
/* XXX */
else
assert
(
i
==
0
);
/* XXX */
printf
(
"ZCB = %jd
\n
"
,
vef
->
tot
);
fflush
(
stdout
);
usleep
(
100
);
return
(
vef
->
tot
);
}
static
int
__match_proto__
()
vfp_esi_bytes_ug
(
struct
sess
*
sp
,
struct
http_conn
*
htc
,
size_t
bytes
)
{
ssize_t
l
,
w
;
char
ibuf
[
1024
*
params
->
gzip_stack_buffer
];
struct
vef_priv
*
vef
;
CHECK_OBJ_NOTNULL
(
sp
,
SESS_MAGIC
);
vef
=
sp
->
wrk
->
vef_priv
;
CHECK_OBJ_NOTNULL
(
vef
,
VEF_MAGIC
);
while
(
bytes
>
0
)
{
if
(
sp
->
wrk
->
storage
==
NULL
)
{
l
=
params
->
fetch_chunksize
*
1024LL
;
sp
->
wrk
->
storage
=
STV_alloc
(
sp
,
l
);
}
if
(
sp
->
wrk
->
storage
==
NULL
)
{
errno
=
ENOMEM
;
return
(
-
1
);
}
l
=
sizeof
ibuf
;
if
(
l
>
bytes
)
l
=
bytes
;
w
=
HTC_Read
(
htc
,
ibuf
,
l
);
if
(
w
<=
0
)
{
printf
(
"RT %jd
\n
"
,
w
);
return
(
w
);
}
bytes
-=
w
;
vef
->
bufp
=
ibuf
;
VEP_parse
(
sp
,
ibuf
,
l
);
if
(
vef
->
error
)
{
// errno = vef->error;
return
(
-
1
);
}
}
return
(
1
);
}
/*---------------------------------------------------------------------
* We receive a gzip'ed object, and want to store it gzip'ed.
*/
...
...
@@ -175,14 +282,26 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
static
void
__match_proto__
()
vfp_esi_begin
(
struct
sess
*
sp
,
size_t
estimate
)
{
struct
vef_priv
*
vef
;
CHECK_OBJ_NOTNULL
(
sp
,
SESS_MAGIC
);
/* XXX: snapshot WS's ? We'll need the space */
VEP_Init
(
sp
,
NULL
);
if
(
sp
->
wrk
->
is_gzip
&&
sp
->
wrk
->
do_gunzip
)
if
(
sp
->
wrk
->
is_gzip
&&
sp
->
wrk
->
do_gunzip
)
{
sp
->
wrk
->
vgz_rx
=
VGZ_NewUngzip
(
sp
,
sp
->
ws
);
VEP_Init
(
sp
,
NULL
);
}
else
if
(
sp
->
wrk
->
is_gunzip
&&
sp
->
wrk
->
do_gzip
)
{
VEP_Init
(
sp
,
vfp_vep_callback
);
vef
=
(
void
*
)
WS_Alloc
(
sp
->
ws
,
sizeof
*
vef
);
AN
(
vef
);
memset
(
vef
,
0
,
sizeof
*
vef
);
vef
->
magic
=
VEF_MAGIC
;
vef
->
vgz
=
VGZ_NewGzip
(
sp
,
sp
->
ws
);
AZ
(
sp
->
wrk
->
vef_priv
);
sp
->
wrk
->
vef_priv
=
vef
;
}
else
{
VEP_Init
(
sp
,
NULL
);
}
(
void
)
estimate
;
}
...
...
@@ -193,6 +312,7 @@ vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
int
i
;
CHECK_OBJ_NOTNULL
(
sp
,
SESS_MAGIC
);
printf
(
"BYTES = %jd
\n
"
,
bytes
);
if
(
sp
->
wrk
->
is_gzip
&&
sp
->
wrk
->
do_gunzip
)
i
=
vfp_esi_bytes_gu
(
sp
,
htc
,
bytes
);
else
if
(
sp
->
wrk
->
is_gunzip
&&
sp
->
wrk
->
do_gzip
)
...
...
@@ -201,6 +321,7 @@ vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
i
=
vfp_esi_bytes_gg
(
sp
,
htc
,
bytes
);
else
i
=
vfp_esi_bytes_uu
(
sp
,
htc
,
bytes
);
printf
(
"BYTES = %d
\n
"
,
i
);
return
(
i
);
}
...
...
@@ -209,8 +330,10 @@ vfp_esi_end(struct sess *sp)
{
struct
storage
*
st
;
struct
vsb
*
vsb
;
struct
vef_priv
*
vef
;
ssize_t
l
;
printf
(
"END
\n
"
);
CHECK_OBJ_NOTNULL
(
sp
,
SESS_MAGIC
);
if
(
sp
->
wrk
->
is_gzip
&&
sp
->
wrk
->
do_gunzip
)
VGZ_Destroy
(
&
sp
->
wrk
->
vgz_rx
);
...
...
@@ -230,6 +353,16 @@ vfp_esi_end(struct sess *sp)
st
=
sp
->
wrk
->
storage
;
sp
->
wrk
->
storage
=
NULL
;
if
(
sp
->
wrk
->
vef_priv
!=
NULL
)
{
vef
=
sp
->
wrk
->
vef_priv
;
sp
->
wrk
->
vef_priv
=
NULL
;
CHECK_OBJ_NOTNULL
(
vef
,
VEF_MAGIC
);
XXXAZ
(
vef
->
error
);
printf
(
"TOT %jd
\n
"
,
vef
->
tot
);
sp
->
obj
->
len
=
vef
->
tot
;
}
if
(
st
==
NULL
)
return
(
0
);
...
...
bin/varnishd/cache_esi_parse.c
View file @
35febe95
...
...
@@ -64,6 +64,7 @@ struct vep_state {
struct
vsb
*
vsb
;
const
struct
sess
*
sp
;
int
dogzip
;
vep_callback_t
*
cb
;
/* Internal Counter for default call-back function */
...
...
@@ -285,9 +286,12 @@ vep_emit_verbatim(const struct vep_state *vep, ssize_t l, ssize_t l_crc)
Debug
(
"---> VERBATIM(%jd)
\n
"
,
(
intmax_t
)
l
);
}
vep_emit_len
(
vep
,
l
,
VEC_V1
,
VEC_V2
,
VEC_V8
);
if
(
vep
->
dogzip
)
{
vep_emit_len
(
vep
,
l_crc
,
VEC_C1
,
VEC_C2
,
VEC_C8
);
vbe32enc
(
buf
,
vep
->
crc
);
vsb_bcat
(
vep
->
vsb
,
buf
,
sizeof
buf
);
}
/* Emit Chunked header */
vsb_printf
(
vep
->
vsb
,
"%lx
\r\n
%c"
,
l
,
0
);
}
...
...
@@ -992,14 +996,19 @@ VEP_Init(const struct sess *sp, vep_callback_t *cb)
memset
(
vep
,
0
,
sizeof
*
vep
);
vep
->
magic
=
VEP_MAGIC
;
vep
->
sp
=
sp
;
if
(
cb
!=
NULL
)
vep
->
vsb
=
vsb_newauto
();
AN
(
vep
->
vsb
);
if
(
cb
!=
NULL
)
{
vep
->
dogzip
=
1
;
/* XXX */
vsb_printf
(
vep
->
vsb
,
"%c"
,
VEC_GZ
);
vep
->
cb
=
cb
;
else
}
else
{
vep
->
cb
=
vep_default_cb
;
}
vep
->
state
=
VEP_START
;
vep
->
vsb
=
vsb_newauto
();
AN
(
vep
->
vsb
);
vep
->
crc
=
crc32
(
0L
,
Z_NULL
,
0
);
vep
->
crcp
=
crc32
(
0L
,
Z_NULL
,
0
);
}
...
...
@@ -1020,8 +1029,9 @@ VEP_Finish(const struct sess *sp)
if
(
vep
->
o_pending
)
vep_mark_common
(
vep
,
vep
->
ver_p
,
vep
->
last_mark
);
if
(
vep
->
o_wait
>
0
)
{
lcb
=
vep
->
cb
(
vep
->
sp
,
0
,
VGZ_
FINISH
);
lcb
=
vep
->
cb
(
vep
->
sp
,
0
,
VGZ_
ALIGN
);
vep_emit_common
(
vep
,
lcb
-
vep
->
o_last
,
vep
->
last_mark
);
(
void
)
vep
->
cb
(
vep
->
sp
,
0
,
VGZ_FINISH
);
}
sp
->
wrk
->
vep
=
NULL
;
...
...
bin/varnishd/cache_fetch.c
View file @
35febe95
...
...
@@ -204,8 +204,8 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
i
=
sp
->
wrk
->
vfp
->
bytes
(
sp
,
htc
,
cl
);
if
(
i
<=
0
)
{
WSP
(
sp
,
SLT_FetchError
,
"straight read_error: %d (%s)"
,
errno
,
strerror
(
errno
));
WSP
(
sp
,
SLT_FetchError
,
"straight read_error: %d
%d
(%s)"
,
i
,
errno
,
strerror
(
errno
));
return
(
-
1
);
}
return
(
0
);
...
...
bin/varnishtest/tests/e00021.vtc
0 → 100644
View file @
35febe95
# $Id$
test "ESI ability to stitch gzip files together"
server s1 {
rxreq
txresp -body {
<esi:remove>
This is a test: Unseen University
<esi:include src="trick question">
<!--esi XXX -->
</esi:remove>
<esX>This is a test: Hello world
}
} -start
varnish v1 -vcl+backend {
sub vcl_recv {
set req.esi = true;
}
sub vcl_fetch {
set beresp.do_esi = true;
set beresp.do_gzip = true;
}
} -start
varnish v1 -cliok "param.set esi_syntax 4"
varnish v1 -cliok "param.set http_gzip_support true"
client c1 {
txreq -hdr "Accept-Encoding: gzip"
rxresp
expect resp.http.content-encoding == gzip
gunzip
expect resp.status == 200
expect resp.bodylen == 40
}
client c1 -run
varnish v1 -expect esi_errors == 2
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment