Skip to content

L
libvdp-pesi
Commit bcd8e84b authored by Geoff Simmons's avatar Geoff Simmons
Browse files
Options

Add the PESI.no_thread counter, and a corresponding log error message.

parent 094feed0
Hide whitespace changes
Inline Side-by-side
Showing with 485 additions and 8 deletions
.gitignore
View file @ bcd8e84b
......@@ -32,6 +32,8 @@ Makefile.in
/src/vcc_if.c
/src/vcc_if.h
/src/vmod_*rst
/src/VSC_pesi.c
/src/VSC_pesi.h
/src/tests/*.log
/src/tests/*.trs
......
configure.ac
View file @ bcd8e84b
......@@ -39,6 +39,7 @@ m4_ifndef([VARNISH_PREREQ], AC_MSG_ERROR([Need varnish.m4 -- see README.rst]))
VARNISH_PREREQ([trunk])
VARNISH_VMODS([pesi])
VARNISH_COUNTERS([pesi])
VMOD_TESTS="$(cd $srcdir/src && echo tests/*.vtc)"
AC_SUBST(VMOD_TESTS)
......
src/Makefile.am
View file @ bcd8e84b
......@@ -16,11 +16,15 @@ libvmod_pesi_la_SOURCES = \
nodist_libvmod_pesi_la_SOURCES = \
vcc_if.c \
vcc_if.h
vcc_if.h \
VSC_pesi.c \
VSC_pesi.h
dist_man_MANS = vdp_pesi.3
vdp_pesi.lo: vcc_if.h
@BUILD_VSC_PESI@
vdp_pesi.lo: vcc_if.h VSC_pesi.c VSC_pesi.h
vcc_if.h vmod_pesi.rst vmod_pesi.man.rst: vcc_if.c
......@@ -45,6 +49,7 @@ TESTS = @VMOD_TESTS@
EXTRA_DIST = \
vdp_pesi.vcc \
selector.vsc \
$(VMOD_TESTS)
CLEANFILES = \
......@@ -52,4 +57,6 @@ CLEANFILES = \
$(builddir)/vcc_if.h \
$(builddir)/vmod_pesi.rst \
$(builddir)/vmod_pesi.man.rst \
$(builddir)/vdp_pesi.3
$(builddir)/vdp_pesi.3 \
$(builddir)/VSC_pesi.c \
$(builddir)/VSC_pesi.h
src/pesi.vsc 0 → 100644
View file @ bcd8e84b
..
This is *NOT* a RST file but the syntax has been chosen so
that it may become an RST file at some later date.
.. varnish_vsc_begin:: pesi
:oneliner: VDP pesi stats
:order: 90
.. varnish_vsc:: no_thread
:type: counter
:oneliner: No threads available for parallel ESI
Number of times no threads were available for parallel ESI.
.. varnish_vsc_end:: pesi
src/tests/00basic.vtc
View file @ bcd8e84b
......@@ -12,6 +12,13 @@ varnish v1 -expect MEMPOOL.pesi.pool > 0
varnish v1 -expect MEMPOOL.pesi.sz_wanted > 0
varnish v1 -expect MEMPOOL.pesi.sz_actual > 0
varnish v1 -vsc PESI.*
varnish v1 -expect PESI.no_thread == 0
varnish v1 -vsc LCK.pesi.*
varnish v1 -expect LCK.pesi.stats.creat == 1
varnish v1 -expect LCK.pesi.stats.locks == 0
varnish v1 -vcl {backend b { .host = "${bad_ip}"; }}
varnish v1 -vsc MEMPOOL.pesi.*
......@@ -42,6 +49,9 @@ varnish v1 -expect MEMPOOL.pesi.pool > 0
varnish v1 -expect MEMPOOL.pesi.sz_wanted > 0
varnish v1 -expect MEMPOOL.pesi.sz_actual > 0
varnish v1 -vsc PESI.*
varnish v1 -expect PESI.no_thread == 0
varnish v1 -cli "vcl.state vcl2 warm"
varnish v1 -vsc MEMPOOL.pesi.*
......@@ -49,6 +59,13 @@ varnish v1 -expect MEMPOOL.pesi.pool > 0
varnish v1 -expect MEMPOOL.pesi.sz_wanted > 0
varnish v1 -expect MEMPOOL.pesi.sz_actual > 0
varnish v1 -vsc PESI.*
varnish v1 -expect PESI.no_thread == 0
varnish v1 -vsc LCK.pesi.*
varnish v1 -expect LCK.pesi.stats.creat == 1
varnish v1 -expect LCK.pesi.stats.locks == 0
varnish v1 -cli "vcl.show vcl1"
varnish v1 -cli "vcl.use vcl2"
varnish v1 -cli "vcl.discard vcl1"
......
src/tests/no_thread.vtc 0 → 100644
View file @ bcd8e84b
varnishtest "thread pool exhaustion"
## e00017.vtc
server s1 {
rxreq
txresp -body {
<html>
Before include
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/00"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/01"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/02"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/03"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/04"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/05"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/06"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/07"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/08"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/09"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/10"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/11"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/12"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/13"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/14"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/15"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/16"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/17"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/18"/>
<esi:include src="/some/very/long/url/with/dozen/of/information/for/esi/subquery/to/munch/and/also/to/try/to/make/object/workspace/explode/by/dumping/a/core/in/some/obscure/directory/on/my/file/system/19"/>
After include
}
} -start
server s2 {
rxreq
txresp -body { <pre>Included file 00</pre> }
} -start
server s3 {
rxreq
txresp -body { <pre>Included file 01</pre> }
} -start
server s4 {
rxreq
txresp -body { <pre>Included file 02</pre> }
} -start
server s5 {
rxreq
txresp -body { <pre>Included file 03</pre> }
} -start
server s6 {
rxreq
txresp -body { <pre>Included file 04</pre> }
} -start
server s7 {
rxreq
txresp -body { <pre>Included file 05</pre> }
} -start
server s8 {
rxreq
txresp -body { <pre>Included file 06</pre> }
} -start
server s9 {
rxreq
txresp -body { <pre>Included file 07</pre> }
} -start
server s10 {
rxreq
txresp -body { <pre>Included file 08</pre> }
} -start
server s11 {
rxreq
txresp -body { <pre>Included file 09</pre> }
} -start
server s12 {
rxreq
txresp -body { <pre>Included file 10</pre> }
} -start
server s13 {
rxreq
txresp -body { <pre>Included file 11</pre> }
} -start
server s14 {
rxreq
txresp -body { <pre>Included file 12</pre> }
} -start
server s15 {
rxreq
txresp -body { <pre>Included file 13</pre> }
} -start
server s16 {
rxreq
txresp -body { <pre>Included file 14</pre> }
} -start
server s17 {
rxreq
txresp -body { <pre>Included file 15</pre> }
} -start
server s18 {
rxreq
txresp -body { <pre>Included file 16</pre> }
} -start
server s19 {
rxreq
txresp -body { <pre>Included file 17</pre> }
} -start
server s20 {
rxreq
txresp -body { <pre>Included file 18</pre> }
} -start
server s21 {
rxreq
txresp -body { <pre>Included file 19</pre> }
} -start
varnish v1 -arg "-p vcc_err_unref=off" -vcl+backend {
import ${vmod_pesi};
sub vcl_backend_response {
set beresp.do_esi = true;
}
sub vcl_backend_fetch {
if (bereq.url == "/") {
set bereq.backend = s1;
}
elsif (bereq.url ~ "/00$") {
set bereq.backend = s2;
}
elsif (bereq.url ~ "/01$") {
set bereq.backend = s3;
}
elsif (bereq.url ~ "/02$") {
set bereq.backend = s4;
}
elsif (bereq.url ~ "/03$") {
set bereq.backend = s5;
}
elsif (bereq.url ~ "/04$") {
set bereq.backend = s6;
}
elsif (bereq.url ~ "/05$") {
set bereq.backend = s7;
}
elsif (bereq.url ~ "/06$") {
set bereq.backend = s8;
}
elsif (bereq.url ~ "/07$") {
set bereq.backend = s9;
}
elsif (bereq.url ~ "/08$") {
set bereq.backend = s10;
}
elsif (bereq.url ~ "/09$") {
set bereq.backend = s11;
}
elsif (bereq.url ~ "/10$") {
set bereq.backend = s12;
}
elsif (bereq.url ~ "/11$") {
set bereq.backend = s13;
}
elsif (bereq.url ~ "/12$") {
set bereq.backend = s14;
}
elsif (bereq.url ~ "/13$") {
set bereq.backend = s15;
}
elsif (bereq.url ~ "/14$") {
set bereq.backend = s16;
}
elsif (bereq.url ~ "/15$") {
set bereq.backend = s17;
}
elsif (bereq.url ~ "/16$") {
set bereq.backend = s18;
}
elsif (bereq.url ~ "/17$") {
set bereq.backend = s19;
}
elsif (bereq.url ~ "/18$") {
set bereq.backend = s20;
}
elsif (bereq.url ~ "/19$") {
set bereq.backend = s21;
}
}
sub vcl_deliver {
pesi.activate();
}
} -start
varnish v1 -expect PESI.no_thread == 0
client c1 {
txreq
rxresp
expect resp.status == 200
expect resp.bodylen == 684
expect resp.body == {
<html>
Before include
<pre>Included file 00</pre>
<pre>Included file 01</pre>
<pre>Included file 02</pre>
<pre>Included file 03</pre>
<pre>Included file 04</pre>
<pre>Included file 05</pre>
<pre>Included file 06</pre>
<pre>Included file 07</pre>
<pre>Included file 08</pre>
<pre>Included file 09</pre>
<pre>Included file 10</pre>
<pre>Included file 11</pre>
<pre>Included file 12</pre>
<pre>Included file 13</pre>
<pre>Included file 14</pre>
<pre>Included file 15</pre>
<pre>Included file 16</pre>
<pre>Included file 17</pre>
<pre>Included file 18</pre>
<pre>Included file 19</pre>
After include
}
} -run
logexpect l1 -v v1 -g vxid -d 1 -q {Error ~ "^vdp pesi"} {
expect 0 * Begin {^req \d+ rxreq$}
expect * = Error {^vdp pesi: No thread available for ESI subrequest \d+}
} -run
varnish v1 -expect PESI.no_thread > 0
## e00024.vtc
server s1 -wait
server s2 -wait
server s3 -wait
server s4 -wait
server s5 -wait
server s6 -wait
server s7 -wait
server s8 -wait
server s9 -wait
server s1 {
rxreq
expect req.url == "/e24"
expect req.http.accept-encoding == gzip
txresp -gzipbody {
<html> 1
Before includes 2
{<esi:include src="/bit0"/>} bit0 3
{<esi:include src="/bit1"/>} bit1 4
{<esi:include src="/bit2"/>} bit2 5
{<esi:include src="/bit3"/>} bit3 6
{<esi:include src="/bit4"/>} bit4 7
{<esi:include src="/bit5"/>} bit5 8
{<esi:include src="/bit6"/>} bit6 9
{<esi:include src="/bit7"/>} bit7 10
After includes 11
}
} -start
server s2 {
rxreq
expect req.url == "/bit0"
expect req.http.accept-encoding == gzip
txresp -gzipresidual 0 -gziplevel 9 -gzipbody {e04c8d0fd604c}
} -start
server s3 {
rxreq
expect req.url == "/bit1"
expect req.http.accept-encoding == gzip
txresp -gzipresidual 1 -gziplevel 9 -gzipbody {1ea86e6cf31bf4ec3d7a86}
} -start
server s4 {
rxreq
expect req.url == "/bit2"
expect req.http.accept-encoding == gzip
txresp -gzipresidual 2 -gziplevel 9 -gzipbody {10}
} -start
server s5 {
rxreq
expect req.url == "/bit3"
expect req.http.accept-encoding == gzip
txresp -gzipresidual 3 -gziplevel 9 -gzipbody {a5e2e2e1c2e2}
} -start
server s6 {
rxreq
expect req.url == "/bit4"
expect req.http.accept-encoding == gzip
txresp -gzipresidual 4 -gziplevel 9 -gzipbody {71c5d18ec5d5d1}
} -start
server s7 {
rxreq
expect req.url == "/bit5"
expect req.http.accept-encoding == gzip
txresp -gzipresidual 5 -gziplevel 9 -gzipbody {39886d28a6d2988}
} -start
server s8 {
rxreq
expect req.url == "/bit6"
expect req.http.accept-encoding == gzip
txresp -gzipresidual 6 -gziplevel 9 -gzipbody {80000}
} -start
server s9 {
rxreq
expect req.url == "/bit7"
expect req.http.accept-encoding == gzip
txresp -gzipresidual 7 -gziplevel 9 -gzipbody {386811868}
} -start
varnish v1 -vcl+backend {
import ${vmod_pesi};
sub vcl_backend_response {
if (bereq.url == "/e24") {
set beresp.do_esi = true;
}
}
sub vcl_backend_fetch {
if (bereq.url == "/e24") {
set bereq.backend = s1;
}
elsif (bereq.url == "/bit0") {
set bereq.backend = s2;
}
elsif (bereq.url == "/bit1") {
set bereq.backend = s3;
}
elsif (bereq.url == "/bit2") {
set bereq.backend = s4;
}
elsif (bereq.url == "/bit3") {
set bereq.backend = s5;
}
elsif (bereq.url == "/bit4") {
set bereq.backend = s6;
}
elsif (bereq.url == "/bit5") {
set bereq.backend = s7;
}
elsif (bereq.url == "/bit6") {
set bereq.backend = s8;
}
elsif (bereq.url == "/bit7") {
set bereq.backend = s9;
}
}
sub vcl_deliver {
pesi.activate();
}
}
varnish v1 -cliok "param.set http_gzip_support true"
varnish v1 -cliok "param.set debug +esi_chop"
logexpect l1 -v v1 -g vxid -d 0 -q {Error ~ "^vdp pesi"} {
expect 0 * Begin {^req \d+ rxreq$}
expect * = Error {^vdp pesi: No thread available for ESI subrequest \d+}
} -start
client c1 {
txreq -url /e24 -hdr "Accept-Encoding: gzip"
rxresp
expect resp.http.content-encoding == gzip
gunzip
expect resp.status == 200
expect resp.bodylen == 252
txreq -url /e24
rxresp
expect resp.http.content-encoding == <undef>
expect resp.status == 200
expect resp.bodylen == 252
} -run
varnish v1 -expect PESI.no_thread > 0
logexpect l1 -wait
src/vdp_pesi.c
View file @ bcd8e84b
......@@ -56,6 +56,7 @@
#include "vqueue.h"
#include "vcc_if.h"
#include "VSC_pesi.h"
#define VFAIL(ctx, fmt, ...) \
VRT_fail((ctx), "vdp pesi failure: " fmt, __VA_ARGS__)
......@@ -328,6 +329,8 @@ struct pesi {
struct pecx pecx[1];
VTAILQ_ENTRY(pesi) list;
unsigned no_thread;
};
/* Forward declarations */
......@@ -348,10 +351,12 @@ const void * const priv_task_id_cfg = &priv_task_id_cfg;
/* shared object globals */
static unsigned loadcnt = 0, warmcnt = 0;
static struct VSC_lck *lck_bytes_tree, *lck_pesi_tree;
static struct vsc_seg *vsc_seg = NULL;
static struct VSC_lck *lck_bytes_tree, *lck_pesi_tree, *lck_stats;
static struct vsc_seg *vsc_seg = NULL, *pesi_vsc_seg = NULL;
static struct mempool *mempool = NULL;
static unsigned node_alloc_sz;
static struct VSC_pesi *stats;
static struct lock stats_lock;
/* also used by our version of cache_esi_deliver.c */
......@@ -1249,9 +1254,13 @@ vped_include(struct req *preq, const char *src, const char *host,
if (Pool_Task_Arg(wrk, TASK_QUEUE_REQ,
ved_task, &req, sizeof req) == 1)
return (1);
pesi->no_thread++;
VSLb(preq->vsl, SLT_Error, "vdp pesi: No thread available "
"for ESI subrequest %u, continuing in serial",
VXID(req->vsl->wid));
/*
* XXX @geoff add counter
*
* we can not use the self-rescheduling facility of
* Pool_Task_Arg because we cannot unschedule ourself
*/
......@@ -2299,6 +2308,12 @@ vdp_pesi_fini(struct req *req, void **priv)
CHECK_OBJ_NOTNULL(bytes_tree, BYTES_TREE_MAGIC);
assert(bytes_tree == pesi_tree->tree);
if (pesi->no_thread != 0) {
Lck_Lock(&stats_lock);
stats->no_thread += pesi->no_thread;
Lck_Unlock(&stats_lock);
}
pesi_destroy(&pesi);
task_fini(pesi_tree, pesi);
......@@ -3104,8 +3119,15 @@ vmod_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
"pesi.bytes_tree");
lck_pesi_tree = Lck_CreateClass(&vsc_seg,
"pesi.pesi_tree");
lck_stats = Lck_CreateClass(&vsc_seg,
"pesi.stats");
AN(lck_bytes_tree);
AN(lck_pesi_tree);
AZ(pesi_vsc_seg);
stats = VSC_pesi_New(NULL, &pesi_vsc_seg, "");
AN(stats);
AN(pesi_vsc_seg);
Lck_New(&stats_lock, lck_stats);
}
VRT_AddVDP(ctx, &VDP_pesi);
......@@ -3114,14 +3136,18 @@ vmod_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
VRT_RemoveVDP(ctx, &VDP_pesi);
AN(loadcnt);
if (--loadcnt == 0)
if (--loadcnt == 0) {
Lck_Delete(&stats_lock);
Lck_DestroyClass(&vsc_seg);
VSC_pesi_Destroy(&pesi_vsc_seg);
}
break;
case VCL_EVENT_WARM:
if (warmcnt++ == 0) {
AZ(mempool);
mempool = mpl_init();
AN(mempool);
VRT_VSC_Reveal(pesi_vsc_seg);
}
break;
case VCL_EVENT_COLD:
......@@ -3130,6 +3156,7 @@ vmod_event(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
AN(mempool);
mpl_fini(&mempool);
AZ(mempool);
VRT_VSC_Hide(pesi_vsc_seg);
}
break;
default:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment