Skip to content

V
varnish-cache
Commit 4073a4b6 authored by Nils Goroll's avatar Nils Goroll
Browse files
Options

Parameter ban_cutoff to limit the number of active (incomplete) bans 

Expurge long tail content from the cache to keep the number of bans
below this value. 0 disables.

This is a safety net to avoid bad response times due to bans being
tested at lookup time. Setting a cutoff trades response time for cache
efficiency. The recommended value is proportional to
rate(bans_lurker_tests_tested) / n_objects while the ban lurker is
working, which is the number of bans the system can sustain. The
additional latency due to request ban testing is in the order of
ban_cutoff / rate(bans_lurker_tests_tested). For example, for
rate(bans_lurker_tests_tested) = 2M/s and a tolerable latency of
100ms, a good value for ban_cutoff may be 200K.

Merges #2131
parent 0b7b8e77
Hide whitespace changes
Inline Side-by-side
Showing with 97 additions and 12 deletions
bin/varnishd/cache/cache_ban_lurker.c
View file @ 4073a4b6
......@@ -196,7 +196,7 @@ ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt)
static void
ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
struct banhead_s *obans, struct ban *bd)
struct banhead_s *obans, struct ban *bd, int kill)
{
struct ban *bl, *bln;
struct objcore *oc;
......@@ -238,16 +238,30 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
VTAILQ_REMOVE(obans, bl, l_list);
continue;
}
AZ(bl->flags & BANS_FLAG_REQ);
tests = 0;
i = ban_evaluate(wrk, bl->spec, oc, NULL, &tests);
VSC_C_main->bans_lurker_tested++;
VSC_C_main->bans_lurker_tests_tested += tests;
if (kill == 1)
i = 1;
else {
AZ(bl->flags & BANS_FLAG_REQ);
tests = 0;
i = ban_evaluate(wrk, bl->spec, oc, NULL,
&tests);
VSC_C_main->bans_lurker_tested++;
VSC_C_main->bans_lurker_tests_tested += tests;
}
if (i) {
VSLb(vsl, SLT_ExpBan, "%u banned by lurker",
ObjGetXID(wrk, oc));
if (kill) {
VSLb(vsl, SLT_ExpBan,
"%u killed for lurker cutoff",
ObjGetXID(wrk, oc));
VSC_C_main->
bans_lurker_obj_killed_cutoff++;
} else {
VSLb(vsl, SLT_ExpBan,
"%u banned by lurker",
ObjGetXID(wrk, oc));
VSC_C_main->bans_lurker_obj_killed++;
}
HSH_Kill(oc);
VSC_C_main->bans_lurker_obj_killed++;
break;
}
}
......@@ -285,12 +299,15 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
struct ban *b, *bd;
struct banhead_s obans;
double d, dt, n;
unsigned count = 0, cutoff = UINT_MAX;
dt = 49.62; // Random, non-magic
if (cache_param->ban_lurker_sleep == 0) {
(void)ban_cleantail(NULL);
return (dt);
}
if (cache_param->ban_cutoff > 0)
cutoff = cache_param->ban_cutoff;
Lck_Lock(&ban_mtx);
b = ban_start;
......@@ -300,14 +317,17 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
VTAILQ_INIT(&obans);
for (; b != NULL; b = VTAILQ_NEXT(b, list)) {
if (bd != NULL && bd != b)
ban_lurker_test_ban(wrk, vsl, b, &obans, bd);
ban_lurker_test_ban(wrk, vsl, b, &obans, bd,
count > cutoff);
if (b->flags & BANS_FLAG_COMPLETED)
continue;
if (b->flags & BANS_FLAG_REQ) {
if (b->flags & BANS_FLAG_REQ &&
count <= cutoff) {
if (bd != NULL)
bd = VTAILQ_NEXT(b, list);
continue;
}
count++;
n = ban_time(b->spec) - d;
if (n < 0) {
VTAILQ_INSERT_TAIL(&obans, b, l_list);
......
bin/varnishtest/tests/c00049.vtc
View file @ 4073a4b6
......@@ -188,3 +188,38 @@ varnish v1 -expect bans_lurker_obj_killed == 4
varnish v1 -expect bans_dups == 0
varnish v1 -expect n_object == 3
# adding more bans than the cutoff purges all untested objects
# (here: all objects)
varnish v1 -cliok "param.set ban_lurker_age 2"
varnish v1 -cliok "param.set ban_cutoff 4"
varnish v1 -cliok "ban.list"
varnish v1 -cliok "ban obj.http.nomatch == 1"
varnish v1 -cliok "ban obj.http.nomatch == 2"
varnish v1 -cliok "ban obj.http.nomatch == 3"
varnish v1 -cliok "ban obj.http.nomatch == 4"
varnish v1 -cliok "ban obj.http.nomatch == 5"
varnish v1 -cliok "ban.list"
varnish v1 -cliok "param.set ban_lurker_age .1"
delay 3
varnish v1 -cliok "ban.list"
varnish v1 -expect bans == 1
varnish v1 -expect bans_completed == 1
varnish v1 -expect bans_req == 0
varnish v1 -expect bans_obj == 1
varnish v1 -expect bans_added == 11
varnish v1 -expect bans_deleted == 10
varnish v1 -expect bans_tested == 2
varnish v1 -expect bans_tests_tested == 2
varnish v1 -expect bans_obj_killed == 1
varnish v1 -expect bans_lurker_tested == 8
varnish v1 -expect bans_lurker_tests_tested == 9
varnish v1 -expect bans_lurker_obj_killed == 4
varnish v1 -expect bans_lurker_obj_killed_cutoff == 3
varnish v1 -expect bans_dups == 0
varnish v1 -expect n_object == 0
include/tbl/params.h
View file @ 4073a4b6
......@@ -136,6 +136,30 @@ PARAM(
/* func */ NULL
)
PARAM(
/* name */ ban_cutoff,
/* typ */ uint,
/* min */ "0",
/* max */ NULL,
/* default */ "0",
/* units */ "bans",
/* flags */ EXPERIMENTAL,
/* s-text */
"Expurge long tail content from the cache to keep the number of bans "
"below this value. 0 disables.\n"
"This is a safety net to avoid bad response times due to bans being "
"tested at lookup time. Setting a cutoff trades response time for "
"cache efficiency. The recommended value is proportional to "
"rate(bans_lurker_tests_tested) / n_objects while the ban lurker is "
"working, which is the number of bans the system can sustain. The "
"additional latency due to request ban testing is in the order of "
"ban_cutoff / rate(bans_lurker_tests_tested). For example, for "
"rate(bans_lurker_tests_tested) = 2M/s and a tolerable latency of "
"100ms, a good value for ban_cutoff may be 200K.",
/* l-text */ "",
/* func */ NULL
)
PARAM(
/* name */ ban_lurker_age,
/* typ */ timeout,
......
include/tbl/vsc_f_main.h
View file @ 4073a4b6
......@@ -571,7 +571,13 @@ VSC_FF(bans_lurker_tests_tested, uint64_t, 0, 'c', 'i', diag,
VSC_FF(bans_lurker_obj_killed, uint64_t, 0, 'c', 'i', diag,
"Objects killed by bans (lurker)",
"Number of objects killed by ban-lurker."
"Number of objects killed by the ban-lurker."
)
VSC_FF(bans_lurker_obj_killed_cutoff, uint64_t, 0, 'c', 'i', diag,
"Objects killed by bans for cutoff (lurker)",
"Number of objects killed by the ban-lurker to keep the number of"
" bans below ban_cutoff."
)
VSC_FF(bans_dups, uint64_t, 0, 'c', 'i', diag,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment