Rewrite the ban-lurker in light of many changes and evidence that

it basically didn't work as intended any more.

bin/varnishd/cache/cache.h

@@ -437,7 +437,6 @@ struct objcore {
 #define OC_F_PASS		(1<<2)
 #define OC_F_OFFLRU		(1<<4)
 #define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
-#define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
 #define OC_F_DYING		(1<<7)
 #define OC_F_PRIVATE		(1<<8)
 #define OC_F_FAILED		(1<<9)

bin/varnishtest/tests/c00049.vtc

-varnishtest "ban lurker test"
-
+varnishtest "New ban-lurker test"
 
 server s1 {
 	rxreq
-	expect req.url == "/alpha"
-	txresp -hdr "Foo: /alpha"
+	expect req.url == /1
+	txresp -hdr "Foo: bar1"
 
 	rxreq
-	expect req.url == "/beta"
-	txresp -hdr "Foo: /beta"
+	expect req.url == /2
+	txresp -hdr "Foo: bar2"
 
 	rxreq
-	expect req.url == "/gamma"
-	txresp -hdr "Foo: /gamma"
+	expect req.url == /3
+	txresp -hdr "Foo: bar3"
 
 	rxreq
-	expect req.url == "/delta"
-	txresp -hdr "Foo: /delta"
+	expect req.url == /4
+	txresp -hdr "Foo: bar4"
 
 	rxreq
-	expect req.url == "/alpha"
-	txresp -hdr "Foo: /alpha2"
+	expect req.url == /5
+	txresp -hdr "Foo: bar5"
 
 	rxreq
-	expect req.url == "/beta"
-	txresp -hdr "Foo: /beta2"
+	expect req.url == /6
+	txresp -hdr "Foo: bar6"
 
 	rxreq
-	expect req.url == "/delta"
-	txresp -hdr "Foo: /delta2"
+	expect req.url == /7
+	txresp -hdr "Foo: bar7"
 
-} -start
+	rxreq
+	expect req.url == /4
+	txresp -hdr "Foo: bar4.1"
 
-varnish v1 -vcl+backend {
 } -start
 
+varnish v1 -vcl+backend {} -start
+
 varnish v1 -cliok "param.set ban_lurker_sleep 0"
 varnish v1 -cliok "param.set debug +lurker"
 
-varnish v1 -cliok "ban.list"
 
 client c1 {
-	txreq -url "/alpha"
+	txreq -url /1
 	rxresp
-	expect resp.http.foo == /alpha
+	expect resp.http.foo == bar1
+
+	txreq -url /2
+	rxresp
+	expect resp.http.foo == bar2
 } -run
 
-delay 0.1
-varnish v1 -cliok "ban req.url == /alpha"
-varnish v1 -cliok "ban.list"
-varnish v1 -expect bans == 2
-varnish v1 -expect bans_gone == 1
+varnish v1 -cliok "ban obj.http.foo == bar1"
 
 client c1 {
-	txreq -url "/beta"
+	txreq -url /3
 	rxresp
-	expect resp.http.foo == /beta
+	expect resp.http.foo == bar3
 } -run
 
-delay 0.1
-varnish v1 -cliok "ban obj.http.foo == /beta"
-varnish v1 -cliok "ban.list"
-varnish v1 -expect bans == 3
+varnish v1 -cliok "ban obj.http.foo == bar2 && obj.http.foo != foof"
 
 client c1 {
-	txreq -url "/gamma"
+	txreq -url /4
 	rxresp
-	expect resp.http.foo == /gamma
+	expect resp.http.foo == bar4
 } -run
 
-delay 0.1
-varnish v1 -cliok "ban obj.http.foo == /gamma"
-varnish v1 -cliok "ban.list"
-varnish v1 -expect bans == 4
+varnish v1 -cliok "ban req.http.kill == yes"
 
 client c1 {
-	txreq -url "/delta"
+	txreq -url /5
 	rxresp
-	expect resp.http.foo == /delta
+	expect resp.http.foo == bar5
 } -run
 
-delay 0.1
-varnish v1 -cliok "ban req.url == /delta"
+varnish v1 -cliok "ban obj.http.foo == bar5"
 
-varnish v1 -expect bans_gone == 1
-varnish v1 -cliok "ban obj.http.foo == /gamma"
-# Dup-check should have added one
-varnish v1 -expect bans_gone == 2
+client c1 {
+	txreq -url /6
+	rxresp
+	expect resp.http.foo == bar6
+} -run
+
+varnish v1 -cliok "ban obj.http.foo == bar6"
+
+client c1 {
+	txreq -url /7
+	rxresp
+	expect resp.http.foo == bar7
+} -run
+
+# Get the VSL out of the way
+delay .1
 
-varnish v1 -cliok "ban req.url == /epsilon"
 varnish v1 -cliok "ban.list"
-varnish v1 -expect bans == 7
-varnish v1 -expect bans_gone == 2
+
+varnish v1 -expect bans == 6
+varnish v1 -expect bans_gone == 1
+varnish v1 -expect bans_req == 1
+varnish v1 -expect bans_obj == 4
+varnish v1 -expect bans_added == 6
+varnish v1 -expect bans_deleted == 0
+varnish v1 -expect bans_tested == 0
+varnish v1 -expect bans_tests_tested == 0
+varnish v1 -expect bans_obj_killed == 0
+varnish v1 -expect bans_lurker_tested == 0
+varnish v1 -expect bans_lurker_tests_tested == 0
+varnish v1 -expect bans_lurker_obj_killed == 0
+varnish v1 -expect bans_dups == 0
+
 
 varnish v1 -cliok "param.set ban_lurker_sleep .01"
-delay 1
-varnish v1 -cliok "param.set ban_lurker_sleep .00"
+
+delay 2
+
 varnish v1 -cliok "ban.list"
-varnish v1 -expect bans == 7
+
+varnish v1 -expect bans == 5
 varnish v1 -expect bans_gone == 4
+varnish v1 -expect bans_req == 1
+varnish v1 -expect bans_obj == 4
+varnish v1 -expect bans_added == 6
+varnish v1 -expect bans_deleted == 1
+varnish v1 -expect bans_tested == 0
+varnish v1 -expect bans_tests_tested == 0
+varnish v1 -expect bans_obj_killed == 0
+varnish v1 -expect bans_lurker_tested == 10
+varnish v1 -expect bans_lurker_tests_tested == 11
+varnish v1 -expect bans_lurker_obj_killed == 4
+varnish v1 -expect bans_dups == 0
 
 client c1 {
-	txreq -url "/alpha"
+	txreq -url /3
 	rxresp
-	expect resp.http.foo == /alpha2
+	expect resp.http.foo == bar3
 } -run
 
+# Give lurker time to trim tail
 delay 1
-varnish v1 -cliok "ban.list"
-varnish v1 -expect bans == 4
-
-client c1 {
-	txreq -url "/beta"
-	rxresp
-	expect resp.http.foo == /beta2
-} -run
 
 varnish v1 -cliok "ban.list"
 
+varnish v1 -expect bans == 4
+varnish v1 -expect bans_gone == 3
+varnish v1 -expect bans_req == 1
+varnish v1 -expect bans_obj == 3
+varnish v1 -expect bans_added == 6
+varnish v1 -expect bans_deleted == 2
+varnish v1 -expect bans_tested == 1
+varnish v1 -expect bans_tests_tested == 1
+varnish v1 -expect bans_obj_killed == 0
+varnish v1 -expect bans_lurker_tested == 10
+varnish v1 -expect bans_lurker_tests_tested == 11
+varnish v1 -expect bans_lurker_obj_killed == 4
+varnish v1 -expect bans_dups == 0
 
 client c1 {
-	txreq -url "/delta"
+	txreq -url /4 -hdr "kill: yes"
 	rxresp
-	expect resp.http.foo == /delta2
+	expect resp.http.foo == bar4.1
 } -run
 
+# Give lurker time to trim tail
 delay 1
+
 varnish v1 -cliok "ban.list"
+
 varnish v1 -expect bans == 1
-varnish v1 -expect bans_gone == 0
-varnish v1 -expect bans_added == 7
-varnish v1 -expect bans_deleted == 6
+varnish v1 -expect bans_gone == 1
+varnish v1 -expect bans_req == 0
+varnish v1 -expect bans_obj == 1
+varnish v1 -expect bans_added == 6
+varnish v1 -expect bans_deleted == 5
+varnish v1 -expect bans_tested == 2
+varnish v1 -expect bans_tests_tested == 2
+varnish v1 -expect bans_obj_killed == 1
+varnish v1 -expect bans_lurker_tested == 10
+varnish v1 -expect bans_lurker_tests_tested == 11
+varnish v1 -expect bans_lurker_obj_killed == 4
+varnish v1 -expect bans_dups == 0
+
+varnish v1 -expect n_object == 3

bin/varnishtest/tests/r01030.vtc

@@ -40,10 +40,10 @@ client c1 {
 } -run
 
 #delay 0.1
-varnish v1 -expect bans_tests_tested == 0
+varnish v1 -expect bans_lurker_tests_tested == 0
 
 delay 1.0
-varnish v1 -expect bans_tests_tested == 1
+varnish v1 -expect bans_lurker_tests_tested == 1
 
 varnish v1 -cliok "param.set ban_lurker_sleep 5.01"
 
@@ -58,7 +58,7 @@ client c2 {
 } -run
 
 #delay 0.1
-varnish v1 -expect bans_tests_tested == 1
+varnish v1 -expect bans_lurker_tests_tested == 1
 
 delay 1.1
-varnish v1 -expect bans_tests_tested == 2
+varnish v1 -expect bans_lurker_tests_tested == 2

include/tbl/vsc_f_main.h

@@ -490,6 +490,11 @@ VSC_F(bans_gone,		uint64_t, 0, 'g', diag,
 	"Number of bans which are no longer active, either because they"
 	" got checked by the ban-lurker or superseded by newer identical bans."
 )
+VSC_F(bans_obj,			uint64_t, 0, 'g', diag,
+    "Number of bans using obj.*",
+	"Number of bans which use obj.* variables.  These bans can possibly"
+	" be washed by the ban-lurker."
+)
 VSC_F(bans_req,			uint64_t, 0, 'g', diag,
     "Number of bans using req.*",
 	"Number of bans which use req.* variables.  These bans can not"
@@ -505,20 +510,45 @@ VSC_F(bans_deleted,		uint64_t, 0, 'c', diag,
 )
 
 VSC_F(bans_tested,		uint64_t, 0, 'c', diag,
-    "Bans tested against objects",
+    "Bans tested against objects (lookup)",
+	"Count of how many bans and objects have been tested against"
+	" each other during hash lookup."
+)
+VSC_F(bans_obj_killed,	uint64_t, 0, 'c', diag,
+    "Objects killed by bans (lookup)",
+	"Number of objects killed by bans during object lookup."
+)
+VSC_F(bans_lurker_tested,	uint64_t, 0, 'c', diag,
+    "Bans tested against objects (lurker)",
 	"Count of how many bans and objects have been tested against"
-	" each other."
+	" each other by the ban-lurker."
 )
 VSC_F(bans_tests_tested,	uint64_t, 0, 'c', diag,
-    "Ban tests tested against objects",
+    "Ban tests tested against objects (lookup)",
 	"Count of how many tests and objects have been tested against"
-	" each other.  'ban req.url == foo && req.http.host == bar'"
+	" each other during lookup."
+	"  'ban req.url == foo && req.http.host == bar'"
 	" counts as one in 'bans_tested' and as two in 'bans_tests_tested'"
 )
+VSC_F(bans_lurker_tests_tested,	uint64_t, 0, 'c', diag,
+    "Ban tests tested against objects (lurker)",
+	"Count of how many tests and objects have been tested against"
+	" each other by the ban-lurker."
+	"  'ban req.url == foo && req.http.host == bar'"
+	" counts as one in 'bans_tested' and as two in 'bans_tests_tested'"
+)
+VSC_F(bans_lurker_obj_killed,	uint64_t, 0, 'c', diag,
+    "Objects killed by bans (lurker)",
+	"Number of objects killed by ban-lurker."
+)
 VSC_F(bans_dups,		uint64_t, 0, 'c', diag,
     "Bans superseded by other bans",
 	"Count of bans replaced by later identical bans."
 )
+VSC_F(bans_lurker_contention,	uint64_t, 0, 'c', diag,
+    "Lurker gave way for lookup",
+	"Number of times the ban-lurker had to wait for lookups."
+)
 VSC_F(bans_persisted_bytes,	uint64_t, 0, 'g', diag,
     "Bytes used by the persisted ban lists",
         "Number of bytes used by the persisted ban lists."