Add saintmode_threshold

When saintmode_threshold items have been added to the trouble-list, the
backend will be considerd sick until one has timed out. Setting the
threshold to 0 disables saintmode and any lock contention associated with
it.

git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@4328 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/cache_backend.c

@@ -231,7 +231,12 @@ VBE_GetFd(struct sess *sp)
 /*
  * It evaluates if a backend is healthy _for_a_specific_object_.
  * That means that it relies on sp->objhead. This is mainly for saint-mode,
- * but also takes backend->healthy into account.
+ * but also takes backend->healthy into account. If
+ * params->saintmode_threshold is 0, this is basically just a test of
+ * backend->healthy.
+ *
+ * The threshold has to be evaluated _after_ the timeout check, otherwise
+ * items would never time out once the threshold is reached.
  */
 unsigned int
 backend_is_healthy(const struct sess *sp, struct backend *backend)
@@ -239,6 +244,7 @@ backend_is_healthy(const struct sess *sp, struct backend *backend)
 	struct trouble *tr;
 	struct trouble *tr2;
 	struct trouble *old = NULL;
+	unsigned i = 0;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC);
@@ -246,6 +252,10 @@ backend_is_healthy(const struct sess *sp, struct backend *backend)
 	if (!backend->healthy)
 		return 0;
 
+	/* Saintmode is disabled */
+	if (params->saintmode_threshold == 0)
+		return 1;
+
 	/* No need to test if we don't have an object head to test against.
 	 * FIXME: Should check the magic too, but probably not assert?
 	 */
@@ -265,6 +275,15 @@ backend_is_healthy(const struct sess *sp, struct backend *backend)
 			Lck_Unlock(&backend->mtx);
 			return 0;
 		}
+
+		/* If the threshold is at 1, a single entry on the list
+		 * will disable the backend. Since 0 is disable, ++i
+		 * instead of i++ to allow this behavior.
+		 */
+		if (++i >= params->saintmode_threshold) {
+			Lck_Unlock(&backend->mtx);
+			return 0;
+		}
 	}
 	Lck_Unlock(&backend->mtx);
 

bin/varnishd/heritage.h

@@ -199,6 +199,9 @@ struct params {
 
 	/* How long time does the ban lurker sleep */
 	double			ban_lurker_sleep;
+
+	/* Max size of the saintmode list. 0 == no saint mode. */
+	unsigned		saintmode_threshold;
 };
 
 extern volatile struct params *params;

bin/varnishd/mgt_param.c

@@ -765,6 +765,12 @@ static const struct parspec input_parspec[] = {
 		"A value of zero disables the ban lurker.",
 		0,
 		"0.0", "s" },
+	{ "saintmode_threshold", tweak_uint, &master.saintmode_threshold, 0, UINT_MAX,
+		"The maximum number of objects held off by saint mode before no further "
+		"will be made to the backend until one times out. A value of 0 disables "
+		"saintmode.",
+		EXPERIMENTAL,
+		"10", "objects" }, 
 	{ NULL, NULL, NULL }
 };
 

bin/varnishtest/tests/c00029.vtc

+# $Id$
+
+test "Test that saintmode_threshold correctly marks a backend as sick"
+
+server s1 {
+	rxreq
+	txresp
+
+	rxreq
+	txresp
+
+	rxreq
+	txresp
+
+	rxreq
+	txresp
+
+	rxreq
+	txresp -hdr "X-Saint: yes"
+
+	rxreq
+	txresp -hdr "X-Saint: yes"
+
+	rxreq
+	txresp -hdr "X-Saint: yes"
+
+	rxreq
+	txresp -hdr "X-Saint: yes"
+} -start
+
+varnish v1 -arg "-p saintmode_threshold=2" -vcl+backend {
+	sub vcl_recv {
+		set req.grace = 1h;
+	}
+
+	sub vcl_fetch {
+		if (beresp.http.X-Saint == "yes") {
+			set beresp.saintmode = 20s;
+			restart;
+		}
+		set beresp.grace = 1h;
+		set beresp.ttl = 1s;
+	}
+	sub vcl_deliver {
+		set resp.http.X-Restarts = req.restarts;
+	}
+ } -start
+
+client c1 {
+        txreq -url "/one"
+        rxresp
+        expect resp.status == 200
+        expect resp.http.X-Saint != "yes"
+	expect resp.http.X-Restarts == "0"
+
+        txreq -url "/two"
+        rxresp
+        expect resp.status == 200
+        expect resp.http.X-Saint != "yes"
+	expect resp.http.X-Restarts == "0"
+
+        txreq -url "/three"
+        rxresp
+        expect resp.status == 200
+        expect resp.http.X-Saint != "yes"
+	expect resp.http.X-Restarts == "0"
+
+        txreq -url "/four"
+        rxresp
+        expect resp.status == 200
+        expect resp.http.X-Saint != "yes"
+	expect resp.http.X-Restarts == "0"
+
+	delay 2
+
+        txreq -url "/one"
+        rxresp
+        expect resp.status == 200
+        expect resp.http.X-Saint != "yes"
+	expect resp.http.X-Restarts == "1"
+
+        txreq -url "/two"
+        rxresp
+        expect resp.status == 200
+        expect resp.http.X-Saint != "yes"
+	expect resp.http.X-Restarts == "1"
+
+        txreq -url "/three"
+        rxresp
+        expect resp.status == 200
+        expect resp.http.X-Saint != "yes"
+	expect resp.http.X-Restarts == "0"
+
+        txreq -url "/four"
+        rxresp
+        expect resp.status == 200
+        expect resp.http.X-Saint != "yes"
+	expect resp.http.X-Restarts == "0"
+
+} -run
+

lib/libvcl/vcc_fixed_token.c

 /*
- * $Id: vcc_gen_fixed_token.tcl 4208 2009-08-24 14:51:26Z kristian $
+ * $Id$
  *
  * NB:  This file is machine generated, DO NOT EDIT!
  *
@@ -159,10 +159,9 @@ vcl_output_lang_h(struct vsb *sb)
 
 	/* ../../include/vcl.h */
 
-	vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4208 2009-08-24 14");
-	vsb_cat(sb, ":51:26Z kristian $\n *\n * NB:  This file is machine g");
-	vsb_cat(sb, "enerated, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fi");
-	vsb_cat(sb, "xed_token.tcl instead\n */\n\nstruct sess;\n");
+	vsb_cat(sb, "/*\n * $Id$\n *\n * NB:  This file is machine generate");
+	vsb_cat(sb, "d, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_tok");
+	vsb_cat(sb, "en.tcl instead\n */\n\nstruct sess;\n");
 	vsb_cat(sb, "struct cli;\n\ntypedef void vcl_init_f(struct cli *);\n");
 	vsb_cat(sb, "typedef void vcl_fini_f(struct cli *);\n");
 	vsb_cat(sb, "typedef int vcl_func_f(struct sess *sp);\n");
@@ -228,24 +227,25 @@ vcl_output_lang_h(struct vsb *sb)
 	vsb_cat(sb, " * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWI");
 	vsb_cat(sb, "SE) ARISING IN ANY WAY\n * OUT OF THE USE OF THIS SOFT");
 	vsb_cat(sb, "WARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n");
-	vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id: vrt.h 4235 2009-09-11 13:");
-	vsb_cat(sb, "06:15Z phk $\n *\n * Runtime support for compiled VCL ");
-	vsb_cat(sb, "programs.\n *\n * XXX: When this file is changed, lib/");
-	vsb_cat(sb, "libvcl/vcc_gen_fixed_token.tcl\n");
-	vsb_cat(sb, " * XXX: *MUST* be rerun.\n */\n");
-	vsb_cat(sb, "\nstruct sess;\nstruct vsb;\nstruct cli;\n");
-	vsb_cat(sb, "struct director;\nstruct VCL_conf;\n");
-	vsb_cat(sb, "struct sockaddr;\n\n/*\n * A backend probe specificati");
-	vsb_cat(sb, "on\n */\n\nextern const void *vrt_magic_string_end;\n");
-	vsb_cat(sb, "\nstruct vrt_backend_probe {\n\tconst char\t*url;\n");
-	vsb_cat(sb, "\tconst char\t*request;\n\tdouble\t\ttimeout;\n");
-	vsb_cat(sb, "\tdouble\t\tinterval;\n\tunsigned\twindow;\n");
-	vsb_cat(sb, "\tunsigned\tthreshold;\n\tunsigned\tinitial;\n");
-	vsb_cat(sb, "};\n\n/*\n * A backend is a host+port somewhere on the");
-	vsb_cat(sb, " network\n */\nstruct vrt_backend {\n");
-	vsb_cat(sb, "\tconst char\t\t\t*vcl_name;\n\tconst char\t\t\t*ident");
-	vsb_cat(sb, ";\n\n\tconst char\t\t\t*hosthdr;\n");
-	vsb_cat(sb, "\n\tconst unsigned char\t\t*ipv4_sockaddr;\n");
+	vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id$\n *\n");
+	vsb_cat(sb, " * Runtime support for compiled VCL programs.\n");
+	vsb_cat(sb, " *\n * XXX: When this file is changed, lib/libvcl/vcc_");
+	vsb_cat(sb, "gen_fixed_token.tcl\n * XXX: *MUST* be rerun.\n");
+	vsb_cat(sb, " */\n\nstruct sess;\nstruct vsb;\n");
+	vsb_cat(sb, "struct cli;\nstruct director;\n");
+	vsb_cat(sb, "struct VCL_conf;\nstruct sockaddr;\n");
+	vsb_cat(sb, "\n/*\n * A backend probe specification\n");
+	vsb_cat(sb, " */\n\nextern const void * const vrt_magic_string_end;");
+	vsb_cat(sb, "\n\nstruct vrt_backend_probe {\n");
+	vsb_cat(sb, "\tconst char\t*url;\n\tconst char\t*request;\n");
+	vsb_cat(sb, "\tdouble\t\ttimeout;\n\tdouble\t\tinterval;\n");
+	vsb_cat(sb, "\tunsigned\twindow;\n\tunsigned\tthreshold;\n");
+	vsb_cat(sb, "\tunsigned\tinitial;\n};\n\n/*\n");
+	vsb_cat(sb, " * A backend is a host+port somewhere on the network\n");
+	vsb_cat(sb, " */\nstruct vrt_backend {\n\tconst char\t\t\t*vcl_name");
+	vsb_cat(sb, ";\n\tconst char\t\t\t*ident;\n\n");
+	vsb_cat(sb, "\tconst char\t\t\t*hosthdr;\n\n");
+	vsb_cat(sb, "\tconst unsigned char\t\t*ipv4_sockaddr;\n");
 	vsb_cat(sb, "\tconst unsigned char\t\t*ipv6_sockaddr;\n");
 	vsb_cat(sb, "\n\tdouble\t\t\t\tconnect_timeout;\n");
 	vsb_cat(sb, "\tdouble\t\t\t\tfirst_byte_timeout;\n");
@@ -317,27 +317,25 @@ vcl_output_lang_h(struct vsb *sb)
 
 	/* ../../include/vrt_obj.h */
 
-	vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4208 2009-08-24 14");
-	vsb_cat(sb, ":51:26Z kristian $\n *\n * NB:  This file is machine g");
-	vsb_cat(sb, "enerated, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fi");
-	vsb_cat(sb, "xed_token.tcl instead\n */\n\nstruct sockaddr * VRT_r_");
-	vsb_cat(sb, "client_ip(const struct sess *);\n");
-	vsb_cat(sb, "struct sockaddr * VRT_r_server_ip(struct sess *);\n");
-	vsb_cat(sb, "const char * VRT_r_server_hostname(struct sess *);\n");
-	vsb_cat(sb, "const char * VRT_r_server_identity(struct sess *);\n");
-	vsb_cat(sb, "int VRT_r_server_port(struct sess *);\n");
-	vsb_cat(sb, "const char * VRT_r_req_request(const struct sess *);\n");
-	vsb_cat(sb, "void VRT_l_req_request(const struct sess *, const char");
-	vsb_cat(sb, " *, ...);\nconst char * VRT_r_req_url(const struct ses");
-	vsb_cat(sb, "s *);\nvoid VRT_l_req_url(const struct sess *, const c");
-	vsb_cat(sb, "har *, ...);\nconst char * VRT_r_req_proto(const struc");
-	vsb_cat(sb, "t sess *);\nvoid VRT_l_req_proto(const struct sess *, ");
-	vsb_cat(sb, "const char *, ...);\nvoid VRT_l_req_hash(struct sess *");
-	vsb_cat(sb, ", const char *);\nstruct director * VRT_r_req_backend(");
-	vsb_cat(sb, "struct sess *);\nvoid VRT_l_req_backend(struct sess *,");
-	vsb_cat(sb, " struct director *);\nint VRT_r_req_restarts(const str");
-	vsb_cat(sb, "uct sess *);\ndouble VRT_r_req_grace(struct sess *);\n");
-	vsb_cat(sb, "void VRT_l_req_grace(struct sess *, double);\n");
+	vsb_cat(sb, "/*\n * $Id$\n *\n * NB:  This file is machine generate");
+	vsb_cat(sb, "d, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_tok");
+	vsb_cat(sb, "en.tcl instead\n */\n\nstruct sockaddr * VRT_r_client_");
+	vsb_cat(sb, "ip(const struct sess *);\nstruct sockaddr * VRT_r_serv");
+	vsb_cat(sb, "er_ip(struct sess *);\nconst char * VRT_r_server_hostn");
+	vsb_cat(sb, "ame(struct sess *);\nconst char * VRT_r_server_identit");
+	vsb_cat(sb, "y(struct sess *);\nint VRT_r_server_port(struct sess *");
+	vsb_cat(sb, ");\nconst char * VRT_r_req_request(const struct sess *");
+	vsb_cat(sb, ");\nvoid VRT_l_req_request(const struct sess *, const ");
+	vsb_cat(sb, "char *, ...);\nconst char * VRT_r_req_url(const struct");
+	vsb_cat(sb, " sess *);\nvoid VRT_l_req_url(const struct sess *, con");
+	vsb_cat(sb, "st char *, ...);\nconst char * VRT_r_req_proto(const s");
+	vsb_cat(sb, "truct sess *);\nvoid VRT_l_req_proto(const struct sess");
+	vsb_cat(sb, " *, const char *, ...);\nvoid VRT_l_req_hash(struct se");
+	vsb_cat(sb, "ss *, const char *);\nstruct director * VRT_r_req_back");
+	vsb_cat(sb, "end(struct sess *);\nvoid VRT_l_req_backend(struct ses");
+	vsb_cat(sb, "s *, struct director *);\nint VRT_r_req_restarts(const");
+	vsb_cat(sb, " struct sess *);\ndouble VRT_r_req_grace(struct sess *");
+	vsb_cat(sb, ");\nvoid VRT_l_req_grace(struct sess *, double);\n");
 	vsb_cat(sb, "const char * VRT_r_req_xid(struct sess *);\n");
 	vsb_cat(sb, "unsigned VRT_r_req_esi(struct sess *);\n");
 	vsb_cat(sb, "void VRT_l_req_esi(struct sess *, unsigned);\n");

lib/libvcl/vcc_obj.c

 /*
- * $Id: vcc_gen_fixed_token.tcl 4208 2009-08-24 14:51:26Z kristian $
+ * $Id$
  *
  * NB:  This file is machine generated, DO NOT EDIT!
  *