Saint mode: add beresp.saintmode to vcl_fetch.

This allows us to put items on a blacklist for a specific backend based on
the response it gave us. The basic syntax is (vcl_fetch):

if (beresp.status == 500) {
	set beresp.saintmode = 20s;
	restart;
}

Health checks are modified accordingly, which means graced objects are used
if no healthy backends are left to try.

git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@4208 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/cache_backend.c

@@ -228,6 +228,52 @@ VBE_GetFd(struct sess *sp)
 	sp->vbe = sp->director->getfd(sp);
 }
 
+/*
+ * It evaluates if a backend is healthy _for_a_specific_object_.
+ * That means that it relies on sp->objhead. This is mainly for saint-mode,
+ * but also takes backend->healthy into account.
+ */
+unsigned int
+backend_is_healthy(const struct sess *sp, struct backend *backend)
+{
+	struct trouble *tr;
+	struct trouble *tr2;
+	struct trouble *old = NULL;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC);
+
+	if (!backend->healthy)
+		return 0;
+
+	/* No need to test if we don't have an object head to test against.
+	 * FIXME: Should check the magic too, but probably not assert?
+	 */
+	if (!sp->objhead)
+		return 1;
+
+	Lck_Lock(&backend->mtx);
+	VTAILQ_FOREACH_SAFE(tr, &backend->troublelist, list, tr2) {
+		CHECK_OBJ_NOTNULL(tr, TROUBLE_MAGIC);
+		if (tr->timeout < sp->t_req) {
+			VTAILQ_REMOVE(&backend->troublelist, tr, list);
+			old = tr;
+			break;
+		}
+
+		if (tr->objhead == sp->objhead) {
+			Lck_Unlock(&backend->mtx);
+			return 0;
+		}
+	}
+	Lck_Unlock(&backend->mtx);
+
+	if (old)
+		FREE_OBJ(old);
+
+	return 1;
+}
+
 /*--------------------------------------------------------------------
  * Get a connection to a particular backend.
  */
@@ -265,7 +311,7 @@ VBE_GetVbe(struct sess *sp, struct backend *bp)
 		VBE_ClosedFd(sp);
 	}
 
-	if (!bp->healthy) {
+	if (!backend_is_healthy(sp, bp)) {
 		VSL_stats->backend_unhealthy++;
 		return (NULL);
 	}

bin/varnishd/cache_backend.h

@@ -92,6 +92,18 @@ struct director {
 	void			*priv;
 };
 
+/*--------------------------------------------------------------------
+ * List of objectheads that have recently been rejected by VCL.
+ */
+
+struct trouble {
+	unsigned		magic;
+#define TROUBLE_MAGIC		0x4211ab21
+	void			*objhead; /* NB: only comparison */
+	double			timeout;
+	VTAILQ_ENTRY(trouble)	list;
+};
+
 /*--------------------------------------------------------------------
  * An instance of a backend from a VCL program.
  */
@@ -124,11 +136,13 @@ struct backend {
 
 	struct vbp_target	*probe;
 	unsigned		healthy;
+	VTAILQ_HEAD(, trouble)	troublelist;
 };
 
 /* cache_backend.c */
 void VBE_ReleaseConn(struct vbe_conn *vc);
 struct vbe_conn *VBE_GetVbe(struct sess *sp, struct backend *bp);
+unsigned int backend_is_healthy(const struct sess *sp, struct backend *backend);
 
 /* cache_backend_cfg.c */
 extern struct lock VBE_mtx;

bin/varnishd/cache_backend_cfg.c

@@ -226,6 +226,8 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 	VTAILQ_INIT(&b->connlist);
 	b->hash = u;
 
+	VTAILQ_INIT(&b->troublelist);
+
 	/*
 	 * This backend may live longer than the VCL that instantiated it
 	 * so we cannot simply reference the VCL's copy of things.

bin/varnishd/cache_dir_random.c

@@ -81,7 +81,7 @@ vdi_random_getfd(struct sess *sp)
 		/* Sum up the weights of healty backends */
 		s1 = 0.0;
 		for (i = 0; i < vs->nhosts; i++)
-			if (vs->hosts[i].backend->healthy)
+			if (backend_is_healthy(sp,vs->hosts[i].backend))
 				s1 += vs->hosts[i].weight;
 
 		if (s1 == 0.0)
@@ -94,7 +94,7 @@ vdi_random_getfd(struct sess *sp)
 
 		s1 = 0.0;
 		for (i = 0; i < vs->nhosts; i++)  {
-			if (!vs->hosts[i].backend->healthy)
+			if (!backend_is_healthy(sp, vs->hosts[i].backend))
 				continue;
 			s1 += vs->hosts[i].weight;
 			if (r >= s1)
@@ -120,7 +120,7 @@ vdi_random_healthy(const struct sess *sp)
 	CAST_OBJ_NOTNULL(vs, sp->director->priv, VDI_RANDOM_MAGIC);
 
 	for (i = 0; i < vs->nhosts; i++) {
-		if (vs->hosts[i].backend->healthy)
+		if (backend_is_healthy(sp,vs->hosts[i].backend))
 			return 1;
 	}
 	return 0;

bin/varnishd/cache_dir_round_robin.c

@@ -75,7 +75,7 @@ vdi_round_robin_getfd(struct sess *sp)
 	for (i = 0; i < vs->nhosts; i++) {
 		backend = vs->hosts[vs->next_host].backend;
 		vs->next_host = (vs->next_host + 1) % vs->nhosts;
-		if (!backend->healthy)
+		if (!backend_is_healthy(sp, backend))
 			continue;
 		vbe = VBE_GetVbe(sp, backend);
 		if (vbe != NULL)
@@ -96,7 +96,7 @@ vdi_round_robin_healthy(const struct sess *sp)
 	CAST_OBJ_NOTNULL(vs, sp->director->priv, VDI_ROUND_ROBIN_MAGIC);
 
 	for (i = 0; i < vs->nhosts; i++) {
-		if (vs->hosts[i].backend->healthy)
+		if (backend_is_healthy(sp, vs->hosts[i].backend))
 			return 1;
 	}
 	return 0;

bin/varnishd/cache_dir_simple.c

@@ -75,7 +75,7 @@ vdi_simple_healthy(const struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->director, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(vs, sp->director->priv, VDI_SIMPLE_MAGIC);
-	return vs->backend->healthy;
+	return (backend_is_healthy(sp, vs->backend));
 }
 
 /*lint -e{818} not const-able */

bin/varnishd/cache_hash.c

@@ -493,6 +493,11 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	 * XXX: this until the object is unbusy'ed, so in practice we
 	 * XXX: serialize fetch of all Vary's if grace is possible.
 	 */
+	/* Grace-stuff: sp->objhead is evaluated in healthy() for 'saint
+	 * mode'. Is this entirely wrong, or just ugly? Why isn't objhead
+	 * set here? FIXME:Grace.
+	 */
+	sp->objhead = oh;
 	if (oc == NULL && grace_oc != NULL && 
 	    (busy_oc != NULL || !sp->director->healthy(sp))) {
 		o = grace_oc->obj;
@@ -500,6 +505,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 		if (o->ttl + HSH_Grace(sp->grace) >= sp->t_req)
 			oc = grace_oc;
 	}
+	sp->objhead = NULL;
 
 	if (oc != NULL) {
 		o = oc->obj;

bin/varnishd/cache_vrt.c

@@ -279,6 +279,59 @@ VRT_l_obj_status(const struct sess *sp, int num)
 	http_SetH(sp->obj->http, HTTP_HDR_STATUS, p);
 }
 
+/* Add an objecthead to the saintmode list for the (hopefully) relevant
+ * backend. Some double-up asserting here to avoid assert-errors when there
+ * is no object. 
+ */
+void
+VRT_l_beresp_saintmode(const struct sess *sp, double a)
+{
+	struct trouble *new;
+	struct trouble *tr;
+	struct trouble *tr2;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	if (!sp->vbe)
+		return;
+	CHECK_OBJ_NOTNULL(sp->vbe, VBE_CONN_MAGIC);
+	if (!sp->vbe->backend)
+		return;
+	CHECK_OBJ_NOTNULL(sp->vbe->backend, BACKEND_MAGIC);
+	if (!sp->objhead)
+		return;
+	CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC);
+
+	/* Setting a negative holdoff period is a mistake. Detecting this
+	 * when compiling the VCL would be better.
+	 */
+	assert(a > 0);
+	
+	ALLOC_OBJ(new, TROUBLE_MAGIC);
+	new->objhead = sp->objhead;
+	new->timeout = sp->t_req + a;
+
+	/* Insert the new item on the list before the first item with a
+	 * timeout at a later date (ie: sort by which entry will time out
+	 * from the list
+	 */	
+	Lck_Lock(&sp->vbe->backend->mtx);
+	VTAILQ_FOREACH_SAFE(tr, &sp->vbe->backend->troublelist, list, tr2) {
+		if (tr->timeout < new->timeout) {
+			VTAILQ_INSERT_BEFORE(tr, new, list);
+			new = NULL;
+			break;
+		}
+	}
+
+	/* Insert the item at the end if the list is empty or all other
+	 * items have a longer timeout.
+	 */
+	if (new)
+		VTAILQ_INSERT_TAIL(&sp->vbe->backend->troublelist, new, list);
+	
+	Lck_Unlock(&sp->vbe->backend->mtx);
+}
+
 int
 VRT_r_obj_status(const struct sess *sp)
 {

bin/varnishtest/tests/s00003.vtc

+# $Id$
+
+test "Check saint mode with sick pages"
+
+server s1 -listen 127.0.0.1:9080 {
+	timeout 10
+
+	rxreq
+	expect req.url == "/"
+	txresp -status 200 -hdr "foo: 1"
+	rxreq
+	expect req.url == "/"
+	txresp -status 200 -hdr "foo: 2"
+	rxreq
+	expect req.url == "/"
+	txresp -status 200 -hdr "foo: 3"
+} -start
+
+varnish v1 -vcl {
+	backend b { .host = "127.0.0.1"; .port = "9080"; }
+	sub vcl_fetch {
+		set beresp.ttl = 1s;
+		set beresp.grace = 10m;
+		set beresp.cacheable = true;
+		if (beresp.http.foo == "2")
+		{
+			set beresp.saintmode = 2s;
+			restart;
+		}
+		deliver;
+	}
+} -start
+
+client c1 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.foo == 1
+} -run
+
+delay 2
+
+client c2 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.foo == 1
+} -run
+
+delay 2
+
+client c3 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.foo == 3
+} -run

include/vrt_obj.h

@@ -41,6 +41,7 @@ double VRT_r_bereq_between_bytes_timeout(struct sess *);
 void VRT_l_bereq_between_bytes_timeout(struct sess *, double);
 const char * VRT_r_beresp_proto(const struct sess *);
 void VRT_l_beresp_proto(const struct sess *, const char *, ...);
+void VRT_l_beresp_saintmode(const struct sess *, double);
 int VRT_r_beresp_status(const struct sess *);
 void VRT_l_beresp_status(const struct sess *, int);
 const char * VRT_r_beresp_response(const struct sess *);

lib/libvcl/vcc_fixed_token.c

 /*
- * $Id: vcc_gen_fixed_token.tcl 4188 2009-08-18 08:29:27Z phk $
+ * $Id$
  *
  * NB:  This file is machine generated, DO NOT EDIT!
  *
@@ -159,10 +159,9 @@ vcl_output_lang_h(struct vsb *sb)
 
 	/* ../../include/vcl.h */
 
-	vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4188 2009-08-18 08");
-	vsb_cat(sb, ":29:27Z phk $\n *\n * NB:  This file is machine genera");
-	vsb_cat(sb, "ted, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_t");
-	vsb_cat(sb, "oken.tcl instead\n */\n\nstruct sess;\n");
+	vsb_cat(sb, "/*\n * $Id$\n *\n * NB:  This file is machine generate");
+	vsb_cat(sb, "d, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_tok");
+	vsb_cat(sb, "en.tcl instead\n */\n\nstruct sess;\n");
 	vsb_cat(sb, "struct cli;\n\ntypedef void vcl_init_f(struct cli *);\n");
 	vsb_cat(sb, "typedef void vcl_fini_f(struct cli *);\n");
 	vsb_cat(sb, "typedef int vcl_func_f(struct sess *sp);\n");
@@ -228,15 +227,15 @@ vcl_output_lang_h(struct vsb *sb)
 	vsb_cat(sb, " * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWI");
 	vsb_cat(sb, "SE) ARISING IN ANY WAY\n * OUT OF THE USE OF THIS SOFT");
 	vsb_cat(sb, "WARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n");
-	vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id: vrt.h 4185 2009-08-17 11:");
-	vsb_cat(sb, "53:01Z phk $\n *\n * Runtime support for compiled VCL ");
-	vsb_cat(sb, "programs.\n *\n * XXX: When this file is changed, lib/");
-	vsb_cat(sb, "libvcl/vcc_gen_fixed_token.tcl\n");
-	vsb_cat(sb, " * XXX: *MUST* be rerun.\n */\n");
-	vsb_cat(sb, "\nstruct sess;\nstruct vsb;\nstruct cli;\n");
-	vsb_cat(sb, "struct director;\nstruct VCL_conf;\n");
-	vsb_cat(sb, "struct sockaddr;\n\n/*\n * A backend probe specificati");
-	vsb_cat(sb, "on\n */\n\nextern void *vrt_magic_string_end;\n");
+	vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id$\n *\n");
+	vsb_cat(sb, " * Runtime support for compiled VCL programs.\n");
+	vsb_cat(sb, " *\n * XXX: When this file is changed, lib/libvcl/vcc_");
+	vsb_cat(sb, "gen_fixed_token.tcl\n * XXX: *MUST* be rerun.\n");
+	vsb_cat(sb, " */\n\nstruct sess;\nstruct vsb;\n");
+	vsb_cat(sb, "struct cli;\nstruct director;\n");
+	vsb_cat(sb, "struct VCL_conf;\nstruct sockaddr;\n");
+	vsb_cat(sb, "\n/*\n * A backend probe specification\n");
+	vsb_cat(sb, " */\n\nextern void *vrt_magic_string_end;\n");
 	vsb_cat(sb, "\nstruct vrt_backend_probe {\n\tconst char\t*url;\n");
 	vsb_cat(sb, "\tconst char\t*request;\n\tdouble\t\ttimeout;\n");
 	vsb_cat(sb, "\tdouble\t\tinterval;\n\tunsigned\twindow;\n");
@@ -317,26 +316,25 @@ vcl_output_lang_h(struct vsb *sb)
 
 	/* ../../include/vrt_obj.h */
 
-	vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4188 2009-08-18 08");
-	vsb_cat(sb, ":29:27Z phk $\n *\n * NB:  This file is machine genera");
-	vsb_cat(sb, "ted, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_t");
-	vsb_cat(sb, "oken.tcl instead\n */\n\nstruct sockaddr * VRT_r_clien");
-	vsb_cat(sb, "t_ip(const struct sess *);\nstruct sockaddr * VRT_r_se");
-	vsb_cat(sb, "rver_ip(struct sess *);\nconst char * VRT_r_server_hos");
-	vsb_cat(sb, "tname(struct sess *);\nconst char * VRT_r_server_ident");
-	vsb_cat(sb, "ity(struct sess *);\nint VRT_r_server_port(struct sess");
-	vsb_cat(sb, " *);\nconst char * VRT_r_req_request(const struct sess");
-	vsb_cat(sb, " *);\nvoid VRT_l_req_request(const struct sess *, cons");
-	vsb_cat(sb, "t char *, ...);\nconst char * VRT_r_req_url(const stru");
-	vsb_cat(sb, "ct sess *);\nvoid VRT_l_req_url(const struct sess *, c");
-	vsb_cat(sb, "onst char *, ...);\nconst char * VRT_r_req_proto(const");
-	vsb_cat(sb, " struct sess *);\nvoid VRT_l_req_proto(const struct se");
-	vsb_cat(sb, "ss *, const char *, ...);\nvoid VRT_l_req_hash(struct ");
-	vsb_cat(sb, "sess *, const char *);\nstruct director * VRT_r_req_ba");
-	vsb_cat(sb, "ckend(struct sess *);\nvoid VRT_l_req_backend(struct s");
-	vsb_cat(sb, "ess *, struct director *);\nint VRT_r_req_restarts(con");
-	vsb_cat(sb, "st struct sess *);\ndouble VRT_r_req_grace(struct sess");
-	vsb_cat(sb, " *);\nvoid VRT_l_req_grace(struct sess *, double);\n");
+	vsb_cat(sb, "/*\n * $Id$\n *\n * NB:  This file is machine generate");
+	vsb_cat(sb, "d, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_tok");
+	vsb_cat(sb, "en.tcl instead\n */\n\nstruct sockaddr * VRT_r_client_");
+	vsb_cat(sb, "ip(const struct sess *);\nstruct sockaddr * VRT_r_serv");
+	vsb_cat(sb, "er_ip(struct sess *);\nconst char * VRT_r_server_hostn");
+	vsb_cat(sb, "ame(struct sess *);\nconst char * VRT_r_server_identit");
+	vsb_cat(sb, "y(struct sess *);\nint VRT_r_server_port(struct sess *");
+	vsb_cat(sb, ");\nconst char * VRT_r_req_request(const struct sess *");
+	vsb_cat(sb, ");\nvoid VRT_l_req_request(const struct sess *, const ");
+	vsb_cat(sb, "char *, ...);\nconst char * VRT_r_req_url(const struct");
+	vsb_cat(sb, " sess *);\nvoid VRT_l_req_url(const struct sess *, con");
+	vsb_cat(sb, "st char *, ...);\nconst char * VRT_r_req_proto(const s");
+	vsb_cat(sb, "truct sess *);\nvoid VRT_l_req_proto(const struct sess");
+	vsb_cat(sb, " *, const char *, ...);\nvoid VRT_l_req_hash(struct se");
+	vsb_cat(sb, "ss *, const char *);\nstruct director * VRT_r_req_back");
+	vsb_cat(sb, "end(struct sess *);\nvoid VRT_l_req_backend(struct ses");
+	vsb_cat(sb, "s *, struct director *);\nint VRT_r_req_restarts(const");
+	vsb_cat(sb, " struct sess *);\ndouble VRT_r_req_grace(struct sess *");
+	vsb_cat(sb, ");\nvoid VRT_l_req_grace(struct sess *, double);\n");
 	vsb_cat(sb, "const char * VRT_r_req_xid(struct sess *);\n");
 	vsb_cat(sb, "unsigned VRT_r_req_esi(struct sess *);\n");
 	vsb_cat(sb, "void VRT_l_req_esi(struct sess *, unsigned);\n");
@@ -357,7 +355,8 @@ vcl_output_lang_h(struct vsb *sb)
 	vsb_cat(sb, "(struct sess *, double);\nconst char * VRT_r_beresp_pr");
 	vsb_cat(sb, "oto(const struct sess *);\nvoid VRT_l_beresp_proto(con");
 	vsb_cat(sb, "st struct sess *, const char *, ...);\n");
-	vsb_cat(sb, "int VRT_r_beresp_status(const struct sess *);\n");
+	vsb_cat(sb, "void VRT_l_beresp_saintmode(const struct sess *, doubl");
+	vsb_cat(sb, "e);\nint VRT_r_beresp_status(const struct sess *);\n");
 	vsb_cat(sb, "void VRT_l_beresp_status(const struct sess *, int);\n");
 	vsb_cat(sb, "const char * VRT_r_beresp_response(const struct sess *");
 	vsb_cat(sb, ");\nvoid VRT_l_beresp_response(const struct sess *, co");

lib/libvcl/vcc_gen_fixed_token.tcl

@@ -219,6 +219,11 @@ set spobj {
 	{ fetch }
 	"const struct sess *"
     }
+    { beresp.saintmode			TIME
+    	WO
+	{ fetch }
+	"const struct sess *"
+    }
     { beresp.status			INT
 	RW
 	{ fetch }

lib/libvcl/vcc_obj.c

 /*
- * $Id: vcc_gen_fixed_token.tcl 4188 2009-08-18 08:29:27Z phk $
+ * $Id$
  *
  * NB:  This file is machine generated, DO NOT EDIT!
  *
@@ -159,6 +159,11 @@ struct var vcc_vars[] = {
 	    V_RW,	    0,
 	    VCL_MET_FETCH
 	},
+	{ "beresp.saintmode", TIME, 16,
+	    NULL,	    "VRT_l_beresp_saintmode(sp, ",
+	    V_WO,	    0,
+	    VCL_MET_FETCH
+	},
 	{ "beresp.status", INT, 13,
 	    "VRT_r_beresp_status(sp)",	    "VRT_l_beresp_status(sp, ",
 	    V_RW,	    0,