Added health checks for individual servers within a balanced backend cluster.

The health checks work the same as for a general backend. The worse the health of
an individual server within a balanced backend, the less likely it will be to get new
connections.



git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@1940 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/cache.h

@@ -337,6 +337,7 @@ typedef void vbe_recycle_f(struct worker *w, struct vbe_conn *vc);
 typedef void vbe_init_f(void);
 typedef const char *vbe_gethostname_f(struct backend *);
 typedef void vbe_cleanup_f(struct backend *);
+typedef void vbe_updatehealth_f(struct sess *sp, struct vbe_conn *vc, int);
 
 struct backend_method {
 	const char		*name;
@@ -345,6 +346,7 @@ struct backend_method {
 	vbe_recycle_f		*recycle;
 	vbe_cleanup_f		*cleanup;
 	vbe_gethostname_f	*gethostname;
+	vbe_updatehealth_f	*updatehealth;
 	vbe_init_f		*init;
 };
 
@@ -396,6 +398,7 @@ void VBE_DropRefLocked(struct backend *);
 struct backend *VBE_NewBackend(struct backend_method *method);
 struct vbe_conn *VBE_NewConn(void);
 void VBE_ReleaseConn(struct vbe_conn *);
+void VBE_UpdateHealth(struct sess *sp, struct vbe_conn *, int);
 
 /* cache_backend_simple.c */
 extern struct backend_method	backend_method_simple;

bin/varnishd/cache_backend.c

@@ -231,6 +231,22 @@ VBE_RecycleFd(struct worker *w, struct vbe_conn *vc)
 	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
 }
 
+/* Update health ----------------------------------------------*/
+
+void
+VBE_UpdateHealth(struct sess *sp, struct vbe_conn *vc, int a)
+{
+	struct backend *b;
+
+	CHECK_OBJ_NOTNULL(vc, VBE_CONN_MAGIC);
+	CHECK_OBJ_NOTNULL(vc->backend, BACKEND_MAGIC);
+	b = vc->backend;
+	AN(b->method);
+	AN(b->method->updatehealth);
+	b->method->updatehealth(sp, vc, a);
+	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+}
+
 /*--------------------------------------------------------------------*/
 
 void

bin/varnishd/cache_backend_random.c

@@ -50,6 +50,7 @@ struct ber {
 	unsigned		magic;
 #define BER_MAGIC		0x645b03f4
 	struct brspec 		*blist;
+	int			count;
 #if 0
 	/* Store a hash of the backend info given in
 	 * vcl for comparison when a new vcl file is 
@@ -72,6 +73,7 @@ struct brspec {
 	double			dnstime;
 	unsigned		dnsseq;
 	TAILQ_HEAD(, vbe_conn)	connlist;
+	int			health;
 };
 
 /*--------------------------------------------------------------------*/
@@ -217,6 +219,8 @@ ber_nextfd(struct sess *sp)
 	struct ber *ber;
 	struct brspec *bs;
 	double r;
+	int min_health = -10;
+	int num = 0;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->backend, BACKEND_MAGIC);
@@ -230,6 +234,19 @@ ber_nextfd(struct sess *sp)
 		bs = bs->next;
 		CHECK_OBJ_NOTNULL(bs, BRSPEC_MAGIC);
 	}
+
+	/* If health is low (bad), use round-robin to find
+	 * a server with better health (if possible).
+	 */
+	while (bs->health < min_health) {
+		bs = bs->next;
+		num++;
+		if (num > ber->count) {
+			min_health *= 10;
+			num = 0;
+		}
+	}	
+
 	while (1) {
 		LOCK(&bp->mtx);
 		vc = TAILQ_FIRST(&bs->connlist);
@@ -378,6 +395,30 @@ ber_GetHostname(struct backend *b)
 
 /*--------------------------------------------------------------------*/
 
+static void
+ber_UpdateHealth(struct sess *sp, struct vbe_conn *vc, int add)
+{
+	struct brspec *bs, *first;
+	struct ber *ber;
+	
+	if (vc != NULL) {
+		CAST_OBJ_NOTNULL(bs, vc->priv, BRSPEC_MAGIC);
+	
+		if (bs->health + add >= -10000 || bs->health + add <= 10000)
+			bs->health += add;
+	} else {
+		CAST_OBJ_NOTNULL(ber, sp->backend->priv, BRSPEC_MAGIC);
+		first = ber->blist;
+		bs = first;
+		do {
+			bs = bs->next;
+			bs->health = (int)((double)bs->health / 2);
+		} while (bs != first);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
 static void
 ber_Init(void)
 {
@@ -392,6 +433,7 @@ struct backend_method backend_method_random = {
 	.close =		ber_ClosedFd,
 	.recycle =		ber_RecycleFd,
 	.gethostname =		ber_GetHostname,
+	.updatehealth =		ber_UpdateHealth,
 	.cleanup =		ber_Cleanup,
 	.init =			ber_Init
 };
@@ -426,6 +468,7 @@ VRT_init_random_backend(struct backend **bp, struct vrt_random_backend *t)
 	ber = calloc(sizeof *ber, 1);
 	XXXAN(ber);
 	ber->magic = BER_MAGIC;
+	ber->count = t->count;
 
 	b->priv = ber;
 
@@ -454,6 +497,7 @@ VRT_init_random_backend(struct backend **bp, struct vrt_random_backend *t)
 		bs->limit = limit;
 		
 		bs->dnsttl = 300;
+		bs->health = 0;
 		
 		if (bs_first == NULL)
 			bs_first = bs;

bin/varnishd/cache_backend_round_robin.c

@@ -50,6 +50,7 @@ struct brr {
 	unsigned		magic;
 #define BRR_MAGIC		0x66f05894
 	struct bspec 		*blist;
+	int			count;
 #if 0
 	/* Store a hash of the backend info given in
 	 * vcl for comparison when a new vcl file is 
@@ -71,6 +72,7 @@ struct bspec {
 	double			dnstime;
 	unsigned		dnsseq;
 	TAILQ_HEAD(, vbe_conn)	connlist;
+	int			health;
 };
 
 /*--------------------------------------------------------------------*/
@@ -215,13 +217,22 @@ brr_nextfd(struct sess *sp)
 	int reuse = 0;
 	struct brr *brr;
 	struct bspec *bs;
+	int min_health = -10;
+	int num = 0;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->backend, BACKEND_MAGIC);
 	bp = sp->backend;
 	CAST_OBJ_NOTNULL(brr, bp->priv, BRR_MAGIC);
 	
-	bs = brr->blist = brr->blist->next;
+	do {
+		bs = brr->blist = brr->blist->next;
+		num++;
+		if (num > brr->count) {
+			min_health *= 10;
+			num = 0;
+		}
+	} while (bs->health < min_health);
 	
 	while (1) {
 		LOCK(&bp->mtx);
@@ -371,6 +382,31 @@ brr_GetHostname(struct backend *b)
 
 /*--------------------------------------------------------------------*/
 
+static void
+brr_UpdateHealth(struct sess *sp, struct vbe_conn *vc, int add)
+{
+	struct bspec *bs, *first;
+	struct brr *brr;
+	
+	if (vc != NULL) {
+	
+		CAST_OBJ_NOTNULL(bs, vc->priv, BSPEC_MAGIC);
+	
+		if (bs->health + add >= -10000 || bs->health + add <= 10000)
+			bs->health += add;
+	} else {
+		CAST_OBJ_NOTNULL(brr, sp->backend->priv, BSPEC_MAGIC);
+		first = brr->blist;
+		bs = first;
+		do {
+			bs = bs->next;
+			bs->health = (int)((double)bs->health / 2);
+		} while (bs != first);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
 static void
 brr_Init(void)
 {
@@ -385,6 +421,7 @@ struct backend_method backend_method_round_robin = {
 	.close =		brr_ClosedFd,
 	.recycle =		brr_RecycleFd,
 	.gethostname =		brr_GetHostname,
+	.updatehealth =		brr_UpdateHealth,
 	.cleanup =		brr_Cleanup,
 	.init =			brr_Init
 };
@@ -417,6 +454,7 @@ VRT_init_round_robin_backend(struct backend **bp, struct vrt_round_robin_backend
 	brr = calloc(sizeof *brr, 1);
 	XXXAN(brr);
 	brr->magic = BRR_MAGIC;
+	brr->count = t->count;
 
 	b->priv = brr;
 
@@ -437,6 +475,7 @@ VRT_init_round_robin_backend(struct backend **bp, struct vrt_round_robin_backend
 		XXXAN(bs->hostname);
 		
 		bs->dnsttl = 300;
+		bs->health = 0;
 		
 		if (bs_first == NULL)
 			bs_first = bs;

bin/varnishd/cache_backend_simple.c

@@ -347,6 +347,21 @@ bes_GetHostname(struct backend *b)
 
 /*--------------------------------------------------------------------*/
 
+static void
+bes_UpdateHealth(struct sess *sp, struct vbe_conn *vc, int a)
+{
+	(void)sp;
+	(void)vc;
+	(void)a;
+	
+	/* 
+	 * Not of any use for simple backend. The global health
+	 * parameter of the backend should be enough.
+	 */
+}
+
+/*--------------------------------------------------------------------*/
+
 static void
 bes_Init(void)
 {
@@ -361,6 +376,7 @@ struct backend_method backend_method_simple = {
 	.close =		bes_ClosedFd,
 	.recycle =		bes_RecycleFd,
 	.gethostname =		bes_GetHostname,
+	.updatehealth =		bes_UpdateHealth,
 	.cleanup =		bes_Cleanup,
 	.init =			bes_Init
 };

bin/varnishd/cache_center.c

@@ -434,6 +434,7 @@ cnt_hit(struct sess *sp)
 	if (minutes > sp->backend->minute_limit) {
 		sp->backend->minute_limit++;
 		sp->backend->health = (int)((double)sp->backend->health / 2);
+		VBE_UpdateHealth(sp, NULL, 0);
 	}
 
 	VCL_hit_method(sp);

bin/varnishd/cache_fetch.c

@@ -377,6 +377,12 @@ Fetch(struct sess *sp)
 		cls = 1;
 
 	CHECK_OBJ_NOTNULL(sp->backend, BACKEND_MAGIC);
+	
+	if (http_GetStatus(sp->bereq->http) == 200)
+		VBE_UpdateHealth(sp, vc, 1);
+	else if(http_GetStatus(sp->bereq->http) == 504)
+		VBE_UpdateHealth(sp, vc, -1);
+	
 	if (cls)
 		VBE_ClosedFd(sp->wrk, vc);
 	else

include/vcl.h

@@ -21,7 +21,7 @@ struct VCL_conf {
         struct vrt_ref  *ref;
         unsigned        nref;
         unsigned        busy;
-
+        
 	unsigned	nsrc;
 	const char	**srcname;
 	const char	**srcbody;

include/vrt.h

@@ -55,6 +55,7 @@ struct vrt_backend_entry {
 
 struct vrt_round_robin_backend {
 	const char	*name;
+	unsigned	count;
 	struct vrt_backend_entry *bentry;
 };
 

lib/libvcl/vcc_backend.c

@@ -311,6 +311,7 @@ vcc_ParseBalancedBackend(struct tokenlist *tl)
 			Fc(tl, 0, "\nstatic struct vrt_round_robin_backend sbe_%.*s = {\n",
 			    PF(t_be));
 			Fc(tl, 0, "\t.name = \"%.*s\",\n", PF(t_be));
+			Fc(tl, 0, "\t.count = %d,\n", cnt);
 			Fc(tl, 0, "\t.bentry = &bentry_%.*s_%d\n", PF(t_be), cnt-1);
 			Fc(tl, 0, "};\n");
 			Fi(tl, 0, "\tVRT_init_round_robin_backend(&VGC_backend_%.*s , &sbe_%.*s);\n",

lib/libvcl/vcc_fixed_token.c

@@ -433,6 +433,7 @@ vcl_output_lang_h(struct vsb *sb)
 	vsb_cat(sb, "\n");
 	vsb_cat(sb, "struct vrt_round_robin_backend {\n");
 	vsb_cat(sb, "	const char	*name;\n");
+	vsb_cat(sb, "	unsigned	count;\n");
 	vsb_cat(sb, "	struct vrt_backend_entry *bentry;\n");
 	vsb_cat(sb, "};\n");
 	vsb_cat(sb, "\n");