Add support for directors reporting time of last health-change

bin/varnishd/cache/cache_backend.c

@@ -230,15 +230,14 @@ vbe_NewConn(void)
  * items would never time out once the threshold is reached.
  */
 
-static unsigned int
-vbe_Healthy(const struct vdi_simple *vs)
+unsigned
+VBE_Healthy(const struct backend *backend, double *changed)
 {
-	struct backend *backend;
-
-	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
-	backend = vs->backend;
 	CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC);
 
+	if (changed != NULL)
+		*changed = backend->health_changed;
+
 	if (backend->admin_health == ah_probe && !backend->healthy)
 		return (0);
 
@@ -301,7 +300,7 @@ vbe_GetVbe(struct busyobj *bo, struct vdi_simple *vs)
 		VBE_ReleaseConn(vc);
 	}
 
-	if (!vbe_Healthy(vs)) {
+	if (!VBE_Healthy(bp, NULL)) {
 		VSC_C_main->backend_unhealthy++;
 		return (NULL);
 	}
@@ -391,13 +390,16 @@ vdi_simple_getfd(const struct director *d, struct busyobj *bo)
 }
 
 static unsigned
-vdi_simple_healthy(const struct director *d)
+vdi_simple_healthy(const struct director *d, double *changed)
 {
 	struct vdi_simple *vs;
+	struct backend *be;
 
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC);
-	return (vbe_Healthy(vs));
+	be = vs->backend;
+	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
+	return (VBE_Healthy(be, changed));
 }
 
 static void

bin/varnishd/cache/cache_backend.h

@@ -77,7 +77,7 @@ struct vrt_backend_probe;
 
 typedef struct vbc *vdi_getfd_f(const struct director *, struct busyobj *);
 typedef void vdi_fini_f(const struct director *);
-typedef unsigned vdi_healthy(const struct director *);
+typedef unsigned vdi_healthy(const struct director *, double *changed);
 
 struct director {
 	unsigned		magic;
@@ -124,6 +124,7 @@ struct backend {
 	struct vbp_target	*probe;
 	unsigned		healthy;
 	enum admin_health	admin_health;
+	double			health_changed;
 
 	struct VSC_C_vbe	*vsc;
 };
@@ -156,6 +157,7 @@ void VBE_ReleaseConn(struct vbc *vc);
 void VBE_DropRefConn(struct backend *);
 void VBE_DropRefVcl(struct backend *);
 void VBE_DropRefLocked(struct backend *b);
+unsigned VBE_Healthy(const struct backend *b, double *changed);
 
 /* cache_backend_poll.c */
 void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p,

bin/varnishd/cache/cache_backend_cfg.c

@@ -43,6 +43,7 @@
 #include "vcli_priv.h"
 #include "vsa.h"
 #include "vrt.h"
+#include "vtim.h"
 
 /*
  * The list of backends is not locked, it is only ever accessed from
@@ -230,6 +231,7 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 	assert(b->ipv4 != NULL || b->ipv6 != NULL);
 
 	b->healthy = 1;
+	b->health_changed = VTIM_real();
 	b->admin_health = ah_probe;
 
 	VTAILQ_INSERT_TAIL(&backends, b, list);
@@ -415,6 +417,8 @@ do_list(struct cli *cli, struct backend *b, void *priv)
 		VBP_Summary(cli, b->probe);
 	}
 
+	/* XXX: report b->health_changed */
+
 	return (0);
 }
 
@@ -435,11 +439,16 @@ static int __match_proto__()
 do_set_health(struct cli *cli, struct backend *b, void *priv)
 {
 	enum admin_health state;
+	unsigned prev;
 
 	(void)cli;
 	state = *(enum admin_health*)priv;
 	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+	prev = VBE_Healthy(b, NULL);
 	b->admin_health = state;
+	if (prev != VBE_Healthy(b, NULL))
+		b->health_changed = VTIM_real();
+
 	return (0);
 }
 

bin/varnishd/cache/cache_backend_poll.c

@@ -280,13 +280,16 @@ vbp_has_poked(struct vbp_target *vt)
 	if (vt->good >= vt->probe.threshold) {
 		if (vt->backend->healthy)
 			logmsg = "Still healthy";
-		else
+		else {
 			logmsg = "Back healthy";
+			vt->backend->health_changed = VTIM_real();
+		}
 		vt->backend->healthy = 1;
 	} else {
-		if (vt->backend->healthy)
+		if (vt->backend->healthy) {
 			logmsg = "Went sick";
-		else
+			vt->backend->health_changed = VTIM_real();
+		} else
 			logmsg = "Still sick";
 		vt->backend->healthy = 0;
 	}

bin/varnishd/cache/cache_dir.c

@@ -116,10 +116,6 @@ VDI_GetFd(const struct director *d, struct busyobj *bo)
 }
 
 /* Check health ------------------------------------------------------
- *
- * The target is really an objhead pointer, but since it can not be
- * dereferenced during health-checks, we pass it as uintptr_t, which
- * hopefully will make people investigate, before mucking about with it.
  */
 
 int
@@ -127,5 +123,5 @@ VDI_Healthy(const struct director *d)
 {
 
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	return (d->healthy(d));
+	return (d->healthy(d, NULL));
 }

lib/libvmod_directors/fall_back.c

@@ -45,12 +45,12 @@ struct vmod_directors_fallback {
 };
 
 static unsigned __match_proto__(vdi_healthy)
-vmod_rr_healthy(const struct director *dir)
+vmod_rr_healthy(const struct director *dir, double *changed)
 {
 	struct vmod_directors_fallback *rr;
 
 	CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_FALLBACK_MAGIC);
-	return (vdir_any_healthy(rr->vd));
+	return (vdir_any_healthy(rr->vd, changed));
 }
 
 static struct vbc * __match_proto__(vdi_getfd_f)
@@ -65,7 +65,7 @@ vmod_rr_getfd(const struct director *dir, struct busyobj *bo)
 	for (u = 0; u < rr->vd->n_backend; u++) {
 		be = rr->vd->backend[u];
 		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
-		if (be->healthy(be))
+		if (be->healthy(be, NULL))
 			break;
 	}
 	vdir_unlock(rr->vd);

lib/libvmod_directors/random.c

@@ -50,12 +50,12 @@ struct vmod_directors_random {
 };
 
 static unsigned __match_proto__(vdi_healthy)
-vmod_rr_healthy(const struct director *dir)
+vmod_rr_healthy(const struct director *dir, double *changed)
 {
 	struct vmod_directors_random *rr;
 
 	CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_RANDOM_MAGIC);
-	return (vdir_any_healthy(rr->vd));
+	return (vdir_any_healthy(rr->vd, changed));
 }
 
 static struct vbc * __match_proto__(vdi_getfd_f)

lib/libvmod_directors/round_robin.c

@@ -46,12 +46,12 @@ struct vmod_directors_round_robin {
 };
 
 static unsigned __match_proto__(vdi_healthy)
-vmod_rr_healthy(const struct director *dir)
+vmod_rr_healthy(const struct director *dir, double *changed)
 {
 	struct vmod_directors_round_robin *rr;
 
 	CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC);
-	return (vdir_any_healthy(rr->vd));
+	return (vdir_any_healthy(rr->vd, changed));
 }
 
 static struct vbc * __match_proto__(vdi_getfd_f)
@@ -68,7 +68,7 @@ vmod_rr_getfd(const struct director *dir, struct busyobj *bo)
 		be = rr->vd->backend[rr->nxt];
 		rr->nxt++;
 		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
-		if (be->healthy(be))
+		if (be->healthy(be, NULL))
 			break;
 	}
 	vdir_unlock(rr->vd);

lib/libvmod_directors/vdir.c

@@ -128,21 +128,25 @@ vdir_add_backend(struct vdir *vd, VCL_BACKEND be, double weight)
 }
 
 unsigned
-vdir_any_healthy(struct vdir *vd)
+vdir_any_healthy(struct vdir *vd, double *changed)
 {
 	unsigned retval = 0;
 	VCL_BACKEND be;
 	unsigned u;
+	double c;
 
 	CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC);
 	vdir_lock(vd);
+	if (changed != NULL)
+		*changed = 0;
 	for (u = 0; u < vd->n_backend; u++) {
 		be = vd->backend[u];
 		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
-		if (be->healthy(be)) {
-			retval = 1;
+		retval = be->healthy(be, &c);
+		if (changed != NULL && c > *changed)
+			*changed = c;
+		if (retval)
 			break;
-		}
 	}
 	vdir_unlock(vd);
 	return (retval);
@@ -185,7 +189,7 @@ vdir_pick_be(struct vdir *vd, double w, unsigned nloops)
 		u = vdir_pick_by_weight(vd, w * tw, vbm);
 		be = vd->backend[u];
 		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
-		if (be->healthy(be))
+		if (be->healthy(be, NULL))
 			break;
 		if (l == 0) {
 			vbm = vd->vbm;

lib/libvmod_directors/vdir.h

@@ -47,5 +47,5 @@ void vdir_delete(struct vdir **vdp);
 void vdir_lock(struct vdir *vd);
 void vdir_unlock(struct vdir *vd);
 unsigned vdir_add_backend(struct vdir *vd, VCL_BACKEND be, double weight);
-unsigned vdir_any_healthy(struct vdir *vd);
+unsigned vdir_any_healthy(struct vdir *vd, double *changed);
 VCL_BACKEND vdir_pick_be(struct vdir *vd, double w, unsigned nloops);