Pull in vdir random and its tests from varnish-cache master

As of cf3c9d9213c12e3c58a391b068398f8fd1d1c17c

src/Makefile.am

@@ -4,6 +4,9 @@ AM_LDFLAGS = $(VARNISHAPI_LIBS) $(VMOD_LDFLAGS)
 vmod_LTLIBRARIES = libvmod_weightadjust.la
 
 libvmod_weightadjust_la_SOURCES = \
+	vdir.c \
+	vdir.h \
+	random.c \
 	vmod_weightadjust.c
 
 nodist_libvmod_weightadjust_la_SOURCES = \

src/random.c

+/*-
+ * Copyright (c) 2013-2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+
+#include "cache/cache.h"
+#include "cache/cache_director.h"
+
+#include "vbm.h"
+#include "vrnd.h"
+#include "vrt.h"
+
+#include "vdir.h"
+
+#include "vcc_if.h"
+
+struct vmod_directors_random {
+	unsigned				magic;
+#define VMOD_DIRECTORS_RANDOM_MAGIC		0x4732d092
+	struct vdir				*vd;
+};
+
+static unsigned __match_proto__(vdi_healthy)
+vmod_random_healthy(const struct director *dir, const struct busyobj *bo,
+    double *changed)
+{
+	struct vmod_directors_random *rr;
+
+	CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_RANDOM_MAGIC);
+	return (vdir_any_healthy(rr->vd, bo, changed));
+}
+
+static const struct director * __match_proto__(vdi_resolve_f)
+vmod_random_resolve(const struct director *dir, struct worker *wrk,
+    struct busyobj *bo)
+{
+	struct vmod_directors_random *rr;
+	VCL_BACKEND be;
+	double r;
+
+	CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_RANDOM_MAGIC);
+	r = scalbn(VRND_RandomTestable(), -31);
+	assert(r >= 0 && r < 1.0);
+	be = vdir_pick_be(rr->vd, r, bo);
+	return (be);
+}
+
+VCL_VOID __match_proto__()
+vmod_random__init(VRT_CTX, struct vmod_directors_random **rrp,
+    const char *vcl_name)
+{
+	struct vmod_directors_random *rr;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AN(rrp);
+	AZ(*rrp);
+	ALLOC_OBJ(rr, VMOD_DIRECTORS_RANDOM_MAGIC);
+	AN(rr);
+	*rrp = rr;
+	vdir_new(&rr->vd, "random", vcl_name, vmod_random_healthy,
+	    vmod_random_resolve, rr);
+}
+
+VCL_VOID __match_proto__()
+vmod_random__fini(struct vmod_directors_random **rrp)
+{
+	struct vmod_directors_random *rr;
+
+	rr = *rrp;
+	*rrp = NULL;
+	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_RANDOM_MAGIC);
+	vdir_delete(&rr->vd);
+	FREE_OBJ(rr);
+}
+
+VCL_VOID __match_proto__()
+vmod_random_add_backend(VRT_CTX,
+    struct vmod_directors_random *rr, VCL_BACKEND be, double w)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_RANDOM_MAGIC);
+	(void)vdir_add_backend(rr->vd, be, w);
+}
+
+VCL_VOID vmod_random_remove_backend(VRT_CTX,
+    struct vmod_directors_random *rr, VCL_BACKEND be) __match_proto__()
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_RANDOM_MAGIC);
+	vdir_remove_backend(rr->vd, be, NULL);
+}
+
+VCL_BACKEND __match_proto__()
+vmod_random_backend(VRT_CTX, struct vmod_directors_random *rr)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_RANDOM_MAGIC);
+	return (rr->vd->dir);
+}

src/tests/d00002.vtc

+varnishtest "Test random director"
+
+server s1 {
+	rxreq
+	txresp -body "foo1"
+	rxreq
+	txresp -body "bar1"
+} -start
+
+varnish v1 -vcl+backend {
+	import directors;
+
+	sub vcl_init {
+		new foo = directors.random();
+		foo.add_backend(s1, 1);
+		foo.add_backend(s1, 1);
+	}
+
+	sub vcl_backend_fetch {
+		set bereq.backend = foo.backend();
+	}
+
+	sub vcl_backend_response {
+		set beresp.http.where = bereq.backend + "-->" + beresp.backend;
+	}
+} -start
+
+client c1 {
+	txreq -url "/foo"
+	rxresp
+	expect resp.http.where == "foo-->s1"
+	txreq -url "/bar"
+	rxresp
+	expect resp.http.where == "foo-->s1"
+} -run
+
+server s1 -start
+server s2 {
+	loop 20 {
+		rxreq
+		txresp
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	import directors;
+
+	sub vcl_init {
+		new foo = directors.random();
+		foo.add_backend(s1, 1);
+		foo.add_backend(s2, 1);
+	}
+
+	sub vcl_recv {
+		if (req.method == "DELETE") {
+			foo.remove_backend(s1);
+			return(synth(204));
+		}
+		set req.backend_hint = foo.backend();
+		return (pass);
+	}
+
+	sub vcl_backend_response {
+		set beresp.http.where = bereq.backend + "-->" + beresp.backend;
+	}
+}
+
+client c1 {
+	txreq -req "DELETE"
+	rxresp
+	expect resp.status == 204
+	loop 20 {
+		txreq
+		rxresp
+		expect resp.status == 200
+		expect resp.http.where == "foo-->s2"
+	}
+} -run

src/tests/d00004.vtc

+varnishtest "Deeper test of random director"
+
+server s1 {
+	rxreq
+	txresp -body "1"
+	rxreq
+	txresp -body "1"
+} -start
+
+server s2 {
+	rxreq
+	txresp -body "22"
+	rxreq
+	txresp -body "22"
+} -start
+
+server s3 {
+	rxreq
+	txresp -body "333"
+	rxreq
+	txresp -body "333"
+	rxreq
+	txresp -body "333"
+	rxreq
+	txresp -body "333"
+} -start
+
+server s4 {
+	rxreq
+	txresp -body "4444"
+	rxreq
+	txresp -body "4444"
+} -start
+
+varnish v1 -vcl+backend {
+	import directors;
+
+	sub vcl_init {
+		new foo = directors.random();
+		foo.add_backend(s1, 1);
+		foo.add_backend(s2, 1);
+		foo.add_backend(s3, 1);
+		foo.add_backend(s4, 1);
+	}
+
+	sub vcl_recv {
+		return(pass);
+	}
+
+	sub vcl_backend_fetch {
+		set bereq.backend = foo.backend();
+	}
+} -start
+
+varnish v1 -cliok "debug.srandom"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.bodylen == 3
+	txreq
+	rxresp
+	expect resp.bodylen == 4
+	txreq
+	rxresp
+	expect resp.bodylen == 1
+	txreq
+	rxresp
+	expect resp.bodylen == 1
+	txreq
+	rxresp
+	expect resp.bodylen == 2
+	txreq
+	rxresp
+	expect resp.bodylen == 2
+	txreq
+	rxresp
+	expect resp.bodylen == 3
+	txreq
+	rxresp
+	expect resp.bodylen == 4
+	txreq
+	rxresp
+	expect resp.bodylen == 3
+	txreq
+	rxresp
+	expect resp.bodylen == 3
+} -run

src/tests/test01.vtc

-varnishtest "Test weightadjust vmod"
-
-server s1 {
-       rxreq
-       txresp
-} -start
-
-varnish v1 -vcl+backend {
-	import std;
-	import ${vmod_weightadjust};
-
-	sub vcl_init {
-		std.log(weightadjust.info());
-	}
-
-	sub vcl_deliver {
-		set resp.http.hello = weightadjust.hello("World");
-		set resp.http.info = weightadjust.info();
-	}
-} -start
-
-logexpect l1 -v v1 -g raw -d 1 {
-	expect 0 0    CLI     {^Rd vcl.load}
-	expect 0 =    VCL_Log {^vmod_weightadjust loaded at }
-} -start
-
-client c1 {
-	txreq -url "/"
-	rxresp
-	expect resp.http.hello == "Hello, World"
-	expect resp.http.info  ~  "^vmod_weightadjust warmed at "
-}
-
-client c1 -run
-
-logexpect l1 -wait

src/vdir.c

+/*-
+ * Copyright (c) 2013-2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+
+#include "cache/cache.h"
+#include "cache/cache_director.h"
+
+#include "vrt.h"
+#include "vbm.h"
+
+#include "vdir.h"
+
+static void
+vdir_expand(struct vdir *vd, unsigned n)
+{
+	CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC);
+
+	vd->backend = realloc(vd->backend, n * sizeof *vd->backend);
+	AN(vd->backend);
+	vd->weight = realloc(vd->weight, n * sizeof *vd->weight);
+	AN(vd->weight);
+	vd->l_backend = n;
+}
+
+void
+vdir_new(struct vdir **vdp, const char *name, const char *vcl_name,
+    vdi_healthy_f *healthy, vdi_resolve_f *resolve, void *priv)
+{
+	struct vdir *vd;
+
+	AN(name);
+	AN(vcl_name);
+	AN(vdp);
+	AZ(*vdp);
+	ALLOC_OBJ(vd, VDIR_MAGIC);
+	AN(vd);
+	*vdp = vd;
+	AZ(pthread_rwlock_init(&vd->mtx, NULL));
+
+	ALLOC_OBJ(vd->dir, DIRECTOR_MAGIC);
+	AN(vd->dir);
+	vd->dir->name = name;
+	REPLACE(vd->dir->vcl_name, vcl_name);
+	vd->dir->priv = priv;
+	vd->dir->healthy = healthy;
+	vd->dir->resolve = resolve;
+	vd->vbm = vbit_new(8);
+	AN(vd->vbm);
+}
+
+void
+vdir_delete(struct vdir **vdp)
+{
+	struct vdir *vd;
+
+	TAKE_OBJ_NOTNULL(vd, vdp, VDIR_MAGIC);
+
+	free(vd->backend);
+	free(vd->weight);
+	AZ(pthread_rwlock_destroy(&vd->mtx));
+	free(vd->dir->vcl_name);
+	FREE_OBJ(vd->dir);
+	vbit_destroy(vd->vbm);
+	FREE_OBJ(vd);
+}
+
+void
+vdir_rdlock(struct vdir *vd)
+{
+	CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC);
+	AZ(pthread_rwlock_rdlock(&vd->mtx));
+}
+
+void
+vdir_wrlock(struct vdir *vd)
+{
+	CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC);
+	AZ(pthread_rwlock_wrlock(&vd->mtx));
+}
+
+void
+vdir_unlock(struct vdir *vd)
+{
+	CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC);
+	AZ(pthread_rwlock_unlock(&vd->mtx));
+}
+
+
+unsigned
+vdir_add_backend(struct vdir *vd, VCL_BACKEND be, double weight)
+{
+	unsigned u;
+
+	CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC);
+	AN(be);
+	vdir_wrlock(vd);
+	if (vd->n_backend >= vd->l_backend)
+		vdir_expand(vd, vd->l_backend + 16);
+	assert(vd->n_backend < vd->l_backend);
+	u = vd->n_backend++;
+	vd->backend[u] = be;
+	vd->weight[u] = weight;
+	vd->total_weight += weight;
+	vdir_unlock(vd);
+	return (u);
+}
+
+void
+vdir_remove_backend(struct vdir *vd, VCL_BACKEND be, unsigned *cur)
+{
+	unsigned u, n;
+
+	CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC);
+	if (be == NULL)
+		return;
+	CHECK_OBJ(be, DIRECTOR_MAGIC);
+	vdir_wrlock(vd);
+	for (u = 0; u < vd->n_backend; u++)
+		if (vd->backend[u] == be)
+			break;
+	if (u == vd->n_backend) {
+		vdir_unlock(vd);
+		return;
+	}
+	vd->total_weight -= vd->weight[u];
+	n = (vd->n_backend - u) - 1;
+	memmove(&vd->backend[u], &vd->backend[u+1], n * sizeof(vd->backend[0]));
+	memmove(&vd->weight[u], &vd->weight[u+1], n * sizeof(vd->weight[0]));
+	vd->n_backend--;
+
+	if (cur) {
+		assert(*cur <= vd->n_backend);
+		if (u < *cur)
+			(*cur)--;
+		else if (*cur == vd->n_backend)
+			*cur = 0;
+	}
+	vdir_unlock(vd);
+}
+
+unsigned
+vdir_any_healthy(struct vdir *vd, const struct busyobj *bo, double *changed)
+{
+	unsigned retval = 0;
+	VCL_BACKEND be;
+	unsigned u;
+	double c;
+
+	CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC);
+	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
+	vdir_rdlock(vd);
+	if (changed != NULL)
+		*changed = 0;
+	for (u = 0; u < vd->n_backend; u++) {
+		be = vd->backend[u];
+		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
+		retval = be->healthy(be, bo, &c);
+		if (changed != NULL && c > *changed)
+			*changed = c;
+		if (retval)
+			break;
+	}
+	vdir_unlock(vd);
+	return (retval);
+}
+
+static unsigned
+vdir_pick_by_weight(const struct vdir *vd, double w,
+    const struct vbitmap *blacklist)
+{
+	double a = 0.0;
+	VCL_BACKEND be = NULL;
+	unsigned u;
+
+	AN(blacklist);
+	for (u = 0; u < vd->n_backend; u++) {
+		be = vd->backend[u];
+		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
+		if (vbit_test(blacklist, u))
+			continue;
+		a += vd->weight[u];
+		if (w < a)
+			return (u);
+	}
+	WRONG("");
+}
+
+VCL_BACKEND
+vdir_pick_be(struct vdir *vd, double w, const struct busyobj *bo)
+{
+	unsigned u;
+	double tw = 0.0;
+	VCL_BACKEND be = NULL;
+
+	vdir_wrlock(vd);
+	for (u = 0; u < vd->n_backend; u++) {
+		if (vd->backend[u]->healthy(vd->backend[u], bo, NULL)) {
+			vbit_clr(vd->vbm, u);
+			tw += vd->weight[u];
+		} else
+			vbit_set(vd->vbm, u);
+	}
+	if (tw > 0.0) {
+		u = vdir_pick_by_weight(vd, w * tw, vd->vbm);
+		assert(u < vd->n_backend);
+		be = vd->backend[u];
+		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
+	}
+	vdir_unlock(vd);
+	return (be);
+}

src/vdir.h

+/*-
+ * Copyright (c) 2013-2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+struct vbitmap;
+
+struct vdir {
+	unsigned				magic;
+#define VDIR_MAGIC				0x99f4b726
+	pthread_rwlock_t			mtx;
+	unsigned				n_backend;
+	unsigned				l_backend;
+	VCL_BACKEND				*backend;
+	double					*weight;
+	double					total_weight;
+	struct director				*dir;
+	struct vbitmap				*vbm;
+};
+
+void vdir_new(struct vdir **vdp, const char *name, const char *vcl_name,
+    vdi_healthy_f *healthy, vdi_resolve_f *resolve, void *priv);
+void vdir_delete(struct vdir **vdp);
+void vdir_rdlock(struct vdir *vd);
+void vdir_wrlock(struct vdir *vd);
+void vdir_unlock(struct vdir *vd);
+unsigned vdir_add_backend(struct vdir *, VCL_BACKEND be, double weight);
+void vdir_remove_backend(struct vdir *, VCL_BACKEND be, unsigned *cur);
+unsigned vdir_any_healthy(struct vdir *, const struct busyobj *,
+    double *changed);
+VCL_BACKEND vdir_pick_be(struct vdir *, double w, const struct busyobj *);