shard director: LAZY mode (vdi resolve function), parameter objects

We introduce a shard_param object to hold the shard director lookup
parameters which until now could only be passed to the .backend()
method.

By associating a parameter object with a shard director, we enable
LAZY lookups as with the other directors. Parameter objects are
defined with VCL scope (normal vmod objects), but can be overridden
per backend request using a task priv.

We use the same concept to carry shard.backend() parameters to
vdi resolve for LAZY mode: They get saved in a per-director task
scope parameter object.

Each object points to another object providing defaults for
values which are not defined.

Actual resolution of the various parameter objects does not happen
before they are used to allow changing them independently (ie,
shard .backend() parameters have precedence over an associated
parameter object, which by itself can be overridden).

Overview of parameter objects (pointers are alternatives)

shard() director	shard_param() object	default praram

	     --------------------------------->	  vmod static
  VCL obj   /				     ->
  .param  -+--------->	  VCL obj	    /  _
  			  .default  --------   /|
					      /
			     ^		     /
			     |		    /
					   /
			  .default	  /
	------------->	  TASK priv	 /
       /				/
  .default -----------------------------
  TASK priv

bin/varnishtest/tests/d00021.vtc

+varnishtest "shard director LAZY"
+
+server s1 {
+	rxreq
+	txresp -body "ech3Ooj"
+} -start
+
+server s2 {
+	rxreq
+	txresp -body "ieQu2qua"
+} -start
+
+server s3 {
+	rxreq
+	txresp -body "xiuFi3Pe"
+} -start
+
+varnish v1 -vcl+backend {
+	import std;
+	import directors;
+
+	sub vcl_init {
+	    new vd = directors.shard();
+	    if (!vd.add_backend(s1)) {
+		std.log("add s1 failed");
+	    }
+	    if (!vd.add_backend(s2)) {
+		std.log("add s2 failed");
+	    }
+	    if (!vd.add_backend(s3)) {
+		std.log("add s3 failed");
+	    }
+	    if (!vd.reconfigure(replicas=25)) {
+		std.log("reconfigure failed");
+	    }
+	}
+
+	sub vcl_recv {
+	    return(pass);
+	}
+
+	sub vcl_backend_fetch {
+	    if (bereq.url == "/1") {
+		set bereq.backend =
+		    vd.backend(resolve=LAZY, by=KEY, key=1);
+	    }
+	    if (bereq.url == "/2") {
+		set bereq.backend =
+		  vd.backend(resolve=LAZY, by=KEY, key=2147483647);
+	    }
+	    if (bereq.url == "/3") {
+		set bereq.backend =
+		  vd.backend(resolve=LAZY, by=KEY, key=4294967295);
+	    }
+	}
+
+	sub vcl_backend_response {
+	    set beresp.http.healthy = std.healthy(
+	      vd.backend(resolve=LAZY, by=KEY, key=1));
+	}
+} -start
+
+
+client c1 {
+	txreq -url /1
+	rxresp
+	expect resp.body == "ech3Ooj"
+	expect resp.http.healthy == "true"
+
+	txreq -url /2
+	rxresp
+	expect resp.body == "ieQu2qua"
+	expect resp.http.healthy == "true"
+
+	txreq -url /3
+	rxresp
+	expect resp.body == "xiuFi3Pe"
+	expect resp.http.healthy == "true"
+} -run

bin/varnishtest/tests/d00029.vtc

+varnishtest "shard director LAZY - d18.vtc"
+
+server s1 {
+	rxreq
+	txresp -body "ech3Ooj"
+} -start
+
+server s2 {
+	rxreq
+	txresp -body "ieQu2qua"
+} -start
+
+server s3 {
+	rxreq
+	txresp -body "xiuFi3Pe"
+} -start
+
+varnish v1 -vcl+backend {
+	import std;
+	import directors;
+
+	sub vcl_init {
+	    new vd = directors.shard();
+	    if (!vd.add_backend(s1)) {
+		std.log("add s1 failed");
+	    }
+	    if (!vd.add_backend(s2)) {
+		std.log("add s2 failed");
+	    }
+	    if (!vd.add_backend(s3)) {
+		std.log("add s3 failed");
+	    }
+	    if (!vd.reconfigure(replicas=25)) {
+		std.log("reconfigure failed");
+	    }
+
+	    vd.debug(1);
+
+	    new p = directors.shard_param();
+	    p.set(by=KEY, key=1);
+	    vd.associate(p.use());
+
+	    new p3 = directors.shard_param();
+	    p3.set(by=KEY, key=4294967295);
+	}
+
+	sub vcl_recv {
+	    return(pass);
+	}
+
+	sub vcl_backend_fetch {
+	    set bereq.backend=vd.backend(resolve=LAZY);
+
+	    if (bereq.url == "/1") {
+		# default
+	    } else
+	    if (bereq.url == "/2") {
+		# backend override parameter set
+		p.set(by=KEY, key=2147483647);
+	    } else
+	    if (bereq.url == "/3") {
+		# backend override association
+		vd.backend(resolve=LAZY, param=p3.use());
+	    }
+	}
+
+	sub vcl_backend_response {
+	    set beresp.http.backend = bereq.backend;
+	}
+} -start
+
+
+client c1 {
+	txreq -url /1
+	rxresp
+	expect resp.body == "ech3Ooj"
+	expect resp.http.backend == "vd"
+
+	txreq -url /2
+	rxresp
+	expect resp.body == "ieQu2qua"
+	expect resp.http.backend == "vd"
+
+	txreq -url /3
+	rxresp
+	expect resp.body == "xiuFi3Pe"
+	expect resp.http.backend == "vd"
+} -run

bin/varnishtest/tests/d00030.vtc

+varnishtest "shard director error handling"
+
+varnish v1 -vcl {
+    import directors;
+    import blob;
+
+    backend dummy { .host = "${bad_backend}"; }
+
+    sub vcl_init {
+	new shard = directors.shard();
+	new p = directors.shard_param();
+	p.set(by=BLOB, key_blob=blob.decode(HEX, encoded=""));
+    }
+
+    sub vcl_recv {
+	if (req.url == "/1") {
+	    set req.backend_hint = shard.backend(
+	      param=blob.decode(HEX, encoded=""));
+	} else if (req.url == "/2") {
+	    p.set(by=HASH);
+	}
+    }
+} -start
+
+logexpect l1 -v v1 -g raw -d 1 {
+    expect 0 0    CLI       {^Rd vcl.load}
+    expect 0 0    Error     {by=BLOB but no or empty key_blob - using key 0}
+} -start -wait
+
+logexpect l2 -v v1 -g raw {
+    expect * 1001 VCL_Error {shard .backend param invalid}
+} -start
+logexpect l3 -v v1 -g raw {
+    expect * 1003 VCL_Error {shard_param.set.. may only be used in vcl_init and in backend context}
+} -start
+
+client c1 {
+    txreq -url "/1"
+    rxresp
+    expect resp.status == 503
+    expect_close
+} -run
+
+client c1 {
+    txreq -url "/2"
+    rxresp
+    expect resp.status == 503
+    expect_close
+} -run
+
+logexpect l2 -wait
+logexpect l3 -wait
+
+varnish v1 -errvcl {shard .associate param invalid} {
+    import directors;
+    import blob;
+
+    backend dummy { .host = "${bad_backend}"; }
+
+    sub vcl_init {
+	new shard = directors.shard();
+	shard.associate(blob.decode(encoded=""));
+    }
+}
+
+varnish v1 -errvcl {missing key argument with by=KEY} {
+    import directors;
+    import blob;
+
+    backend dummy { .host = "${bad_backend}"; }
+
+    sub vcl_init {
+	new p = directors.shard_param();
+	p.set(by=KEY);
+    }
+}
+
+varnish v1 -errvcl {invalid key argument -5 with by=KEY} {
+    import directors;
+    import blob;
+
+    backend dummy { .host = "${bad_backend}"; }
+
+    sub vcl_init {
+	new p = directors.shard_param();
+	p.set(by=KEY, key=-5);
+    }
+}
+
+varnish v1 -errvcl {missing key_blob argument with by=BLOB} {
+    import directors;
+    import blob;
+
+    backend dummy { .host = "${bad_backend}"; }
+
+    sub vcl_init {
+	new p = directors.shard_param();
+	p.set(by=BLOB);
+    }
+}
+
+varnish v1 -errvcl {key and key_blob arguments are invalid with by=URL} {
+    import directors;
+    import blob;
+
+    backend dummy { .host = "${bad_backend}"; }
+
+    sub vcl_init {
+	new p = directors.shard_param();
+	p.set(by=URL, key=0);
+    }
+}
+
+varnish v1 -errvcl {key and key_blob arguments are invalid with by=HASH (default)} {
+    import directors;
+    import blob;
+
+    backend dummy { .host = "${bad_backend}"; }
+
+    sub vcl_init {
+	new p = directors.shard_param();
+	p.set(key=0);
+    }
+}
+
+varnish v1 -errvcl {invalid alt argument -1} {
+    import directors;
+    import blob;
+
+    backend dummy { .host = "${bad_backend}"; }
+
+    sub vcl_init {
+	new p = directors.shard_param();
+	p.set(alt=-1);
+    }
+}
+
+varnish v1 -errvcl {invalid warmup argument -0.5} {
+    import directors;
+    import blob;
+
+    backend dummy { .host = "${bad_backend}"; }
+
+    sub vcl_init {
+	new p = directors.shard_param();
+	p.set(warmup=-0.5);
+    }
+}
+
+varnish v1 -errvcl {invalid warmup argument 1.1} {
+    import directors;
+    import blob;
+
+    backend dummy { .host = "${bad_backend}"; }
+
+    sub vcl_init {
+	new p = directors.shard_param();
+	p.set(warmup=1.1);
+    }
+}
+
+varnish v1 -errvcl {resolve=LAZY with other parameters can only be used in backend context} {
+    import directors;
+    import blob;
+
+    backend dummy { .host = "${bad_backend}"; }
+
+    sub vcl_init {
+	new shard = directors.shard();
+	new rr = directors.round_robin();
+	rr.add_backend(shard.backend(resolve=LAZY, by=KEY));
+    }
+}
+
+varnish v1 -errvcl {resolve=NOW can not be used in vcl_init} {
+    import directors;
+    import blob;
+
+    backend dummy { .host = "${bad_backend}"; }
+
+    sub vcl_init {
+	new shard = directors.shard();
+	new rr = directors.round_robin();
+	rr.add_backend(shard.backend());
+    }
+}

doc/changes.rst

@@ -104,6 +104,31 @@ VCL and bundled VMODs
       <dir>.backend(by=BLOB, key_blob=blobdigest.hash(RS,
 	blob.decode(encoded=<string>)))
 
+* The ``shard`` director now offers resolution at the time the actual
+  backend connection is made, which is how all other bundled directors
+  work as well: With the ``resolve=LAZY`` argument, other shard
+  parameters are saved for later reference and a director object is
+  returned.
+
+  This enables layering the shard director below other directors.
+
+* The ``shard`` director now also supports getting other parameters
+  from a parameter set object: Rather than passing the required
+  parameters with each ``.backend()`` call, an object can be
+  associated with a shard director defining the parameters. The
+  association can be changed in ``vcl_backend_fetch()`` and individual
+  parameters can be overridden in each ``.backend()`` call.
+
+  The main use case is to segregate shard parameters from director
+  selection: By associating a parameter object with many directors,
+  the same load balancing decision can easily be applied independent
+  of which set of backends is to be used.
+
+* To support parameter overriding, support for positional arguments of
+  the shard director ``.backend()`` method had to be removed. In other
+  words, all parameters to the shard director ``.backend()`` method
+  now need to be named.
+
 Logging / statistics
 --------------------
 

lib/libvmod_directors/Makefile.am

@@ -13,7 +13,8 @@ libvmod_directors_la_SOURCES = \
 	shard_dir.c \
 	shard_dir.h \
 	tbl_by.h \
-	tbl_healthy.h
+	tbl_healthy.h \
+	tbl_resolve.h
 
 # Use vmodtool.py generated automake boilerplate
 include $(srcdir)/automake_boilerplate.am

lib/libvmod_directors/shard_dir.c

@@ -304,27 +304,39 @@ init_state(struct shard_state *state,
 	state->last.hostid = -1;
 }
 
+/* basically same as vdir_any_healthy
+ * - XXX we should embed a vdir
+ * - XXX should we return the health state of the actual backend
+ *   for healthy=IGNORE ?
+ */
+unsigned
+sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo,
+    double *changed)
+{
+	unsigned retval = 0;
+	VCL_BACKEND be;
+	unsigned u;
+	double c;
+
+	CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC);
+	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
+	sharddir_rdlock(shardd);
+	if (changed != NULL)
+		*changed = 0;
+	for (u = 0; u < shardd->n_backend; u++) {
+		be = shardd->backend[u].backend;
+		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
+		retval = be->healthy(be, bo, &c);
+		if (changed != NULL && c > *changed)
+			*changed = c;
+		if (retval)
+			break;
+	}
+	sharddir_unlock(shardd);
+	return (retval);
+}
 /*
- * core function for the director backend method
- *
- * while other directors return a reference to their own backend object (on
- * which varnish will call the resolve method to resolve to a non-director
- * backend), this director immediately reolves in the backend method, to make
- * the director choice visible in VCL
- *
- * consequences:
- * - we need no own struct director
- * - we can only respect a busy object when being called on the backend side,
- *   which probably is, for all practical purposes, only relevant when the
- *   saintmode vmod is used
- *
- * if we wanted to offer delayed resolution, we'd need something like
- * per-request per-director state or we'd need to return a dynamically created
- * director object. That should be straight forward once we got director
- * refcounting #2072. Until then, we could create it on the workspace, but then
- * we'd need to keep other directors from storing any references to our dynamic
- * object for longer than the current task
- *
+ * core function for the director backend/resolve method
  */
 VCL_BACKEND
 sharddir_pick_be(VRT_CTX, struct sharddir *shardd,

lib/libvmod_directors/shard_dir.h

@@ -41,6 +41,13 @@ enum healthy_e {
 	_HEALTHY_E_MAX
 };
 
+enum resolve_e {
+	_RESOLVE_E_INVALID = 0,
+#define VMODENUM(x) x,
+#include "tbl_resolve.h"
+	_RESOLVE_E_MAX
+};
+
 struct vbitmap;
 
 struct shard_circlepoint {
@@ -121,6 +128,8 @@ void sharddir_new(struct sharddir **sharddp, const char *vcl_name);
 void sharddir_delete(struct sharddir **sharddp);
 void sharddir_wrlock(struct sharddir *shardd);
 void sharddir_unlock(struct sharddir *shardd);
+unsigned sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo,
+    double *changed);
 VCL_BACKEND sharddir_pick_be(VRT_CTX, struct sharddir *, uint32_t, VCL_INT,
    VCL_REAL, VCL_BOOL, enum healthy_e);
 

lib/libvmod_directors/tbl_resolve.h

+VMODENUM(NOW)
+VMODENUM(LAZY)
+#undef VMODENUM

lib/libvmod_directors/vmod.vcc

@@ -3,7 +3,7 @@
 # itself. See LICENCE for details.
 #
 # Copyright (c) 2013-2015 Varnish Software AS
-# Copyright 2009-2016 UPLEX - Nils Goroll Systemoptimierung
+# Copyright 2009-2018 UPLEX - Nils Goroll Systemoptimierung
 # All rights reserved.
 #
 # Authors: Poul-Henning Kamp <phk@FreeBSD.org>
@@ -233,6 +233,9 @@ Note that the shard director needs to be configured using at least one
 ``shard.add_backend()`` call(s) **followed by a**
 ``shard.reconfigure()`` **call** before it can hand out backends.
 
+_Note_ that due to various restrictions (documented below), it is
+recommended to use the shard director on the backend side.
+
 Introduction
 ````````````
 
@@ -354,6 +357,16 @@ Set the default rampup duration. See `rampup` parameter of
 
 Default: 0s (no rampup)
 
+$Method VOID .associate(BLOB param=0)
+
+Associate a default `obj_shard_param`_ object or clear an association.
+
+The value of the `param` argument must be a call to the
+`func_shard_param.use`_ method. No argument clears the association.
+
+The association can be changed per backend request using the `param`
+argument of `func_shard.backend`_.
+
 $Method BOOL .add_backend(PRIV_TASK, BACKEND backend,
 	STRING ident=0, DURATION rampup=973279260)
 
@@ -409,14 +422,15 @@ To generate sharding keys using other hashes, use a custom vmod like
 .. _vmod blobdigest: https://code.uplex.de/uplex-varnish/libvmod-blobdigest/blob/master/README.rst
 
 $Method BACKEND .backend(
-	ENUM {HASH, URL, KEY, BLOB} by=HASH,
-	INT key=0,
-	BLOB key_blob=0,
-	INT alt=0,
-	REAL warmup=-1,
-	BOOL rampup=1,
-	ENUM {CHOSEN, IGNORE, ALL} healthy=CHOSEN)
-
+	[ ENUM {HASH, URL, KEY, BLOB} by ],
+	[ INT key ],
+	[ BLOB key_blob ],
+	[ INT alt ],
+	[ REAL warmup ],
+	[ BOOL rampup ],
+	[ ENUM {CHOSEN, IGNORE, ALL} healthy ],
+	[ BLOB param ],
+	[ ENUM {NOW, LAZY} resolve] )
 
 Lookup a backend on the consistent hashing ring.
 
@@ -519,15 +533,149 @@ is _not_ the order given when backends are added.
     For `alt > 0`, return the `alt`-th alternative backend of all
     those healthy, the last healthy backend found or none.
 
+* `resolve`
+
+  * ``NOW``: look up a backend and return it.
+
+    Can not be used in ``vcl_init{}``.
+
+  * ``LAZY``: return an instance of this director for later backend resolution.
+
+    ``LAZY`` mode is required for referencing shard director instances,
+    for example as backends for other directors (director layering).
+
+    In ``vcl_init{}`` and on the client side, ``LAZY`` mode can not be
+    used with any other argument.
+
+    On the backend side, parameters from arguments or an associated
+    parameter set affect the shard director instance for the backend
+    request irrespective of where it is referenced.
+
+* `param`
+
+  Use or associate a parameter set. The value of the `param` argument
+  must be a call to the `func_shard_param.use`_ method.
+
+  default: as set by `func_shard.associate`_ or unset.
+
+  * for ``resolve=NOW`` take parameter defaults from the
+    `obj_shard_param`_ parameter set
+
+  * for ``resolve=LAZY`` associate the `obj_shard_param`_ parameter
+    set for this backend request
+
+    Implementation notes for use of parameter sets with
+    ``resolve=LAZY``:
+
+    * A `param` argument remains associated and any changes to the
+      associated parameter set affect the sharding decision once the
+      director resolves to an actual backend.
+
+    * If other paramter arguments are also given, they have preference
+      and are kept even if the parameter set given by the `param`
+      argument is subsequently changed within the same backend request.
+
+    * Each call to `func_shard.backend`_ overrides any previous call.
+
 $Method VOID .debug(INT)
 
 `intentionally undocumented`
 
+$Object shard_param()
+
+Create a shard parameter set.
+
+A parameter set allows for re-use of `func_shard.backend`_ arguments
+across many shard director instances and simplifies advanced use cases
+(e.g. shard director with custom parameters layered below other
+directors).
+
+Parameter sets have two scopes:
+
+* per-VCL scope defined in ``vcl_init{}``
+* per backend request scope
+
+The per-VCL scope defines defaults for the per backend scope. Any
+changes to a parameter set in backend context only affect the
+respective backend request.
+
+Parameter sets can not be used in client context.
+
+$Method VOID .clear()
+
+Reset the parameter set to default values as documented for
+`func_shard.backend`_.
+
+* in ``vcl_init{}``, resets the parameter set default for this VCL
+* in backend context, resets the parameter set for this backend
+  request to the VCL defaults
+
+This method may not be used in client context
+
+$Method VOID .set(
+	[ ENUM {HASH, URL, KEY, BLOB} by ],
+	[ INT key ],
+	[ BLOB key_blob ],
+	[ INT alt ],
+	[ REAL warmup ],
+	[ BOOL rampup ],
+	[ ENUM {CHOSEN, IGNORE, ALL} healthy ])
+
+Change the given parameters of a parameter set as documented for
+`func_shard.backend`_.
+
+* in ``vcl_init{}``, changes the parameter set default for this VCL
+
+* in backend context, changes the parameter set for this backend
+  request, keeping the defaults set for this VCL for unspecified
+  arguments.
+
+This method may not be used in client context
+
+$Method STRING .get_by()
+
+Get a string representation of the `by` enum argument which denotes
+how a shard director using this parameter object would derive the
+shard key. See `func_shard.backend`_.
+
+$Method INT .get_key()
+
+Get the key which a shard director using this parameter object would
+use. See `func_shard.backend`_.
+
+$Method INT .get_alt()
+
+Get the `alt` paramter which a shard director using this parameter
+object would use. See `func_shard.backend`_.
+
+$Method REAL .get_warmup()
+
+Get the `warmup` paramter which a shard director using this parameter
+object would use. See `func_shard.backend`_.
+
+$Method BOOL .get_rampup()
+
+Get the `rampup` paramter which a shard director using this parameter
+object would use. See `func_shard.backend`_.
+
+$Method STRING .get_healthy()
+
+Get a string representation of the `healthy` enum argument which a
+shard director using this parameter object would use. See
+`func_shard.backend`_.
+
+$Method BLOB .use()
+
+This method may only be used in backend context.
+
+For use with the `param` argument of `func_shard.backend`_ to associate
+this shard parameter set with a shard director.
+
 ACKNOWLEDGEMENTS
 ================
 
-Development of a previous version of the shard director was partly sponsored
-by Deutsche Telekom AG - Products & Innovation.
+Development of a previous version of the shard director was partly
+sponsored by Deutsche Telekom AG - Products & Innovation.
 
-Development of this version of the shard director was partly sponsored
-by BILD GmbH & Co KG.
+Development of a previous version of the shard director was partly
+sponsored by BILD GmbH & Co KG.