Add the optional defaultKey field to self-sharding configuration.

XXX: currently only used for sharding by cookie. Will be extended
for use with all by=KEY cases.

charts/viking-controller/crds/varnishcfg-crd.yaml

@@ -91,6 +91,8 @@ spec:
                                 - SHA3_224
                                 - SHA3_256
                                 - SHA3_512
+                            defaultKey:
+                              type: string
                         conditions:
                           type: array
                           minItems: 1

examples/self-sharding/Makefile

@@ -58,6 +58,11 @@ deploy-shard-by-cookie-helm:
 	@helm install viking-ingress-shard-by-cookie \
 		$(CHARTDIR)/viking-test-app --values values-shard-by-cookie.yaml
 
+deploy-shard-by-cookie-default-helm:
+	@helm install viking-ingress-shard-by-cookie-default \
+		$(CHARTDIR)/viking-test-app \
+		--values values-shard-by-cookie-default.yaml
+
 deploy-primary-only-by-clientid-helm:
 	@helm install viking-ingress-primary-only-by-clientid $(CHARTDIR)/viking-test-app \
 		--values values-primary-only-by-clientid.yaml
@@ -98,6 +103,9 @@ verify:
 verify-cookie:
 	$(mkdir)/verify-cookie.sh
 
+verify-cookie-default:
+	$(mkdir)/verify-cookie-default.sh
+
 wait:
 	@echo Waiting until varnish-ingress Pods are not configured for Ingress
 	$(TESTDIR)/wait.sh app=varnish-ingress
@@ -126,6 +134,10 @@ undeploy-shard-by-cookie-helm:
 	@helm uninstall viking-ingress-shard-by-cookie
 	$(MAKE) wait
 
+undeploy-shard-by-cookie-default-helm:
+	@helm uninstall viking-ingress-shard-by-cookie-default
+	$(MAKE) wait
+
 undeploy-primary-only-by-clientid-helm:
 	@helm uninstall viking-ingress-primary-only-by-clientid
 	$(MAKE) wait
@@ -201,6 +213,8 @@ deploy-shard-by-key: deploy-shard-by-key-helm
 undeploy-shard-by-key: undeploy-shard-by-key-helm
 deploy-shard-by-cookie: deploy-shard-by-cookie-helm
 undeploy-shard-by-cookie: undeploy-shard-by-cookie-helm
+deploy-shard-by-cookie-default: deploy-shard-by-cookie-default-helm
+undeploy-shard-by-cookie-default: undeploy-shard-by-cookie-default-helm
 deploy-primary-only-by-clientid: deploy-primary-only-by-clientid-helm
 undeploy-primary-only-by-clientid: undeploy-primary-only-by-clientid-helm
 deploy-shard-conditions: deploy-shard-conditions-helm
@@ -225,7 +239,9 @@ undeploy: undeploy-shard-by-key
 else ifeq ($(EXAMPLE),shard-by-cookie)
 deploy: deploy-shard-by-cookie
 undeploy: undeploy-shard-by-cookie
-verify: verify-cookie
+else ifeq ($(EXAMPLE),shard-by-cookie-default)
+deploy: deploy-shard-by-cookie-default
+undeploy: undeploy-shard-by-cookie-default
 else ifeq ($(EXAMPLE),primary-only-by-clientid)
 deploy: deploy-primary-only-by-clientid
 undeploy: undeploy-primary-only-by-clientid
@@ -234,7 +250,7 @@ deploy: deploy-shard-conditions
 undeploy: undeploy-shard-conditions
 else
 deploy undeploy:
-	$(error EXAMPLE must be set to self-sharding, shard-conditions, primary-only[-by-clientid], or shard-by-[digest|url|key|cookie])
+	$(error EXAMPLE must be set to self-sharding, shard-conditions, primary-only[-by-clientid], or shard-by-[digest|url|key|cookie[-default]])
 endif
 
 .PHONY: all $(MAKECMDGOALS)

examples/self-sharding/cafe-cookie-default.vtc

+# looks like -*- vcl -*-
+
+varnishtest "cafe example with self-sharding by Cookie, and default key"
+
+# The beresp may send Connection:close, if Varnish went to pipe due to
+# primary-only. So we run each test in a separate connection.
+client c1 -connect "${localhost} ${localport}" {
+	txreq -url /coffee/foo -hdr "Host: cafe.example.com" \
+            -hdr "Cookie: baz=quux; foo=abcdefghijklmnopqrstuvwxyz; 47=11"
+	rxresp
+	expect resp.status == 200
+	expect resp.body ~ "(?m)^URI: /coffee/foo$"
+	expect resp.body ~ "(?m)^Server name: coffee-[a-z0-9]+-[a-z0-9]+$"
+} -run
+
+client c1 -connect "${localhost} ${localport}" {
+	txreq -url /tea/bar -hdr "Host: cafe.example.com" \
+            -hdr "Cookie: baz=quux; foo=foobar"
+	rxresp
+	expect resp.status == 200
+	expect resp.body ~ "(?m)^URI: /tea/bar"
+	expect resp.body ~ "(?m)^Server name: tea-[a-z0-9]+-[a-z0-9]+$"
+} -run
+
+client c1 -connect "${localhost} ${localport}" {
+	txreq -url /coffee/baz -hdr "Host: cafe.example.com" \
+            -hdr "Cookie: foo=47; baz=quux"
+	rxresp
+	expect resp.status == 200
+	expect resp.body ~ "(?m)^URI: /coffee/baz"
+	expect resp.body ~ "(?m)^Server name: coffee-[a-z0-9]+-[a-z0-9]+$"
+} -run
+
+client c1 -connect "${localhost} ${localport}" {
+	txreq -url /tea/quux -hdr "Host: cafe.example.com" \
+            -hdr "Cookie: foo=fighter"
+	rxresp
+	expect resp.status == 200
+	expect resp.body ~ "(?m)^URI: /tea/quux"
+	expect resp.body ~ "(?m)^Server name: tea-[a-z0-9]+-[a-z0-9]+$"
+} -run
+
+client c1 -connect "${localhost} ${localport}" {
+	txreq -url /coffee/foo -hdr "Host: cafe.example.com" \
+            -hdr "Cookie: baz=quux; 47=11"
+	rxresp
+	expect resp.status == 200
+	expect resp.body ~ "(?m)^URI: /coffee/foo$"
+	expect resp.body ~ "(?m)^Server name: coffee-[a-z0-9]+-[a-z0-9]+$"
+} -run
+
+client c1 -connect "${localhost} ${localport}" {
+	txreq -url /coffee/foo -hdr "Host: cafe.example.com"
+	rxresp
+	expect resp.status == 200
+	expect resp.body ~ "(?m)^URI: /coffee/foo$"
+	expect resp.body ~ "(?m)^Server name: coffee-[a-z0-9]+-[a-z0-9]+$"
+} -run

examples/self-sharding/values-shard-by-cookie-default.yaml

+apps:
+  coffee:
+    image: nginxdemos/hello:plain-text
+    replicas: 2
+  tea:
+    image: nginxdemos/hello:plain-text
+    replicas: 3
+
+ingress:
+  name: cafe-ingress
+  rules:
+  - host: cafe.example.com
+    paths:
+    - path: /tea
+      type: Prefix
+      app: tea
+    - path: /coffee
+      type: Prefix
+      app: coffee
+
+vikingAdmSvc: varnish-ingress-admin
+
+selfSharding:
+  rules:
+    - shard:
+        key: cookie=foo
+        primaryOnly: true
+        defaultKey: ""
+
+# Use reqDisposition to bypass builtin vcl_recv so that responses to
+# requests with the Cookie head may be cacheable.
+# cf. "cookie pass" in examples/req-disposition
+reqDisposition:
+  - conditions:
+    - comparand: req.http.Host
+      compare: not-exists
+    - comparand: req.esi_level
+      count: 0
+    - comparand: req.proto
+      compare: prefix
+      values:
+        - HTTP/1.1
+      match-flags:
+        case-sensitive: false
+    disposition:
+      action: synth
+      status: 400
+
+  - conditions:
+    - comparand: req.method
+      compare: not-equal
+      values:
+        - GET
+        - HEAD
+        - PUT
+        - POST
+        - TRACE
+        - OPTIONS
+        - DELETE
+        - PATCH
+        - CONNECT
+    disposition:
+      action: synth
+      status: 405
+
+  - conditions:
+    - comparand: req.method
+      compare: not-equal
+      values:
+        - GET
+        - HEAD
+    disposition:
+      action: pass

examples/self-sharding/verify-cookie-default.sh

+#! /bin/bash -ex
+
+MYDIR=$(dirname ${BASH_SOURCE[0]})
+source ${MYDIR}/../../test/utils.sh
+
+LOCALPORT=${LOCALPORT:-8888}
+
+wait_until_ready app=varnish-ingress
+wait_until_configured app=varnish-ingress
+
+kubectl port-forward svc/varnish-ingress ${LOCALPORT}:80 >/dev/null &
+trap 'kill $(jobs -p)' EXIT
+wait_for_port ${LOCALPORT}
+
+# XXX hackish, see the comments in verify.sh
+sleep 10
+varnishtest ${TESTOPTS} -Dlocalport=${LOCALPORT} cafe-cookie-default.vtc

pkg/apis/varnishingress/v1alpha1/types.go

@@ -76,9 +76,10 @@ type ShardRule struct {
 
 // ShardSpec specifies the configuration details for sharding.
 type ShardSpec struct {
-	Key         string `json:"key,omitempty"`
-	Digest      string `json:"digest,omitempty"`
-	PrimaryOnly bool   `json:"primaryOnly,omitempty"`
+	Key         string  `json:"key,omitempty"`
+	Digest      string  `json:"digest,omitempty"`
+	DefaultKey  *string `json:"defaultKey,omitempty"`
+	PrimaryOnly bool    `json:"primaryOnly,omitempty"`
 }
 
 // ProbeSpec specifies health probes for self-sharding and BackendConfig.

pkg/controller/varnishconfig.go

@@ -446,6 +446,7 @@ func (worker *NamespaceWorker) configSharding(spec *vcl.Spec,
 		vclRule := vcl.ShardRule{
 			Conditions:  reqConds2vclConds(rule.Conditions),
 			PrimaryOnly: rule.Sharding.PrimaryOnly,
+			DefaultKey:  rule.Sharding.DefaultKey,
 			By:          vcl.ByHash,
 		}
 		if rule.Sharding.Digest != "" &&

pkg/varnish/vcl/self_shard_tmpl.go

@@ -31,7 +31,6 @@ package vcl
 import (
 	"fmt"
 	"regexp"
-	"strconv"
 	"text/template"
 )
 
@@ -151,7 +150,15 @@ sub vcl_recv {
 		{{- digest_update 'c' $rule }}
 		{{- if isCookieKey $rule }}
 		if (!vk8s_shard_cookie_{{$ridx}}.match(req.http.Cookie)) {
+			{{- if hasDefaultKey $rule }}
+			set req.http.VK8S-Shard-Key = "{{ $rule.DefaultKey }}";
+			{{- else }}
 			return (synth(400));
+			{{- end }}
+		}
+		else {
+			set req.http.VK8S-Shard-Key
+				= vk8s_shard_cookie_{{$ridx}}.backref(1);
 		}
 		{{- end }}
 		vk8s_cluster_primary.set(vk8s_cluster.backend(resolve=NOW
@@ -190,7 +197,15 @@ sub vk8s_cluster_fetch {
 		{{- digest_update 'b' $rule }}
 		{{- if isCookieKey $rule }}
 		if (!vk8s_shard_cookie_{{$ridx}}.match(bereq.http.Cookie)) {
+			{{- if hasDefaultKey $rule }}
+			set bereq.http.VK8S-Shard-Key = "{{ $rule.DefaultKey }}";
+			{{- else }}
 			return (error(400));
+			{{- end }}
+		}
+		else {
+			set bereq.http.VK8S-Shard-Key
+				= vk8s_shard_cookie_{{$ridx}}.backref(1);
 		}
 		{{- end }}
 		vk8s_cluster_param.set({{ key 'b' $rule $ridx }});
@@ -257,8 +272,10 @@ func context(ctx rune, key string) string {
 
 func keyParams(ctx rune, rule ShardRule, idx int) string {
 	pfx := ""
+	http := "bereq"
 	if ctx == 'c' {
 		pfx = ", "
+		http = "req"
 	}
 	switch rule.By {
 	case ByHash:
@@ -269,8 +286,8 @@ func keyParams(ctx rune, rule ShardRule, idx int) string {
 		return pfx + "by=KEY, key=vk8s_cluster.key(" +
 			context(ctx, rule.Key) + ")"
 	case Cookie:
-		return pfx + "by=KEY, key=vk8s_cluster.key(vk8s_shard_cookie_" +
-			strconv.Itoa(idx) + ".backref(1))"
+		return pfx + "by=KEY, key=vk8s_cluster.key(" + http +
+			".http.VK8S-Shard-Key)"
 	}
 	return pfx + "by=BLOB, key_blob=vk8s_shard_digest.final()"
 }
@@ -279,6 +296,10 @@ func isCookieKey(rule ShardRule) bool {
 	return rule.By == Cookie
 }
 
+func hasDefaultKey(rule ShardRule) bool {
+	return rule.DefaultKey != nil
+}
+
 func digestInit(rule ShardRule) string {
 	if rule.By != Blob {
 		return ""
@@ -336,6 +357,7 @@ const selfShardName = "self-sharding"
 var shardFuncMap = template.FuncMap{
 	"key":           keyParams,
 	"isCookieKey":   isCookieKey,
+	"hasDefaultKey": hasDefaultKey,
 	"digest_init":   digestInit,
 	"digest_update": digestUpdate,
 	"hasPrimary":    hasPrimary,

pkg/varnish/vcl/shard_test.go

@@ -30,6 +30,7 @@ package vcl
 
 import (
 	"bytes"
+	_ "io/ioutil"
 	"testing"
 )
 
@@ -150,3 +151,16 @@ func TestShardByCookie(t *testing.T) {
 	varnishCluster.Rules[0].Key = "foobar"
 	templateTest(t, shardTmpl, varnishCluster, "shard_by_cookie.golden")
 }
+
+func TestShardByCookieDefault(t *testing.T) {
+	defaultKey := "defaultKey"
+	varnishCluster.Rules = []ShardRule{{
+		PrimaryOnly: true,
+		By:          Cookie,
+		Key:         "bazquux",
+		DefaultKey:  &defaultKey,
+		Conditions:  []Condition{},
+	}}
+	templateTest(t, shardTmpl, varnishCluster,
+		"shard_by_cookie_default.golden")
+}

pkg/varnish/vcl/spec.go

@@ -432,6 +432,7 @@ const (
 // conditions under which the configuration holds.
 type ShardRule struct {
 	Conditions  []Condition
+	DefaultKey  *string
 	Key         string
 	By          KeyBy
 	Algo        HashAlgo

pkg/varnish/vcl/testdata/shard_by_cookie.golden

@@ -82,7 +82,11 @@ sub vcl_recv {
 		if (!vk8s_shard_cookie_0.match(req.http.Cookie)) {
 			return (synth(400));
 		}
-		vk8s_cluster_primary.set(vk8s_cluster.backend(resolve=NOW, by=KEY, key=vk8s_cluster.key(vk8s_shard_cookie_0.backref(1))));
+		else {
+			set req.http.VK8S-Shard-Key
+				= vk8s_shard_cookie_0.backref(1);
+		}
+		vk8s_cluster_primary.set(vk8s_cluster.backend(resolve=NOW, by=KEY, key=vk8s_cluster.key(req.http.VK8S-Shard-Key)));
 		if (remote.ip !~ vk8s_cluster_acl
 		    && "" + vk8s_cluster_primary.get() != server.identity) {
 			set req.backend_hint = vk8s_cluster_primary.get();

test/e2e.sh

@@ -87,6 +87,9 @@ make EXAMPLE=shard-by-key deploy verify undeploy
 echo Self-sharding by cookie example
 make EXAMPLE=shard-by-cookie deploy verify-cookie undeploy
 
+echo Self-sharding by cookie with default key example
+make EXAMPLE=shard-by-cookie-default deploy verify-cookie-default undeploy
+
 echo Primary-only self-sharding by client.identity as key
 make EXAMPLE=primary-only-by-clientid deploy verify undeploy
 

test/e2e/tmplcfg/values.yaml

@@ -703,7 +703,15 @@ templates:
                     {{- digest_update 'c' $rule }}
                     {{- if isCookieKey $rule }}
                     if (!vk8s_shard_cookie_{{$ridx}}.match(req.http.Cookie)) {
+                            {{- if hasDefaultKey $rule }}
+                            set req.http.VK8S-Shard-Key = "{{ $rule.DefaultKey }}";
+                            {{- else }}
                             return (synth(400));
+                            {{- end }}
+                    }
+                    else {
+                            set req.http.VK8S-Shard-Key
+                                    = vk8s_shard_cookie_{{$ridx}}.backref(1);
                     }
                     {{- end }}
                     vk8s_cluster_primary.set(vk8s_cluster.backend(resolve=NOW
@@ -742,7 +750,15 @@ templates:
                     {{- digest_update 'b' $rule }}
                     {{- if isCookieKey $rule }}
                     if (!vk8s_shard_cookie_{{$ridx}}.match(bereq.http.Cookie)) {
+                            {{- if hasDefaultKey $rule }}
+                            set bereq.http.VK8S-Shard-Key = "{{ $rule.DefaultKey }}";
+                            {{- else }}
                             return (error(400));
+                            {{- end }}
+                    }
+                    else {
+                            set bereq.http.VK8S-Shard-Key
+                                    = vk8s_shard_cookie_{{$ridx}}.backref(1);
                     }
                     {{- end }}
                     vk8s_cluster_param.set({{ key 'b' $rule $ridx }});