Live- and readiness probe configurable in the viking svc helm chart.

The probes themselves are fixed, but it is now possible to set such
properties as timeouts, delays, success/failure thresholds etc in
values.yaml.

k8s defaults hold for values that are not set.

Closes #41

charts/viking-service/templates/deployment.yaml

@@ -104,10 +104,16 @@ spec:
                 - -P
                 - "0"
                 - varnishd
+            {{- range $k, $v := .Values.vikingService.varnish.livenessProbeConfig }}
+            {{ $k }}: {{ $v }}
+            {{- end }}
           readinessProbe:
             httpGet:
               path: /ready
               port: k8s
+            {{- range $k, $v := .Values.vikingService.varnish.readinessProbeConfig }}
+            {{ $k }}: {{ $v }}
+            {{- end }}
           ports:
             - name: http
               containerPort: 80
@@ -156,10 +162,16 @@ spec:
                 - -P
                 - "0"
                 - haproxy
+            {{- range $k, $v := .Values.vikingService.haproxy.livenessProbeConfig }}
+            {{ $k }}: {{ $v }}
+            {{- end }}
           readinessProbe:
              httpGet:
                path: /healthz
                port: k8s
+             {{- range $k, $v := .Values.vikingService.haproxy.readinessProbeConfig }}
+             {{ $k }}: {{ $v }}
+             {{- end }}
           ports:
             - name: https
               containerPort: 443

charts/viking-service/values.yaml

@@ -14,6 +14,20 @@ vikingService:
       path: /run/varnish
       file: _.secret
 
+    # Configuration of the Varnish liveness probe, other than the probe itself
+    livenessProbeConfig: {}
+      # These fields may be set, defaults are k8s defaults
+      #
+      # initialDelaySeconds:
+      # periodSeconds:
+      # timeoutSeconds:
+      # successThreshold:
+      # failureThreshold:
+
+    # Configuration of the Varnish readiness probe, other than the probe itself
+    # Possible fields as above for liveness probes
+    readinessProbeConfig: {}
+
   haproxy:
     image:
       repository: registry.gitlab.com/uplex/varnish/k8s-ingress/varnish-ingress/haproxy
@@ -25,6 +39,12 @@ vikingService:
 
     resources: {}
 
+    # Configuration of the haproxy liveness probe, as above for Varnish
+    livenessProbeConfig: {}
+
+    # Configuration of the haproxy readiness probe, as above for Varnish
+    readinessProbeConfig: {}
+
   # Choose from Deployment or StatefulSet
   kind: Deployment
 

test/e2e.sh

@@ -143,6 +143,10 @@ echo Examples for devmode and the TemplateConfig CRD
 cd ${MYPATH}/e2e/tmplcfg
 make deploy verify undeploy
 
+echo probeConfig fields in the viking-service helm chart, liveness and readiness
+cd ${MYPATH}/e2e/probeCfg
+make deploy verify undeploy
+
 echo Example of an ExternalName Service as an Ingress backend
 cd ${MYPATH}/../examples/externalname/
 make deploy verify undeploy

test/e2e/probeCfg/Makefile

+# Copyright (c) 2021 UPLEX Nils Goroll Systemoptimierung
+# All rights reserved
+#
+# Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+# GNU make is required.
+
+mkpath := $(abspath $(lastword $(MAKEFILE_LIST)))
+mkdir := $(dir $(mkpath))
+
+CHARTDIR=$(mkdir)/../../../charts
+TESTDIR=$(mkdir)/../..
+
+GEN_SECRET=head -c32 /dev/urandom | base64 | tr +/ -_
+
+# For the klarlack image: make VARNISH=klarlack ...
+ifndef VARNISH
+	VARNISH=varnish
+endif
+
+CI_REPO_PFX=registry.gitlab.com/uplex/varnish/k8s-ingress/varnish-ingress
+
+# For tests using the local docker registry: make TEST=local ...
+# For tests using images from the CI pipeline: make TEST=ci ...
+ifeq ($(TEST),local)
+	CONTROLLER_IMAGE=varnish-ingress/controller
+	CONTROLLER_TAG=latest
+	VARNISH_IMAGE=varnish-ingress/$(VARNISH)
+	VARNISH_TAG=latest
+	HAPROXY_IMAGE=varnish-ingress/haproxy
+	HAPROXY_TAG=latest
+else ifeq ($(TEST),ci)
+	CONTROLLER_IMAGE=$(CI_REPO_PFX)/controller
+	CONTROLLER_TAG=master
+	VARNISH_IMAGE=$(CI_REPO_PFX)/$(VARNISH)
+	VARNISH_TAG=master
+	HAPROXY_IMAGE=$(CI_REPO_PFX)/haproxy
+	HAPROXY_TAG=master
+endif
+
+# If not specified, pull the latest "official" images from dockerhub.
+LATEST=0.1.0
+ifndef CONTROLLER_IMAGE
+	CONTROLLER_IMAGE=uplex/viking-controller
+endif
+ifndef CONTROLLER_TAG
+	CONTROLLER_TAG=$(LATEST)
+endif
+ifndef VARNISH_IMAGE
+	VARNISH_IMAGE=uplex/viking-$(VARNISH)
+endif
+ifndef VARNISH_TAG
+	VARNISH_TAG=$(LATEST)
+endif
+ifndef HAPROXY_IMAGE
+	HAPROXY_IMAGE=uplex/viking-haproxy
+endif
+ifndef HAPROXY_TAG
+	HAPROXY_TAG=$(LATEST)
+endif
+
+all: deploy
+
+no-kubectl:
+	$(warning This test runs with helm only)
+	@true
+
+ifeq ($(DEPLOY),kubectl)
+deploy verify wait uninstall undeploy: no-kubectl
+else
+
+deploy:
+	@kubectl apply -f namespace.yaml
+	@helm install viking-service-probe-cfg $(CHARTDIR)/viking-service \
+		--namespace probe-cfg --values values-varnish.yaml \
+		--set vikingService.secrets.admin=$(shell $(GEN_SECRET)) \
+		--set vikingService.secrets.dataplaneapi=$(shell $(GEN_SECRET)) \
+		--set vikingService.varnish.image.repository=$(VARNISH_IMAGE) \
+		--set vikingService.varnish.image.tag=$(VARNISH_TAG) \
+		--set vikingService.haproxy.image.repository=$(HAPROXY_IMAGE) \
+		--set vikingService.haproxy.image.tag=$(HAPROXY_TAG)
+	@helm install viking-ingress-probe-cfg $(CHARTDIR)/viking-test-app \
+		--namespace probe-cfg --values values.yaml
+
+# TESTOPTS are passed to varnishtest, e.g.: make TESTOPTS=-v verify
+verify:
+	$(mkdir)/verify.sh cafe.vtc
+
+undeploy:
+	@helm uninstall --namespace probe-cfg viking-service-probe-cfg
+	@helm uninstall --namespace probe-cfg viking-ingress-probe-cfg
+	@kubectl delete -f namespace.yaml
+endif
+
+.PHONY: all $(MAKECMDGOALS)

test/e2e/probeCfg/cafe.vtc

+# looks like -*- vcl -*-
+
+varnishtest "cafe example (hello world for Ingress)"
+
+client c1 -connect "${localhost} ${localport}" {
+	txreq -url /coffee/foo/bar -hdr "Host: cafe.example.com"
+	rxresp
+	expect resp.status == 200
+	expect resp.body ~ "(?m)^URI: /coffee/foo/bar$"
+	expect resp.body ~ "(?m)^Server name: coffee-[a-z0-9]+-[a-z0-9]+$"
+
+	txreq -url /tea/baz/quux -hdr "Host: cafe.example.com"
+	rxresp
+	expect resp.status == 200
+	expect resp.body ~ "(?m)^URI: /tea/baz/quux$"
+	expect resp.body ~ "(?m)^Server name: tea-[a-z0-9]+-[a-z0-9]+$"
+
+	txreq -url /coffee/foo/bar
+	rxresp
+	expect resp.status == 404
+
+	txreq -url /milk -hdr "Host: cafe.example.com"
+	rxresp
+	expect resp.status == 404
+} -run

test/e2e/probeCfg/namespace.yaml

+apiVersion: v1
+kind: Namespace
+metadata:
+  name: probe-cfg

test/e2e/probeCfg/values-viking.yaml

+vikingService:
+  varnish:
+    livenessProbeConfig:
+      initialDelaySeconds: 1
+      periodSeconds: 5
+      timeoutSeconds: 2
+      successThreshold: 1
+      failureThreshold: 2
+    readinessProbeConfig:
+      initialDelaySeconds: 5
+      periodSeconds: 10
+      timeoutSeconds: 9
+      successThreshold: 2
+      failureThreshold: 3
+  haproxy:
+    livenessProbeConfig:
+      initialDelaySeconds: 3
+      periodSeconds: 3
+      timeoutSeconds: 2
+      successThreshold: 1
+      failureThreshold: 3
+    readinessProbeConfig:
+      initialDelaySeconds: 4
+      periodSeconds: 5
+      timeoutSeconds: 4
+      successThreshold: 3
+      failureThreshold: 2

test/e2e/probeCfg/values.yaml

+apps:
+  coffee:
+    image: uplex/http-echo
+    replicas: 2
+    targetPort: 7357
+  tea:
+    image: uplex/http-echo
+    replicas: 3
+    targetPort: 7357
+
+ingress:
+  name: probe-cfg-ingress
+  rules:
+  - host: cafe.example.com
+    paths:
+    - path: /tea
+      app: tea
+    - path: /coffee
+      app: coffee
+

test/e2e/probeCfg/verify.sh

+#! /bin/bash -ex
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 file.vtc"
+    exit 1
+fi
+
+MYDIR=$(dirname ${BASH_SOURCE[0]})
+source ${MYDIR}/../../utils.sh
+
+LOCALPORT=${LOCALPORT:-8888}
+
+wait_until_ready app.kubernetes.io/name=viking-service probe-cfg
+wait_until_configured app.kubernetes.io/name=viking-service probe-cfg
+
+# Verify the probe configuration properties set in values-viking.yaml
+JSONPATH_VARNISH='{.spec.containers[?(@.name=="varnish")]'
+JSONPATH_VARNISH_LIVE=${JSONPATH_VARNISH}.livenessProbe
+JSONPATH_VARNISH_READY=${JSONPATH_VARNISH}.readinessProbe
+
+JSONPATH_HAPROXY='{.spec.containers[?(@.name=="haproxy")]'
+JSONPATH_HAPROXY_LIVE=${JSONPATH_HAPROXY}.livenessProbe
+JSONPATH_HAPROXY_READY=${JSONPATH_HAPROXY}.readinessProbe
+
+PODS=$(kubectl -n probe-cfg get pod -l app.kubernetes.io/name=viking-service -o=name)
+for pod in ${PODS}; do
+    # Varnish liveness probe
+    DELAY=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_VARNISH_LIVE}.initialDelaySeconds}")
+    if [ ${DELAY} -ne "1" ]; then
+        exit 1
+    fi
+    PERIOD=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_VARNISH_LIVE}.periodSeconds}")
+    if [ ${PERIOD} -ne "5" ]; then
+        exit 1
+    fi
+    TIMEOUT=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_VARNISH_LIVE}.timeoutSeconds}")
+    if [ ${TIMEOUT} -ne "2" ]; then
+        exit 1
+    fi
+    SUCCESS=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_VARNISH_LIVE}.successThreshold}")
+    if [ ${SUCCESS} -ne "1" ]; then
+        exit 1
+    fi
+    FAILURE=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_VARNISH_LIVE}.failureThreshold}")
+    if [ ${FAILURE} -ne "2" ]; then
+        exit 1
+    fi
+
+    # Varnish readiness probe
+    DELAY=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_VARNISH_READY}.initialDelaySeconds}")
+    if [ ${DELAY} -ne "5" ]; then
+        exit 1
+    fi
+    PERIOD=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_VARNISH_READY}.periodSeconds}")
+    if [ ${PERIOD} -ne "10" ]; then
+        exit 1
+    fi
+    TIMEOUT=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_VARNISH_READY}.timeoutSeconds}")
+    if [ ${TIMEOUT} -ne "9" ]; then
+        exit 1
+    fi
+    SUCCESS=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_VARNISH_READY}.successThreshold}")
+    if [ ${SUCCESS} -ne "2" ]; then
+        exit 1
+    fi
+    FAILURE=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_VARNISH_READY}.failureThreshold}")
+    if [ ${FAILURE} -ne "3" ]; then
+        exit 1
+    fi
+
+    # haproxy liveness probe
+    DELAY=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_HAPROXY_LIVE}.initialDelaySeconds}")
+    if [ ${DELAY} -ne "3" ]; then
+        exit 1
+    fi
+    PERIOD=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_HAPROXY_LIVE}.periodSeconds}")
+    if [ ${PERIOD} -ne "3" ]; then
+        exit 1
+    fi
+    TIMEOUT=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_HAPROXY_LIVE}.timeoutSeconds}")
+    if [ ${TIMEOUT} -ne "2" ]; then
+        exit 1
+    fi
+    SUCCESS=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_HAPROXY_LIVE}.successThreshold}")
+    if [ ${SUCCESS} -ne "1" ]; then
+        exit 1
+    fi
+    FAILURE=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_HAPROXY_LIVE}.failureThreshold}")
+    if [ ${FAILURE} -ne "3" ]; then
+        exit 1
+    fi
+
+    # haproxy readiness probe
+    DELAY=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_HAPROXY_READY}.initialDelaySeconds}")
+    if [ ${DELAY} -ne "4" ]; then
+        exit 1
+    fi
+    PERIOD=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_HAPROXY_READY}.periodSeconds}")
+    if [ ${PERIOD} -ne "5" ]; then
+        exit 1
+    fi
+    TIMEOUT=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_HAPROXY_READY}.timeoutSeconds}")
+    if [ ${TIMEOUT} -ne "4" ]; then
+        exit 1
+    fi
+    SUCCESS=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_HAPROXY_READY}.successThreshold}")
+    if [ ${SUCCESS} -ne "3" ]; then
+        exit 1
+    fi
+    FAILURE=$(kubectl -n probe-cfg get $pod -o=jsonpath="${JSONPATH_HAPROXY_READY}.failureThreshold}")
+    if [ ${FAILURE} -ne "2" ]; then
+        exit 1
+    fi
+done
+
+kubectl port-forward -n probe-cfg svc/viking-service-probe-cfg ${LOCALPORT}:80 >/dev/null &
+trap 'kill $(jobs -p)' EXIT
+wait_for_port ${LOCALPORT}
+
+varnishtest ${TESTOPTS} -Dlocalport=${LOCALPORT} $1