Initial commit, first test of VCL generation from template.

.gitignore

+*~
+

pkg/varnish/vcl/backends.tmpl

+# FOO SERVERS #############################
+
+probe foo_probe {
+	.request = "HEAD /foo_health HTTP/1.1"
+		   "Host: foo"
+	           "Connection: close";
+	.interval = 3s;
+	.timeout = 0.3s;
+	.window = 5;
+	.threshold = 1;
+}
+
+{{range $name, $ep := (index .Services "foo-service").Endpoints -}}
+backend {{$name}} {
+	.host = "{{$ep.IP}}";
+	.port = "{{$ep.Port}}";
+	.probe = foo_probe;
+}
+{{end}}
+
+sub vcl_init {
+	# foo sharding init
+	new foo = directors.shard();
+	foo.set_rampup(0s);
+{{- range $name, $ep := (index .Services "foo-service").Endpoints}}
+	foo.add_backend({{$name}});
+{{- end}}
+	foo.reconfigure();
+}
+
+# BAR & BAZ BACKENDS ######################
+
+probe barbaz_probe {
+	.url = "/probe.html";
+	.interval = 5s;
+	.timeout = 3s;
+	.window = 10;
+	.threshold = 1;
+}
+
+{{range $name, $ep := (index .Services "bar-service").Endpoints -}}
+backend {{$name}} {
+	.host = "{{$ep.IP}}";
+	.port = "{{$ep.Port}}";
+	.probe = barbaz_probe;
+}
+{{end}}
+
+{{range $name, $ep := (index .Services "baz-service").Endpoints -}}
+backend {{$name}} {
+	.host = "{{$ep.IP}}";
+	.port = "{{$ep.Port}}";
+	.probe = barbaz_probe;
+}
+{{end}}
+
+sub vcl_init {
+	# shard/rr init
+	new shard_bar_backends = directors.shard();
+	shard_bar_backends.set_rampup(60s);
+{{- range $name, $ep := (index .Services "bar-service").Endpoints}}
+	shard_bar_backends.add_backend({{$name}});
+{{- end}}
+	shard_bar_backends.reconfigure();
+
+	new rr_bar_backends = directors.round_robin();
+{{- range $name, $ep := (index .Services "bar-service").Endpoints}}
+	rr_bar_backends.add_backend({{$name}});
+{{- end}}
+
+	new shard_baz_backends = directors.shard();
+	shard_baz_backends.set_rampup(60s);
+{{- range $name, $ep := (index .Services "baz-service").Endpoints}}
+	shard_baz_backends.add_backend({{$name}});
+{{- end}}
+	shard_baz_backends.reconfigure();
+
+	new rr_baz_backends = directors.round_robin();
+{{- range $name, $ep := (index .Services "baz-service").Endpoints}}
+	rr_baz_backends.add_backend({{$name}});
+{{- end}}
+}
+
+sub set_version {
+	set resp.http.X-VCL-Version = "{{fqVersion .}}";
+}

pkg/varnish/vcl/spec.go

+/*
+ * Copyright (c) 2019 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+package vcl
+
+import (
+	"crypto/sha512"
+	"encoding/binary"
+	"hash"
+	"math/big"
+	"sort"
+)
+
+// Meta represents a k8s object's Metadata
+type Meta struct {
+	Namespace       string
+	UID             string
+	ResourceVersion string
+}
+
+// Endpoint represents a Service endpoint.
+type Endpoint struct {
+	Meta
+	IP   string
+	Port int32
+}
+
+func (ep Endpoint) hash(hash hash.Hash) {
+	portBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(portBytes, uint32(ep.Port))
+	hash.Write([]byte(ep.IP))
+	hash.Write(portBytes)
+}
+
+// interface for sorting []Endpoint
+type byIPPort []Endpoint
+
+func (a byIPPort) Len() int      { return len(a) }
+func (a byIPPort) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a byIPPort) Less(i, j int) bool {
+	if a[i].IP < a[j].IP {
+		return true
+	}
+	return a[i].Port < a[j].Port
+}
+
+// Service represents a backend Service, where the Endpoints are
+// indexed by the Pod names to which they correspond.
+type Service struct {
+	Meta
+	Endpoints map[string]Endpoint
+}
+
+func (svc Service) hash(hash hash.Hash) {
+	for name, ep := range svc.Endpoints {
+		hash.Write([]byte(name))
+		ep.hash(hash)
+	}
+}
+
+// ConfigMap represents metadata about the ConfigMap that contains
+// VCL sources.
+type ConfigMap struct {
+	Meta
+	Name    string
+	Version string
+}
+
+// Spec is the specification for a VCL configuration derived from
+// Services, Endpoints, and a ConfigMap representing VCL sources. This
+// abstracts the VCL to be loaded by a Varnish instance.
+type Spec struct {
+	// Services are the Varnish backends, indexed by k8s Service
+	// name.
+	Services map[string]Service
+	// CfgMap is the ConfigMap representing VCL sources.
+	CfgMap ConfigMap
+	// Main is the name of the main VCL source, which includes any
+	// other sources, and is named as the file to load in a
+	// vcl.load command.
+	Main string
+	// Version is the version string of the VCL config.
+	Version string
+}
+
+// DeepHash computes a alphanumerically encoded hash value from a Spec
+// such that, almost certainly, two Specs are deeply equal iff their
+// hash values are equal (unless we've discovered a SHA512 collision).
+func (spec Spec) DeepHash() string {
+	hash := sha512.New512_224()
+
+	hash.Write([]byte(spec.Version))
+
+	hash.Write([]byte(spec.CfgMap.Namespace))
+	hash.Write([]byte(spec.CfgMap.Name))
+	hash.Write([]byte(spec.CfgMap.UID))
+	hash.Write([]byte(spec.CfgMap.ResourceVersion))
+
+	svcs := make([]string, len(spec.Services))
+	i := 0
+	for k := range spec.Services {
+		svcs[i] = k
+		i++
+	}
+	sort.Strings(svcs)
+	for _, svc := range svcs {
+		hash.Write([]byte(svc))
+		spec.Services[svc].hash(hash)
+	}
+
+	hash.Write([]byte(spec.Main))
+
+	h := new(big.Int)
+	h.SetBytes(hash.Sum(nil))
+	return h.Text(62)
+}
+
+// FQVersion returns a fully qualified version -- the version string
+// appended with the deep hash of the Spec (since the same VCL version
+// may be used with different backend configurations).
+func (spec Spec) FQVersion() string {
+	return spec.Version + "-" + spec.DeepHash()
+}

pkg/varnish/vcl/template.go

+/*
+ * Copyright (c) 2019 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+package vcl
+
+import (
+	"bytes"
+	"fmt"
+	"path"
+	"regexp"
+	"text/template"
+)
+
+var fMap = template.FuncMap{
+	"fqVersion": func(spec Spec) string { return spec.FQVersion() },
+	"vclMangle": func(s string) string { return mangle(s) },
+}
+
+const (
+	tmplSrc = "backends.tmpl"
+)
+
+var (
+	tmpl       *template.Template
+	vclIllegal = regexp.MustCompile("[^[:word:]-]+")
+)
+
+// InitTemplates initializes templates for VCL generation.
+func InitTemplates(tmplDir string) error {
+	var err error
+	tmplPath := path.Join(tmplDir, tmplSrc)
+
+	tmpl, err = template.New(tmplSrc).Funcs(fMap).ParseFiles(tmplPath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func replIllegal(ill []byte) []byte {
+	repl := []byte("_")
+	for _, b := range ill {
+		repl = append(repl, []byte(fmt.Sprintf("%02x", b))...)
+	}
+	repl = append(repl, []byte("_")...)
+	return repl
+}
+
+// GetSrc returns the VCL generated to implement a Spec.
+func (spec Spec) GetSrc() (string, error) {
+	var buf bytes.Buffer
+	if err := tmpl.Execute(&buf, spec); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+func mangle(s string) string {
+	if s == "" {
+		return s
+	}
+	prefixed := "vk8s_" + s
+	bytes := []byte(prefixed)
+	mangled := vclIllegal.ReplaceAllFunc(bytes, replIllegal)
+	return string(mangled)
+}

pkg/varnish/vcl/testdata/backends.golden

+# FOO SERVERS #############################
+
+probe foo_probe {
+	.request = "HEAD /foo_health HTTP/1.1"
+		   "Host: foo"
+	           "Connection: close";
+	.interval = 3s;
+	.timeout = 0.3s;
+	.window = 5;
+	.threshold = 1;
+}
+
+backend foo-stateful-0 {
+	.host = "172.22.225.25";
+	.port = "6081";
+	.probe = foo_probe;
+}
+backend foo-stateful-1 {
+	.host = "172.22.225.39";
+	.port = "6081";
+	.probe = foo_probe;
+}
+
+
+sub vcl_init {
+	# foo sharding init
+	new foo = directors.shard();
+	foo.set_rampup(0s);
+	foo.add_backend(foo-stateful-0);
+	foo.add_backend(foo-stateful-1);
+	foo.reconfigure();
+}
+
+# BAR & BAZ BACKENDS ######################
+
+probe barbaz_probe {
+	.url = "/probe.html";
+	.interval = 5s;
+	.timeout = 3s;
+	.window = 10;
+	.threshold = 1;
+}
+
+backend bar-64548cd7b6-fb2gb {
+	.host = "172.22.225.49";
+	.port = "8080";
+	.probe = barbaz_probe;
+}
+backend bar-64548cd7b6-j5b5w {
+	.host = "172.22.225.13";
+	.port = "8080";
+	.probe = barbaz_probe;
+}
+
+
+backend baz-5ff49fcd4-hx7lj {
+	.host = "172.22.225.8";
+	.port = "8080";
+	.probe = barbaz_probe;
+}
+
+
+sub vcl_init {
+	# shard/rr init
+	new shard_bar_backends = directors.shard();
+	shard_bar_backends.set_rampup(60s);
+	shard_bar_backends.add_backend(bar-64548cd7b6-fb2gb);
+	shard_bar_backends.add_backend(bar-64548cd7b6-j5b5w);
+	shard_bar_backends.reconfigure();
+
+	new rr_bar_backends = directors.round_robin();
+	rr_bar_backends.add_backend(bar-64548cd7b6-fb2gb);
+	rr_bar_backends.add_backend(bar-64548cd7b6-j5b5w);
+
+	new shard_baz_backends = directors.shard();
+	shard_baz_backends.set_rampup(60s);
+	shard_baz_backends.add_backend(baz-5ff49fcd4-hx7lj);
+	shard_baz_backends.reconfigure();
+
+	new rr_baz_backends = directors.round_robin();
+	rr_baz_backends.add_backend(baz-5ff49fcd4-hx7lj);
+}
+
+sub set_version {
+	set resp.http.X-VCL-Version = "4.7.11-aspw8srL8xsO5ObnCdofc5hQCMfOvqmySYu6YM";
+}

pkg/varnish/vcl/vcl_test.go

+/*
+ * Copyright (c) 2018 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+package vcl
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func cmpGold(got []byte, goldfile string) (bool, error) {
+	goldpath := filepath.Join("testdata", goldfile)
+	gold, err := ioutil.ReadFile(goldpath)
+	if err != nil {
+		return false, err
+	}
+	return bytes.Equal(got, gold), nil
+}
+
+func TestMain(m *testing.M) {
+	tmplDir := ""
+	tmplEnv, exists := os.LookupEnv("TEMPLATE_DIR")
+	if !exists {
+		tmplDir = tmplEnv
+	}
+	if err := InitTemplates(tmplDir); err != nil {
+		fmt.Printf("Cannot parse templates: %v\n", err)
+		os.Exit(-1)
+	}
+	code := m.Run()
+	os.Exit(code)
+}
+
+var dfbSpec = Spec{
+	Version: "4.7.11",
+	Services: map[string]Service{
+		"bar-service": Service{
+			Meta: Meta{
+				Namespace:       "default",
+				UID:             "47850e08-0791-11ea-b351-bab55b1b33a8",
+				ResourceVersion: "514058",
+			},
+			Endpoints: map[string]Endpoint{
+				"bar-64548cd7b6-j5b5w": Endpoint{
+					Meta: Meta{
+						Namespace:       "default",
+						UID:             "d7d27ba8-0792-11ea-b351-bab55b1b33a8",
+						ResourceVersion: "515214",
+					},
+					IP:   "172.22.225.13",
+					Port: 8080,
+				},
+				"bar-64548cd7b6-fb2gb": Endpoint{
+					Meta: Meta{
+						Namespace:       "default",
+						UID:             "d52a8e5b-0792-11ea-b351-bab55b1b33a8",
+						ResourceVersion: "515183",
+					},
+					IP:   "172.22.225.49",
+					Port: 8080,
+				},
+			},
+		},
+		"baz-service": Service{
+			Meta: Meta{
+				Namespace:       "default",
+				UID:             "993323b2-077f-11ea-b351-bab55b1b33a8",
+				ResourceVersion: "502259",
+			},
+			Endpoints: map[string]Endpoint{
+				"baz-5ff49fcd4-hx7lj": Endpoint{
+					Meta: Meta{
+						Namespace:       "default",
+						UID:             "efdb0528-0792-11ea-b351-bab55b1b33a8",
+						ResourceVersion: "515894",
+					},
+					IP:   "172.22.225.8",
+					Port: 8080,
+				},
+			},
+		},
+		"foo-service": Service{
+			Meta: Meta{
+				Namespace:       "default",
+				UID:             "5145d363-079b-11ea-b351-bab55b1b33a8",
+				ResourceVersion: "520824",
+			},
+			Endpoints: map[string]Endpoint{
+				"foo-stateful-0": Endpoint{
+					Meta: Meta{
+						Namespace:       "default",
+						UID:             "51712506-079b-11ea-b351-bab55b1b33a8",
+						ResourceVersion: "520869",
+					},
+					IP:   "172.22.225.25",
+					Port: 6081,
+				},
+				"foo-stateful-1": Endpoint{
+					Meta: Meta{
+						Namespace:       "default",
+						UID:             "56f812d1-079b-11ea-b351-bab55b1b33a8",
+						ResourceVersion: "520898",
+					},
+					IP:   "172.22.225.39",
+					Port: 6081,
+				},
+			},
+		},
+	},
+}
+
+func TestTemplate(t *testing.T) {
+	var buf bytes.Buffer
+	gold := "backends.golden"
+	if err := tmpl.Execute(&buf, dfbSpec); err != nil {
+		t.Fatal("Execute():", err)
+	}
+	ok, err := cmpGold(buf.Bytes(), gold)
+	if err != nil {
+		t.Fatalf("Reading %s: %v", gold, err)
+	}
+	if !ok {
+		t.Errorf("Generated VCL does not match gold file: %s", gold)
+		if testing.Verbose() {
+			t.Logf("Generated: %s", buf.String())
+		}
+	}
+}