Fundamental refactoring: controller & Varnishen in different containers.

Docs are not yet updated in this commit.

.gitignore

 *~
 
 # Build artifacts
-/k8s-ingress
-/main_version.go
+/cmd/k8s-ingress
+/cmd/main_version.go

Makefile → cmd/Makefile

@@ -24,13 +24,7 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-all: push
-
-IMAGE = varnish-ingress
-
-DOCKER_BUILD_OPTIONS =
-
-MINIKUBE =
+all: k8s-ingress
 
 PACKAGES = \
 	k8s.io/client-go/kubernetes \
@@ -62,18 +56,7 @@ check: k8s-ingress
 
 test: check
 
-docker-minikube:
-ifeq ($(MINKUBE),1)
-	eval $(minikube docker-env)
-endif
-
-container: docker-minikube
-	docker build --build-arg PACKAGES="$(PACKAGES)" \
-	$(DOCKER_BUILD_OPTIONS) -t $(IMAGE) .
-
-push: docker-minikube container
-	docker push $(IMAGE)
-
 clean:
 	go clean ./...
 	rm -f main_version.go
+	rm -f k8s-ingress

controller/utils.go → cmd/controller/utils.go

@@ -130,6 +130,8 @@ const (
 	Endpoints
 	// Service resource
 	Service
+	// Secret resource
+	Secret
 )
 
 // Task is an element of a taskQueue
@@ -149,6 +151,8 @@ func NewTask(key string, obj interface{}) (Task, error) {
 		k = Endpoints
 	case *api_v1.Service:
 		k = Service
+	case *api_v1.Secret:
+		k = Secret
 	default:
 		return Task{}, fmt.Errorf("Unknown type: %v", t)
 	}
@@ -156,20 +160,14 @@ func NewTask(key string, obj interface{}) (Task, error) {
 	return Task{k, key}, nil
 }
 
-// compareLinks returns true if the 2 self links are equal.
-// func compareLinks(l1, l2 string) bool {
-// 	// TODO: These can be partial links
-// 	return l1 == l2 && l1 != ""
-// }
-
 // StoreToIngressLister makes a Store that lists Ingress.
-// TODO: Move this to cache/listers post 1.1.
 type StoreToIngressLister struct {
 	cache.Store
 }
 
-// GetByKeySafe calls Store.GetByKeySafe and returns a copy of the ingress so it is
-// safe to modify.
+// GetByKeySafe calls Store.GetByKeySafe and returns a copy of the
+// ingress so it is safe to modify.
+
 func (s *StoreToIngressLister) GetByKeySafe(key string) (ing *extensions.Ingress, exists bool, err error) {
 	item, exists, err := s.Store.GetByKey(key)
 	if !exists || err != nil {
@@ -257,3 +255,8 @@ func FindPort(pod *api_v1.Pod, svcPort *api_v1.ServicePort) (int32, error) {
 
 	return 0, fmt.Errorf("no suitable port for manifest: %s", pod.UID)
 }
+
+// StoreToSecretLister makes a Store that lists Secrets
+type StoreToSecretLister struct {
+	cache.Store
+}

main.go → cmd/main.go

@@ -38,8 +38,8 @@ import (
 	"os/signal"
 	"syscall"
 
-	"code.uplex.de/uplex-varnish/k8s-ingress/controller"
-	"code.uplex.de/uplex-varnish/k8s-ingress/varnish"
+	"code.uplex.de/uplex-varnish/k8s-ingress/cmd/controller"
+	"code.uplex.de/uplex-varnish/k8s-ingress/cmd/varnish"
 
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"

container/Dockerfile.controller

+FROM golang:1.10.3 as builder
+ARG PACKAGES
+RUN go get -d -v github.com/slimhazard/gogitversion && \
+    cd /go/src/github.com/slimhazard/gogitversion && \
+    make install
+RUN go get -u -v $PACKAGES
+COPY . /go/src/code.uplex.de/uplex-varnish/k8s-ingress
+WORKDIR /go/src/code.uplex.de/uplex-varnish/k8s-ingress/cmd
+RUN go generate && \
+    CGO_ENABLED=0 GOOS=linux go build -o k8s-ingress *.go
+
+FROM alpine:3.8
+COPY --from=builder /go/src/code.uplex.de/uplex-varnish/k8s-ingress/cmd/k8s-ingress /k8s-ingress
+COPY --from=builder /go/src/code.uplex.de/uplex-varnish/k8s-ingress/cmd/varnish/vcl/vcl.tmpl /vcl.tmpl
+ENTRYPOINT ["/k8s-ingress"]

Dockerfile → container/Dockerfile.varnish

-FROM golang:1.10.3 as builder
-ARG PACKAGES
-RUN go get -d -v github.com/slimhazard/gogitversion && \
-    cd /go/src/github.com/slimhazard/gogitversion && \
-    make install
-RUN go get -v $PACKAGES
-COPY . /go/src/code.uplex.de/uplex-varnish/k8s-ingress
-WORKDIR /go/src/code.uplex.de/uplex-varnish/k8s-ingress
-RUN go generate && \
-    CGO_ENABLED=0 GOOS=linux go build -o k8s-ingress *.go
-
 FROM centos:centos7
 COPY varnishcache_varnish60.repo /etc/yum.repos.d/
 RUN yum install -y epel-release && yum update -y -q && \
@@ -17,6 +6,13 @@ RUN yum install -y epel-release && yum update -y -q && \
     yum install -y -q varnish-6.0.1 && \
     yum install -y -q --nogpgcheck vmod-re2-1.5.1 && \
     yum clean all && rm -rf /var/cache/yum
-COPY varnish/vcl/vcl.tmpl /
-COPY --from=builder /go/src/code.uplex.de/uplex-varnish/k8s-ingress/k8s-ingress /k8s-ingress
-ENTRYPOINT ["/k8s-ingress"]
+COPY bogo_backend.vcl /etc/varnish/
+COPY ready.vcl /etc/varnish/
+COPY notavailable.vcl /etc/varnish
+COPY boot.vcl /etc/varnish
+COPY start.cli /etc/varnish
+RUN mkdir /var/run/varnish
+ENTRYPOINT ["/usr/sbin/varnishd", "-F", "-a", ":80", "-a", "k8s=:8080", \
+            "-S", "/var/run/varnish/_.secret", "-T", "0.0.0.0:6081", \
+            "-p", "vcl_path=/etc/varnish", "-I", "/etc/varnish/start.cli", \
+            "-f", ""]

container/Makefile

+# Copyright (c) 2018 UPLEX Nils Goroll Systemoptimierung
+# All rights reserved
+#
+# Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+all: controller varnish
+
+DOCKER_BUILD_OPTIONS =
+
+MINIKUBE =
+
+PACKAGES = \
+	k8s.io/client-go/kubernetes \
+	k8s.io/client-go/kubernetes/scheme \
+	k8s.io/client-go/kubernetes/typed/core/v1 \
+	k8s.io/client-go/rest \
+	k8s.io/client-go/tools/cache \
+	k8s.io/client-go/tools/record \
+	k8s.io/client-go/util/workqueue \
+	k8s.io/api/core/v1 \
+	k8s.io/api/extensions/v1beta1 \
+	k8s.io/apimachinery/pkg/apis/meta/v1 \
+	k8s.io/apimachinery/pkg/fields \
+	k8s.io/apimachinery/pkg/labels \
+	k8s.io/apimachinery/pkg/util/intstr \
+	k8s.io/apimachinery/pkg/util/wait \
+	code.uplex.de/uplex-varnish/varnishapi/pkg/admin
+
+docker-minikube:
+ifeq ($(MINKUBE),1)
+	eval $(minikube docker-env)
+endif
+
+controller: Dockerfile.controller docker-minikube
+	docker build --build-arg PACKAGES="$(PACKAGES)" \
+	$(DOCKER_BUILD_OPTIONS) -t varnish-ingress/controller \
+	-f Dockerfile.controller ..
+
+varnish: Dockerfile.varnish docker-minikube
+	docker build $(DOCKER_BUILD_OPTIONS) -t varnish-ingress/varnish \
+	-f Dockerfile.varnish .

container/bogo_backend.vcl

+# Bogus backend, to which no request is ever sent.
+# - Sentinel that no backend was determined after a request has been
+#   evaluated according to IngressRules.
+# - Defined when the Ingress is not ready (not currently implementing
+#   any IngressSpec), so that Varnish doesn't complain about no
+#   backend definition.
+backend notfound {
+	# 192.0.2.0/24 reserved for docs & examples (RFC5737).
+	.host = "192.0.2.255";
+	.port = "80";
+}

container/boot.vcl

+vcl 4.1;
+
+include "bogo_backend.vcl";
+
+sub vcl_recv {
+	if (local.socket == "k8s") {
+		if (req.url == "/ready") {
+			return (vcl(vk8s_readiness));
+		}
+		return (synth(404));
+	}
+	return (vcl(vk8s_regular));
+}

container/notavailable.vcl

+vcl 4.0;
+
+include "bogo_backend.vcl";
+
+# Send a synthetic response with status 503 to every request.
+# Used for the readiness check and regular traffic when Varnish is not ready.
+sub vcl_recv {
+        return(synth(503));
+}

container/ready.vcl

+vcl 4.0;
+
+include "bogo_backend.vcl";
+
+# Send a synthetic response with status 200 to every request.
+# Used for the readiness check when Varnish is ready.
+sub vcl_recv {
+        return(synth(200));
+}

container/start.cli

+vcl.load vk8s_ready /etc/varnish/ready.vcl
+vcl.load vk8s_notavailable /etc/varnish/notavailable.vcl
+vcl.label vk8s_readiness vk8s_notavailable
+vcl.label vk8s_regular vk8s_notavailable
+vcl.load boot /etc/varnish/boot.vcl
+vcl.use boot
+start

deploy/adm-secret.yaml

+apiVersion: v1
+kind: Secret
+metadata:
+  name: adm-secret
+  namespace: varnish-ingress
+type: Opaque
+data:
+  admin: XGASQn0dd/oEsWh5WMXUpmKAKNZYnQGsHSmO/nHkv1w=

deploy/controller.yaml

+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: varnish-ingress-controller
+  namespace: varnish-ingress
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: varnish-ingress-controller
+  template:
+    metadata:
+      labels:
+        app: varnish-ingress-controller
+    spec:
+      serviceAccountName: varnish-ingress
+      containers:
+      - image: varnish-ingress/controller
+        imagePullPolicy: IfNotPresent
+        name: varnish-ingress-controller
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name

deploy/rbac.yaml

@@ -12,6 +12,14 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:

deploy/varnish-adm-svc.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  name: varnish-ingress-admin
+  namespace: varnish-ingress
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+spec:
+  clusterIP: None
+  ports:
+  - port: 6081
+    name: varnishadm
+  selector:
+    app: varnish-ingress
+  publishNotReadyAddresses: true

deploy/varnish.yaml

+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: varnish
+  namespace: varnish-ingress
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: varnish-ingress
+  template:
+    metadata:
+      labels:
+        app: varnish-ingress
+    spec:
+      serviceAccountName: varnish-ingress
+      containers:
+      - image: varnish-ingress/varnish
+        imagePullPolicy: IfNotPresent
+        name: varnish-ingress
+        ports:
+        - name: http
+          containerPort: 80
+        - name: k8sport
+          containerPort: 8080
+        - name: admport
+          containerPort: 6081
+        volumeMounts:
+        - name: adm-secret
+          mountPath: "/var/run/varnish"
+          readOnly: true
+        livenessProbe:
+          exec:
+            command:
+            - /usr/bin/pgrep
+            - -P
+            - "0"
+            - varnishd
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: k8sport
+      volumes:
+      - name: adm-secret
+        secret:
+          secretName: adm-secret
+          items:
+          - key: admin
+            path: _.secret

varnish/varnish.go

-/*
- * Copyright (c) 2018 UPLEX Nils Goroll Systemoptimierung
- * All rights reserved
- *
- * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-/*
-// TODO
-* VCL housekeeping
-  * either discard the previously active VCL immediately on new vcl.use
-  * or periodically clean up
-
-* monitoring
-  * periodically call ping, status, panic.show when otherwise idle
-*/
-
-package varnish
-
-import (
-	"bufio"
-	"crypto/rand"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"os/exec"
-	"os/user"
-	"path/filepath"
-	"strconv"
-	"sync/atomic"
-	"syscall"
-	"time"
-
-	"code.uplex.de/uplex-varnish/k8s-ingress/varnish/vcl"
-
-	"code.uplex.de/uplex-varnish/varnishapi/pkg/admin"
-)
-
-// XXX timeout for getting Admin connection (waiting for varnishd start)
-// timeout waiting for child process to stop
-const (
-	vclDir       = "/etc/varnish"
-	vclFile      = "ingress.vcl"
-	varnishLsn   = ":80"
-	varnishdPath = "/usr/sbin/varnishd"
-	admConn      = "localhost:6081"
-	secretFile   = "_.secret"
-	notFoundVCL  = `vcl 4.0;
-
-backend default { .host = "192.0.2.255"; .port = "80"; }
-
-sub vcl_recv {
-	return (synth(404));
-}
-`
-)
-
-var (
-	vclPath     = filepath.Join(vclDir, vclFile)
-	tmpPath     = filepath.Join(os.TempDir(), vclFile)
-	secretPath  = filepath.Join(vclDir, secretFile)
-	varnishArgs = []string{
-		"-a", varnishLsn, "-f", vclPath, "-F", "-S", secretPath,
-		"-M", admConn,
-	}
-	vcacheUID  int
-	varnishGID int
-	currentIng string
-	configCtr  = uint64(0)
-)
-
-type VarnishController struct {
-	varnishdCmd *exec.Cmd
-	adm         *admin.Admin
-	errChan     chan error
-}
-
-func NewVarnishController() *VarnishController {
-	return &VarnishController{}
-}
-
-func (vc *VarnishController) Start(errChan chan error) {
-	vc.errChan = errChan
-
-	log.Print("Starting Varnish controller")
-	vcacheUser, err := user.Lookup("varnish")
-	if err != nil {
-		vc.errChan <- err
-		return
-	}
-	varnishGrp, err := user.LookupGroup("varnish")
-	if err != nil {
-		vc.errChan <- err
-		return
-	}
-	vcacheUID, err = strconv.Atoi(vcacheUser.Uid)
-	if err != nil {
-		vc.errChan <- err
-		return
-	}
-	varnishGID, err = strconv.Atoi(varnishGrp.Gid)
-	if err != nil {
-		vc.errChan <- err
-		return
-	}
-
-	secret := make([]byte, 32)
-	_, err = rand.Read(secret)
-	if err != nil {
-		vc.errChan <- err
-		return
-	}
-	if err := ioutil.WriteFile(secretPath, secret, 0400); err != nil {
-		vc.errChan <- err
-		return
-	}
-	if err := os.Chown(secretPath, vcacheUID, varnishGID); err != nil {
-		vc.errChan <- err
-		return
-	}
-	log.Print("Wrote secret file")
-
-	notFoundBytes := []byte(notFoundVCL)
-	if err := ioutil.WriteFile(vclPath, notFoundBytes, 0644); err != nil {
-		vc.errChan <- err
-		return
-	}
-	if err := os.Chown(vclPath, vcacheUID, varnishGID); err != nil {
-		vc.errChan <- err
-		return
-	}
-	log.Print("Wrote initial VCL file")
-
-	if vc.adm, err = admin.Listen(admConn); err != nil {
-		vc.errChan <- err
-		return
-	}
-	log.Print("Opened port to listen for Varnish adm connection")
-
-	vc.varnishdCmd = exec.Command(varnishdPath, varnishArgs...)
-	if err := vc.varnishdCmd.Start(); err != nil {
-		vc.errChan <- err
-		return
-	}
-	log.Print("Launched varnishd")
-
-	if err := vc.adm.Accept(secret); err != nil {
-		vc.errChan <- err
-		return
-	}
-	log.Print("Accepted varnish admin connection")
-}
-
-func (vc *VarnishController) Update(key string, spec vcl.Spec) error {
-	if currentIng != "" && currentIng != key {
-		return fmt.Errorf("Multiple Ingress definitions currently not "+
-			"supported: current=%s new=%s", currentIng, key)
-	}
-	currentIng = key
-
-	f, err := os.Create(tmpPath)
-	if err != nil {
-		return err
-	}
-	wr := bufio.NewWriter(f)
-	if err := vcl.Tmpl.Execute(wr, spec); err != nil {
-		return err
-	}
-	wr.Flush()
-	f.Close()
-	log.Printf("Wrote new VCL config to %s", tmpPath)
-
-	ctr := atomic.AddUint64(&configCtr, 1)
-	configName := fmt.Sprintf("ingress-%d", ctr)
-	if err := vc.adm.VCLLoad(configName, tmpPath); err != nil {
-		log.Print("Failed to load VCL: ", err)
-		return err
-	}
-	log.Printf("Loaded VCL config %s", configName)
-
-	newVCL, err := ioutil.ReadFile(tmpPath)
-	if err != nil {
-		log.Print(err)
-		return err
-	}
-	if err = ioutil.WriteFile(vclPath, newVCL, 0644); err != nil {
-		log.Print(err)
-		return err
-	}
-	log.Printf("Wrote VCL config to %s", vclPath)
-
-	if err = vc.adm.VCLUse(configName); err != nil {
-		log.Print("Failed to activate VCL: ", err)
-		return err
-	}
-	log.Printf("Activated VCL config %s", configName)
-
-	// XXX discard previously active VCL
-	return nil
-}
-
-// We currently only support one Ingress definition at a time, so
-// deleting the Ingress means that we revert to the "boot" config,
-// which returns synthetic 404 Not Found for all requests.
-func (vc *VarnishController) DeleteIngress(key string) error {
-	if currentIng != "" && currentIng != key {
-		return fmt.Errorf("Unknown Ingress %s", key)
-	}
-
-	if err := vc.adm.VCLUse("boot"); err != nil {
-		log.Print("Failed to activate VCL: ", err)
-		return err
-	}
-	log.Printf("Activated VCL config boot")
-
-	currentIng = ""
-	// XXX discard previously active VCL
-	return nil
-}
-
-// Currently only one Ingress at a time
-func (vc *VarnishController) HasIngress(key string) bool {
-	if currentIng == "" {
-		return false
-	}
-	return key == currentIng
-}
-
-func (vc *VarnishController) Quit() {
-	if err := vc.adm.Stop(); err != nil {
-		log.Print("Failed to stop Varnish child process:", err)
-	} else {
-		for {
-			tmoChan := time.After(time.Minute)
-			select {
-			case <-tmoChan:
-				// XXX config the timeout
-				log.Print("timeout waiting for Varnish child " +
-					"process to finish")
-				return
-			default:
-				state, err := vc.adm.Status()
-				if err != nil {
-					log.Print("Can't get Varnish child "+
-						"process status:", err)
-					return
-				}
-				if state != admin.Stopped {
-					continue
-				}
-			}
-		}
-	}
-	vc.adm.Close()
-
-	if err := vc.varnishdCmd.Process.Signal(syscall.SIGTERM); err != nil {
-		log.Print("Failed to stop Varnish:", err)
-		return
-	}
-	log.Print("Stopped Varnish")
-}