Varnish monitor generates Events for errors or MonitorGood.

This has required the introduction of pkg/interfaces, defining an
interface for Service Event generation, so that the Varnish controller
can hold a reference to IngressController, without introducing
import cycles for pkg/varnish and pkg/controller.

cmd/main.go

@@ -170,16 +170,16 @@ func main() {
 		// 	informers.WithNamespace(*namespaceF))
 	}
 
-	varnishDone := make(chan error, 1)
-	vController.Start(varnishDone)
-
 	ingController, err := controller.NewIngressController(log, kubeClient,
 		vController, informerFactory, vcrInformerFactory)
 	if err != nil {
 		log.Fatalf("Could not initialize controller: %v")
 		os.Exit(-1)
 	}
+	vController.EvtGenerator(ingController)
+	varnishDone := make(chan error, 1)
 	go handleTermination(log, ingController, vController, varnishDone)
+	vController.Start(varnishDone)
 	informerFactory.Start(informerStop)
 	ingController.Run(*readyfileF)
 }

pkg/controller/svc_event.go

+/*
+ * Copyright (c) 2019 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+package controller
+
+import (
+	"fmt"
+
+	api_v1 "k8s.io/api/core/v1"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/tools/cache"
+)
+
+func (ingc *IngressController) svcEvent(svcKey, evtType, reason, msgFmt string,
+	args ...interface{}) {
+
+	namespace, name, err := cache.SplitMetaNamespaceKey(svcKey)
+	if err != nil {
+		e := fmt.Errorf("Cannot parse service key %s, will not "+
+			"generate event(%s, %s): %v", svcKey, evtType, reason,
+			err)
+		utilruntime.HandleError(e)
+		return
+	}
+	nsSvcs := ingc.listers.svc.Services(namespace)
+	svc, err := nsSvcs.Get(name)
+	if err != nil {
+		e := fmt.Errorf("Cannot retrieve service %s/%s, will not "+
+			"generate event(%s, %s): %v", namespace, name, evtType,
+			reason, err)
+		utilruntime.HandleError(e)
+		return
+	}
+	ingc.recorder.Eventf(svc, evtType, reason, msgFmt, args...)
+}
+
+// SvcInfoEvent generates an Event with type "Normal" for the Service
+// whose namespace/name is svcKey.
+func (ingc *IngressController) SvcInfoEvent(svcKey, reason, msgFmt string,
+	args ...interface{}) {
+
+	ingc.svcEvent(svcKey, api_v1.EventTypeNormal, reason, msgFmt, args...)
+}
+
+// SvcWarnEvent generates an Event with type "Warning" for the Service
+// whose namespace/name is svcKey.
+func (ingc *IngressController) SvcWarnEvent(svcKey, reason, msgFmt string,
+	args ...interface{}) {
+
+	ingc.svcEvent(svcKey, api_v1.EventTypeWarning, reason, msgFmt, args...)
+}

pkg/interfaces/interface.go

+/*
+ * Copyright (c) 2019 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+// Package interfaces defines interfaces that allow pkg/controller and
+// pkg/varnish to access code from one another without introducing
+// import cycles.
+package interfaces
+
+// A SvcEventGenerator defines methods to generate Events for the
+// Service whose namespace/name is svcKey.
+type SvcEventGenerator interface {
+	SvcInfoEvent(svcKey, reason, msgFmt string, args ...interface{})
+	SvcWarnEvent(svcKey, reason, msgFmt string, args ...interface{})
+}

pkg/varnish/monitor.go

 /*
- * Copyright (c) 2018 UPLEX Nils Goroll Systemoptimierung
+ * Copyright (c) 2019 UPLEX Nils Goroll Systemoptimierung
  * All rights reserved
  *
  * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
@@ -35,68 +35,117 @@ import (
 	"code.uplex.de/uplex-varnish/varnishapi/pkg/admin"
 )
 
-const monitorIntvl = time.Second * 30
+const (
+	// XXX monitorIntvl configurable
+	monitorIntvl = time.Second * 30
+	noAdmSecret  = "NoAdminSecret"
+	connectErr   = "ConnectFailure"
+	pingErr      = "PingFailure"
+	statusErr    = "StatusFailure"
+	statusNotRun = "StatusNotRunning"
+	panicErr     = "PanicFailure"
+	panic        = "Panic"
+	vclListErr   = "VCLListFailure"
+	discardErr   = "VCLDiscardFailure"
+	updateErr    = "UpdateFailure"
+	monitorGood  = "MonitorGood"
+)
+
+func (vc *VarnishController) infoEvt(svc, reason, msgFmt string,
+	args ...interface{}) {
+
+	vc.log.Infof(msgFmt, args...)
+	vc.svcEvt.SvcInfoEvent(svc, reason, msgFmt, args...)
+}
+
+func (vc *VarnishController) warnEvt(svc, reason, msgFmt string,
+	args ...interface{}) {
+
+	vc.log.Warnf(msgFmt, args...)
+	vc.svcEvt.SvcWarnEvent(svc, reason, msgFmt, args...)
+}
 
-func (vc *VarnishController) checkInst(inst *varnishInst) {
+func (vc *VarnishController) errorEvt(svc, reason, msgFmt string,
+	args ...interface{}) {
+
+	vc.log.Errorf(msgFmt, args...)
+	vc.svcEvt.SvcWarnEvent(svc, reason, msgFmt, args...)
+}
+
+func (vc *VarnishController) checkInst(svc string, inst *varnishInst) bool {
 	if inst.admSecret == nil {
-		vc.log.Warnf("No admin secret known for endpoint %s", inst.addr)
-		return
+		vc.warnEvt(svc, noAdmSecret,
+			"No admin secret known for endpoint %s", inst.addr)
+		return false
 	}
 	inst.admMtx.Lock()
 	defer inst.admMtx.Unlock()
 
 	adm, err := admin.Dial(inst.addr, *inst.admSecret, admTimeout)
 	if err != nil {
-		vc.log.Errorf("Error connecting to %s: %v", inst.addr, err)
-		return
+		vc.errorEvt(svc, connectErr, "Error connecting to %s: %v",
+			inst.addr, err)
+		return false
 	}
 	defer adm.Close()
 	inst.Banner = adm.Banner
-	vc.log.Infof("Connected to Varnish instance %s", inst.addr)
+	vc.log.Infof("Connected to Varnish instance %s, banner: %s", inst.addr,
+		adm.Banner)
 
 	pong, err := adm.Ping()
 	if err != nil {
-		vc.log.Errorf("Error pinging at %s: %v", inst.addr, err)
-		return
+		vc.errorEvt(svc, pingErr, "Error pinging at %s: %v", inst.addr,
+			err)
+		return false
 	}
 	vc.log.Infof("Succesfully pinged instance %s: %+v", inst.addr, pong)
 
 	state, err := adm.Status()
 	if err != nil {
-		vc.log.Errorf("Error getting status at %s: %v", inst.addr, err)
-		return
+		vc.errorEvt(svc, statusErr, "Error getting status at %s: %v",
+			inst.addr, err)
+		return false
+	}
+	if state == admin.Running {
+		vc.log.Infof("Status at %s: %s", inst.addr, state)
+	} else {
+		vc.warnEvt(svc, statusNotRun, "Status at %s: %s", inst.addr,
+			state)
 	}
-	vc.log.Infof("Status at %s: %s", inst.addr, state)
 
 	panic, err := adm.GetPanic()
 	if err != nil {
-		vc.log.Errorf("Error getting panic at %s: %v", inst.addr, err)
-		return
+		vc.errorEvt(svc, panicErr, "Error getting panic at %s: %v",
+			inst.addr, err)
+		return false
 	}
 	if panic == "" {
 		vc.log.Infof("No panic at %s", inst.addr)
 	} else {
-		vc.log.Warnf("Panic at %s: %s", inst.addr, panic)
+		vc.errorEvt(svc, panic, "Panic at %s: %s", inst.addr, panic)
 		// XXX clear the panic? Should be configurable
 	}
 
 	vcls, err := adm.VCLList()
 	if err != nil {
-		vc.log.Errorf("Error getting VCL list at %s: %v", inst.addr, err)
-		return
+		vc.errorEvt(svc, vclListErr,
+			"Error getting VCL list at %s: %v", inst.addr, err)
+		return false
 	}
 	for _, vcl := range vcls {
 		if strings.HasPrefix(vcl.Name, ingressPrefix) &&
 			vcl.State == admin.ColdState {
 			if err = adm.VCLDiscard(vcl.Name); err != nil {
-				vc.log.Errorf("Error discarding VCL %s at %s: "+
-					"%v", vcl.Name, inst.addr, err)
-				return
+				vc.errorEvt(svc, discardErr,
+					"Error discarding VCL %s at %s: "+
+						"%v", vcl.Name, inst.addr, err)
+				return false
 			}
 			vc.log.Infof("Discarded VCL %s at %s", vcl.Name,
 				inst.addr)
 		}
 	}
+	return true
 }
 
 func (vc *VarnishController) monitor() {
@@ -109,13 +158,22 @@ func (vc *VarnishController) monitor() {
 			vc.log.Infof("Monitoring Varnish instances in %s",
 				svcName)
 
+			good := true
 			for _, inst := range svc.instances {
-				vc.checkInst(inst)
+				if !vc.checkInst(svcName, inst) {
+					good = false
+				}
 			}
 
 			if err := vc.updateVarnishSvc(svcName); err != nil {
-				vc.log.Errorf("Errors updating Varnish "+
-					"Service %s: %+v", svcName, err)
+				vc.errorEvt(svcName, updateErr,
+					"Errors updating Varnish "+
+						"Service %s: %+v", svcName, err)
+				good = false
+			}
+			if good {
+				vc.infoEvt(svcName, monitorGood,
+					"Monitor check good")
 			}
 		}
 	}

pkg/varnish/varnish.go

@@ -44,6 +44,7 @@ import (
 	"sync"
 	"time"
 
+	"code.uplex.de/uplex-varnish/k8s-ingress/pkg/interfaces"
 	"code.uplex.de/uplex-varnish/k8s-ingress/pkg/varnish/vcl"
 	"code.uplex.de/uplex-varnish/varnishapi/pkg/admin"
 
@@ -134,6 +135,7 @@ type varnishSvc struct {
 // their current states.
 type VarnishController struct {
 	log     *logrus.Logger
+	svcEvt  interfaces.SvcEventGenerator
 	svcs    map[string]*varnishSvc
 	secrets map[string]*[]byte
 	errChan chan error
@@ -148,7 +150,8 @@ type VarnishController struct {
 // TEMPLATE_DIR. If the env variable does not exist, use the current
 // working directory.
 func NewVarnishController(
-	log *logrus.Logger, tmplDir string) (*VarnishController, error) {
+	log *logrus.Logger,
+	tmplDir string) (*VarnishController, error) {
 
 	if tmplDir == "" {
 		tmplEnv, exists := os.LookupEnv("TEMPLATE_DIR")
@@ -166,6 +169,13 @@ func NewVarnishController(
 	}, nil
 }
 
+// EvtGenerator sets the object that implements interface
+// SvcEventGenerator, and will be used by the monitor goroutine to
+// generate Events for Varnish Services.
+func (vc *VarnishController) EvtGenerator(svcEvt interfaces.SvcEventGenerator) {
+	vc.svcEvt = svcEvt
+}
+
 // Start initiates the Varnish controller and starts the monitor
 // goroutine.
 func (vc *VarnishController) Start(errChan chan error) {