Initial implementation of IngressClass support.

Verified with the hello world example using kubectl/yaml deployment.

The other tests/examples, including all helm deployments, do not work
as of this commit, until they are adapted to the new logic for
configuring the Ingress class.

charts/viking-controller/templates/ingress-class.yaml

+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  name: viking-controller.uplex.de
+spec:
+  controller: viking.uplex.de/ingress-controller

cmd/main.go

@@ -56,7 +56,10 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 )
 
-const defIncomplRetryDelay = 5 * time.Second
+const (
+	defIncomplRetryDelay = 5 * time.Second
+	defIngressClass      = "viking.uplex.de/ingress-controller"
+)
 
 var (
 	versionF = flag.Bool("version", false, "print version and exit")
@@ -77,9 +80,12 @@ var (
 			"Monitor deactivated when <= 0s")
 	metricsPortF = flag.Uint("metricsport", 8080,
 		"port at which to listen for the /metrics endpoint")
-	ingressClassF = flag.String("class", "varnish", "value of the Ingress "+
-		"annotation kubernetes.io/ingress.class\nthe controller only "+
-		"considers Ingresses with this value for the\nannotation")
+	ingressClassF = flag.String("class", defIngressClass,
+		"IngressClass controller identifier\n"+
+			"the controller only considers Ingresses that\n"+
+			"specify an IngressClass with this value in its\n"+
+			"controller field, unless that IngressClass is also\n"+
+			"specified as the cluster default")
 	resyncPeriodF = flag.Duration("resyncPeriod", 30*time.Second,
 		"if non-zero, re-update the controller with the state of\n"+
 			"the cluster this often, even if nothing has changed,\n"+

deploy/Makefile

@@ -93,6 +93,7 @@ deploy-controller-helm:
 deploy-controller-kubectl:
 	@kubectl apply -f serviceaccount-controller.yaml
 	@kubectl apply -f rbac-controller.yaml
+	@kubectl apply -f ingress-class.yaml
 	@kubectl apply -f varnishcfg-crd.yaml
 	@kubectl apply -f backendcfg-crd.yaml
 	@kubectl apply -f templatecfg-crd.yaml
@@ -191,6 +192,7 @@ endif
 	@kubectl delete -f templatecfg-crd.yaml
 	@kubectl delete -f backendcfg-crd.yaml
 	@kubectl delete -f varnishcfg-crd.yaml
+	@kubectl delete -f ingress-class.yaml
 	@kubectl delete -f rbac-controller.yaml
 	@kubectl delete -f serviceaccount-controller.yaml
 	@echo Waiting for the viking-controller Pod to be deleted

deploy/ingress-class.yaml

+../charts/viking-controller/templates/ingress-class.yaml
\ No newline at end of file

deploy/rbac-controller.yaml

@@ -31,6 +31,7 @@ rules:
   - networking.k8s.io
   resources:
   - ingresses
+  - ingressclasses
   verbs:
   - list
   - watch

examples/hello/cafe-ingress.yaml

@@ -2,9 +2,8 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: cafe-ingress-varnish
-  annotations:
-    kubernetes.io/ingress.class: "varnish"
 spec:
+  ingressClassName: viking-controller.uplex.de
   rules:
   - host: cafe.example.com
     http:

pkg/controller/controller.go

@@ -61,6 +61,7 @@ import (
 
 type infrmrs struct {
 	ing   cache.SharedIndexInformer
+	ingcl cache.SharedIndexInformer
 	svc   cache.SharedIndexInformer
 	endp  cache.SharedIndexInformer
 	vsecr cache.SharedIndexInformer
@@ -110,6 +111,7 @@ type SyncObj struct {
 // read data from the client-go cache.
 type Listers struct {
 	ing   net_v1_listers.IngressLister
+	ingcl net_v1_listers.IngressClassLister
 	svc   core_v1_listers.ServiceLister
 	endp  core_v1_listers.EndpointsLister
 	tsecr core_v1_listers.SecretLister
@@ -187,6 +189,7 @@ func NewIngressController(
 
 	ingc.informers = &infrmrs{
 		ing:   infFactory.Networking().V1().Ingresses().Informer(),
+		ingcl: infFactory.Networking().V1().IngressClasses().Informer(),
 		svc:   infFactory.Core().V1().Services().Informer(),
 		endp:  infFactory.Core().V1().Endpoints().Informer(),
 		vsecr: vsecrInfFactory.Core().V1().Secrets().Informer(),
@@ -208,6 +211,7 @@ func NewIngressController(
 	}
 
 	ingc.informers.ing.AddEventHandler(evtFuncs)
+	ingc.informers.ingcl.AddEventHandler(evtFuncs)
 	ingc.informers.svc.AddEventHandler(evtFuncs)
 	ingc.informers.endp.AddEventHandler(evtFuncs)
 	ingc.informers.tsecr.AddEventHandler(evtFuncs)
@@ -229,6 +233,7 @@ func NewIngressController(
 
 	ingc.listers = &Listers{
 		ing:   infFactory.Networking().V1().Ingresses().Lister(),
+		ingcl: infFactory.Networking().V1().IngressClasses().Lister(),
 		svc:   infFactory.Core().V1().Services().Lister(),
 		endp:  infFactory.Core().V1().Endpoints().Lister(),
 		vsecr: vsecrInfFactory.Core().V1().Secrets().Lister(),
@@ -286,6 +291,8 @@ func incWatchCounter(obj interface{}, sync string) {
 	switch obj.(type) {
 	case *net_v1.Ingress:
 		watchCounters.WithLabelValues("Ingress", sync).Inc()
+	case *net_v1.IngressClass:
+		watchCounters.WithLabelValues("IngressClass", sync).Inc()
 	case *api_v1.Service:
 		watchCounters.WithLabelValues("Service", sync).Inc()
 	case *api_v1.Endpoints:
@@ -336,6 +343,8 @@ func (ingc *IngressController) updateObj(old, new interface{}) {
 			switch old.(type) {
 			case *net_v1.Ingress:
 				kind = "Ingress"
+			case *net_v1.IngressClass:
+				kind = "IngressClass"
 			case *api_v1.Service:
 				kind = "Service"
 			case *api_v1.Endpoints:
@@ -415,6 +424,7 @@ func (ingc *IngressController) Run(
 
 	ingc.log.Info("Launching informers")
 	go ingc.informers.ing.Run(ingc.ctx.Done())
+	go ingc.informers.ingcl.Run(ingc.ctx.Done())
 	go ingc.informers.svc.Run(ingc.ctx.Done())
 	go ingc.informers.endp.Run(ingc.ctx.Done())
 	go ingc.informers.tsecr.Run(ingc.ctx.Done())
@@ -451,6 +461,7 @@ func (ingc *IngressController) Run(
 
 	syncs := []cache.InformerSynced{
 		ingc.informers.ing.HasSynced,
+		ingc.informers.ingcl.HasSynced,
 		ingc.informers.svc.HasSynced,
 		ingc.informers.endp.HasSynced,
 		ingc.informers.tsecr.HasSynced,

pkg/controller/endpoints.go

@@ -69,7 +69,12 @@ func (worker *NamespaceWorker) syncEndp(key string) update.Status {
 	worker.log.Tracef("Update ingresses for endpoints %s", key)
 	requeued := make([]string, 0, len(ings))
 	for _, ing := range ings {
-		if !worker.isVikingIngress(ing) {
+		if isViking, err := worker.isVikingIngress(ing); err != nil {
+			return update.MakeRecoverable(
+				"Error checking if Ingress %s/%s is to be "+
+					"implemented by viking: %v",
+				ing.Namespace, ing.Name, err)
+		} else if !isViking {
 			worker.log.Tracef("Ingress %s/%s: not Varnish",
 				ing.Namespace, ing.Name)
 			continue

pkg/controller/ingress.go

@@ -163,7 +163,9 @@ func (worker *NamespaceWorker) getIngsForVarnishSvc(
 
 	ings4Svc := make([]*net_v1.Ingress, 0)
 	for _, ing := range ings {
-		if !worker.isVikingIngress(ing) {
+		if isViking, err := worker.isVikingIngress(ing); err != nil {
+			return nil, err
+		} else if !isViking {
 			continue
 		}
 		namespace := ing.Namespace
@@ -1287,17 +1289,50 @@ func (worker *NamespaceWorker) addOrUpdateIng(
 	return worker.updateIngStatus(svc, ings)
 }
 
-// We only handle Ingresses with the class annotation with the value
-// given as the "class" flag (default "varnish").
-func (worker *NamespaceWorker) isVikingIngress(ing *net_v1.Ingress) bool {
+// We only handle Ingresses whose IngressClass is set to the value
+// given in the "class" CLI option (default
+// "viking.uplex.de/ingress-controller").  Unless that IngressClass
+// has been specified as the cluster default, in which case we also
+// handle Ingresses without any ingress class specification.
+//
+// For backward compatibility, we give precedence to the deprecated
+// ingress.class annotation. The annotation should not be used for new
+// projects, and support will be removed in a future version.
+func (worker *NamespaceWorker) isVikingIngress(
+	ing *net_v1.Ingress,
+) (bool, error) {
 	class, exists := ing.Annotations[ingressClassKey]
-	return exists && class == worker.ingClass
+	if exists {
+		return class == worker.ingClass, nil
+	}
+	if ing.Spec.IngressClassName == nil ||
+		*ing.Spec.IngressClassName == "" {
+		isDefault, err := worker.isVikingDefaultIngressClass()
+		if err != nil {
+			return false, err
+		}
+		return isDefault, nil
+	}
+	name, err := worker.vikingIngressClassName()
+	if err != nil {
+		return false, err
+	}
+	if name == "" {
+		return false, fmt.Errorf("No IngressClass found with "+
+			"spec.controller: %s", worker.ingClass)
+	}
+	return *ing.Spec.IngressClassName == name, nil
 }
 
 func (worker *NamespaceWorker) chkAddOrUpdateIng(
 	ing *net_v1.Ingress) update.Status {
 
-	if !worker.isVikingIngress(ing) {
+	if isViking, err := worker.isVikingIngress(ing); err != nil {
+		return update.MakeRecoverable(
+			"Error checking if Ingress %s/%s is to be "+
+				"implemented by viking: %v", ing.Namespace,
+			ing.Name, err)
+	} else if !isViking {
 		syncCounters.WithLabelValues(worker.namespace, "Ingress",
 			"Ignore").Inc()
 		return update.MakeNoop("Ignoring Ingress %s/%s, "+

pkg/controller/ingress_class.go

+/*
+ * Copyright (c) 2021 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+package controller
+
+import (
+	"code.uplex.de/uplex-varnish/k8s-ingress/pkg/update"
+
+	net_v1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+)
+
+const defIngClassKey = "ingressclass.kubernetes.io/is-default-class"
+
+func (worker *NamespaceWorker) isVikingDefaultIngressClass() (bool, error) {
+	ingClasses, err := worker.listers.ingcl.List(labels.Everything())
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	for _, ingClass := range ingClasses {
+		isDefault, exists := ingClass.Annotations[defIngClassKey]
+		if exists {
+			if isDefault == "true" {
+				return ingClass.Spec.Controller == worker.ingClass, nil
+			} else if ingClass.Spec.Controller == worker.ingClass {
+				return false, nil
+			}
+		} else if ingClass.Spec.Controller == worker.ingClass {
+			return false, nil
+		}
+	}
+	return false, nil
+}
+
+func (worker *NamespaceWorker) vikingIngressClassName() (string, error) {
+	ingClasses, err := worker.listers.ingcl.List(labels.Everything())
+	if err != nil {
+		return "", err
+	}
+	for _, ingClass := range ingClasses {
+		if ingClass.Spec.Controller == worker.ingClass {
+			return ingClass.Name, nil
+		}
+	}
+	return "", nil
+}
+
+func (worker *NamespaceWorker) syncIngClass(key string) update.Status {
+	worker.log.Infof("Syncing IngressClass: %s", key)
+	ingClass, err := worker.listers.ingcl.Get(key)
+	if err != nil {
+		return IncompleteIfNotFound(err, "%v", err)
+	}
+	//	worker.log.Tracef("IngressClass %s: %+v", ingClass.Name, ingClass)
+	worker.log.Infof("IngressClass %s: %+v", ingClass.Name, ingClass)
+
+	worker.log.Infof("IngressClass %s controller: %s", ingClass.Name,
+		ingClass.Spec.Controller)
+	if ingClass.Spec.Controller == worker.ingClass {
+		worker.log.Infof("IngressClass %s is the viking controller "+
+			"ingress class", ingClass.Name)
+	}
+	isDefault, exists := ingClass.Annotations[defIngClassKey]
+	if exists && isDefault == "true" {
+		worker.log.Infof("IngressClass %s is the cluster default "+
+			"ingress class", ingClass.Name)
+	}
+	return update.MakeSuccess("")
+}
+
+func (worker *NamespaceWorker) addIngClass(key string) update.Status {
+	return worker.syncIngClass(key)
+}
+
+func (worker *NamespaceWorker) updateIngClass(key string) update.Status {
+	return worker.syncIngClass(key)
+}
+
+func (worker *NamespaceWorker) deleteIngClass(obj interface{}) update.Status {
+	ingClass, ok := obj.(*net_v1.IngressClass)
+	if !ok || ingClass == nil {
+		return update.MakeNoop("Delete IngressClass: not found: %v",
+			obj)
+	}
+	return update.MakeSuccess("")
+}

pkg/controller/secret.go

@@ -77,7 +77,12 @@ func (worker *NamespaceWorker) getIngsForTLSSecret(
 	}
 
 	for _, ing := range nsIngs {
-		if !worker.isVikingIngress(ing) {
+		if isViking, err := worker.isVikingIngress(ing); err != nil {
+			return nil, update.MakeRecoverable(
+				"Error checking if Ingress %s/%s is to be "+
+					"implemented by viking: %v",
+				ing.Namespace, ing.Name, err)
+		} else if !isViking {
 			continue
 		}
 		for _, tls := range ing.Spec.TLS {

pkg/controller/secret_test.go

@@ -47,18 +47,22 @@ import (
 func setupIngLister(
 	client *fake.Clientset,
 	ns string,
-) net_v1_listers.IngressNamespaceLister {
+) (net_v1_listers.IngressNamespaceLister, net_v1_listers.IngressClassLister) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	infFactory := informers.NewSharedInformerFactory(client, 0)
 	ingInformer := infFactory.Networking().V1().Ingresses().Informer()
+	ingclInformer := infFactory.Networking().V1().IngressClasses().
+		Informer()
 	ingLister := infFactory.Networking().V1().Ingresses().Lister()
+	ingclLister := infFactory.Networking().V1().IngressClasses().Lister()
 	ingNsLister := ingLister.Ingresses(ns)
 
 	infFactory.Start(ctx.Done())
-	cache.WaitForCacheSync(ctx.Done(), ingInformer.HasSynced)
-	return ingNsLister
+	cache.WaitForCacheSync(ctx.Done(),
+		ingInformer.HasSynced, ingclInformer.HasSynced)
+	return ingNsLister, ingclLister
 }
 
 func TestIngsForTLSSecret(t *testing.T) {
@@ -125,12 +129,15 @@ func TestIngsForTLSSecret(t *testing.T) {
 		},
 	)
 
-	ingNsLister := setupIngLister(client, ns)
+	ingNsLister, ingclLister := setupIngLister(client, ns)
 	worker := &NamespaceWorker{
 		log:      &logrus.Logger{Out: ioutil.Discard},
 		ing:      ingNsLister,
 		ingClass: ingClass,
 	}
+	worker.listers = &Listers{
+		ingcl: ingclLister,
+	}
 
 	secret := &api_v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
@@ -227,12 +234,15 @@ func TestNoIngsForTLSSecret(t *testing.T) {
 	ingClass := "viking"
 
 	client := fake.NewSimpleClientset()
-	ingNsLister := setupIngLister(client, ns)
+	ingNsLister, ingclLister := setupIngLister(client, ns)
 	worker := &NamespaceWorker{
 		log:      &logrus.Logger{Out: ioutil.Discard},
 		ing:      ingNsLister,
 		ingClass: ingClass,
 	}
+	worker.listers = &Listers{
+		ingcl: ingclLister,
+	}
 	secret := &api_v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: ns,

pkg/controller/service.go

@@ -129,7 +129,12 @@ func (worker *NamespaceWorker) enqueueIngressForService(
 	}
 	requeued := make([]string, 0, len(ings))
 	for _, ing := range ings {
-		if !worker.isVikingIngress(ing) {
+		if isViking, err := worker.isVikingIngress(ing); err != nil {
+			return update.MakeRecoverable(
+				"Error checking if Ingress %s/%s is to be "+
+					"implemented by viking: %v",
+				ing.Namespace, ing.Name, err)
+		} else if !isViking {
 			continue
 		}
 		worker.queue.Add(&SyncObj{Type: Update, Obj: ing})

pkg/controller/varnishconfig.go

@@ -61,7 +61,12 @@ func (worker *NamespaceWorker) enqueueIngsForVcfg(
 		return update.MakeRecoverable("%v", err)
 	}
 	for _, ing := range ings {
-		if !worker.isVikingIngress(ing) {
+		if isViking, err := worker.isVikingIngress(ing); err != nil {
+			return update.MakeRecoverable(
+				"Error checking if Ingress %s/%s is to be "+
+					"implemented by viking: %v",
+				ing.Namespace, ing.Name, err)
+		} else if !isViking {
 			continue
 		}
 		vSvc, err := worker.getVarnishSvcForIng(ing)

pkg/controller/worker.go

@@ -82,6 +82,9 @@ type NamespaceWorker struct {
 	incomplRetryDelay time.Duration
 }
 
+// Note that there is no NamespaceLister for IngressClass, which is not
+// namespaced. We can only use listers.ingcl
+
 func (worker *NamespaceWorker) event(obj interface{}, evtType, reason,
 	msgFmt string, args ...interface{}) {
 
@@ -101,6 +104,10 @@ func (worker *NamespaceWorker) event(obj interface{}, evtType, reason,
 		ing, _ := eventObj.(*net_v1.Ingress)
 		worker.recorder.Eventf(ing, evtType, reason, msgFmt, args...)
 		kind = "Ingress"
+	case *net_v1.IngressClass:
+		ingcl, _ := eventObj.(*net_v1.IngressClass)
+		worker.recorder.Eventf(ingcl, evtType, reason, msgFmt, args...)
+		kind = "IngressClass"
 	case *api_v1.Service:
 		svc, _ := eventObj.(*api_v1.Service)
 		worker.recorder.Eventf(svc, evtType, reason, msgFmt, args...)
@@ -173,6 +180,8 @@ func (worker *NamespaceWorker) dispatch(obj interface{}) update.Status {
 		switch syncObj.Obj.(type) {
 		case *net_v1.Ingress:
 			return worker.addIng(key)
+		case *net_v1.IngressClass:
+			return worker.addIngClass(key)
 		case *api_v1.Service:
 			return worker.addSvc(key)
 		case *api_v1.Endpoints:
@@ -193,6 +202,8 @@ func (worker *NamespaceWorker) dispatch(obj interface{}) update.Status {
 		switch syncObj.Obj.(type) {
 		case *net_v1.Ingress:
 			return worker.updateIng(key)
+		case *net_v1.IngressClass:
+			return worker.updateIngClass(key)
 		case *api_v1.Service:
 			return worker.updateSvc(key)
 		case *api_v1.Endpoints:
@@ -220,6 +231,8 @@ func (worker *NamespaceWorker) dispatch(obj interface{}) update.Status {
 		switch deletedObj.(type) {
 		case *net_v1.Ingress:
 			return worker.deleteIng(deletedObj)
+		case *net_v1.IngressClass:
+			return worker.deleteIngClass(deletedObj)
 		case *api_v1.Service:
 			return worker.deleteSvc(deletedObj)
 		case *api_v1.Endpoints: