Add the maxSyncRetries CLI option.

$ vikingctrl --help
[...]
  -maxSyncRetries uint
    	maximum number of retires for cluster synchronizations
    	that fail due to recoverable errors, or because
    	necessary information is missing. 0 for unlimited
    	retries (default 0)
[...]

IOW set a maximum number of re-queues for SyncIncomplete and
SyncRecoverable type failures. By default unlimited.

While here, update some missing function parameter docs.

cmd/main.go

@@ -99,6 +99,11 @@ var (
 	varnishImplF = flag.String("varnishImpl", "",
 		"set to 'klarlack' to enable features only implemented by\n"+
 			"the klarlack image for Varnish Ingress")
+	maxSyncRetriesF = flag.Uint("maxSyncRetries", 0,
+		"maximum number of retires for cluster synchronizations\n"+
+			"that fail due to recoverable errors, or because\n"+
+			"necessary information is missing. 0 for unlimited\n"+
+			"retries (default 0)")
 
 	logFormat = logrus.TextFormatter{
 		DisableColors: true,
@@ -190,6 +195,9 @@ func main() {
 	if *devModeF {
 		log.Info("devmode enabled")
 	}
+	if *maxSyncRetriesF > 0 {
+		log.Infof("maxSyncRetries = %d", *maxSyncRetriesF)
+	}
 
 	log.Info("Die Würde des Menschen ist unantastbar")
 	log.Info("Black Lives Matter")
@@ -245,7 +253,7 @@ func main() {
 		*ingressClassF, *namespaceF, *devModeF, *varnishImplF,
 		kubeClient, vController, hController, informerFactory,
 		vcrInformerFactory, vsecrInformerFactory, tsecrInformerFactory,
-		*incomplRetryDelayF)
+		*incomplRetryDelayF, *maxSyncRetriesF)
 	if err != nil {
 		log.Fatalf("Could not initialize controller: %v", err)
 		os.Exit(-1)

deploy/Makefile

@@ -85,6 +85,10 @@ ifndef LOG_LEVEL
        LOG_LEVEL=INFO
 endif
 
+ifndef MAX_RETRIES
+       MAX_RETRIES=0
+endif
+
 all: deploy
 
 deploy-controller-helm:
@@ -93,7 +97,7 @@ deploy-controller-helm:
 		--set vikingController.image.repository=$(CONTROLLER_IMAGE) \
 		--set vikingController.image.tag=$(CONTROLLER_TAG) \
 		--set vikingController.varnishImpl=$(VARNISH) \
-		--set vikingController.extraArgs="{-log-level=$(LOG_LEVEL)}"
+		--set vikingController.extraArgs="{-log-level=$(LOG_LEVEL),-maxSyncRetries=$(MAX_RETRIES)}"
 
 deploy-controller-kubectl:
 	@kubectl apply -f serviceaccount-controller.yaml

pkg/controller/controller.go

@@ -140,11 +140,19 @@ type IngressController struct {
 //
 //	log: logger initialized at startup
 //	ingClass: value of the ingress.class Ingress annotation
+//	namespace: namespace to which the controller is restricted
+//	devMode: true if development mode was specfied at invocation
+//	varnishImpl: name of the varnish implementation (varnish, klarlack)
 //	kubeClient: k8s client initialized at startup
 //	vc: Varnish controller
+//	hc: haproxy controller
 //	infFactory: SharedInformerFactory to create informers & listers for
 //	            the k8s standard client APIs
 //	vcrInfFactory: SharedInformerFactory for the project's own client APIs
+//	vsecrInfFactory: SharedInformerFactory for Secrets annotated for viking
+//	tsecrInfFactory: SharedInformerFactory for TLS Secrets
+//	incomplRetryDelay: duration until SyncIncomplete errors are re-queued
+//	maxSyncRetries: max re-queues for SyncIncomplete or SyncRecoverable
 func NewIngressController(
 	log *logrus.Logger,
 	ingClass string,
@@ -159,6 +167,7 @@ func NewIngressController(
 	vsecrInfFactory informers.SharedInformerFactory,
 	tsecrInfFactory informers.SharedInformerFactory,
 	incomplRetryDelay time.Duration,
+	maxSyncRetries uint,
 ) (*IngressController, error) {
 
 	ingc := IngressController{
@@ -250,7 +259,7 @@ func NewIngressController(
 
 	ingc.nsQs = NewNamespaceQueues(ingc.log, ingClass, vc, hc, ingc.listers,
 		ingc.client, ingc.recorder, incomplRetryDelay, devMode,
-		varnishImpl)
+		varnishImpl, maxSyncRetries)
 
 	return &ingc, nil
 }

pkg/controller/worker.go

@@ -60,6 +60,7 @@ import (
 // namespace is synced separately and sequentially, and all of the
 // namespaces are synced in parallel.
 type NamespaceWorker struct {
+	maxSyncRetries    uint
 	namespace         string
 	ingClass          string
 	varnishImpl       string
@@ -254,6 +255,20 @@ func (worker *NamespaceWorker) dispatch(obj interface{}) update.Status {
 	}
 }
 
+func (worker *NamespaceWorker) tooManyRetries(obj interface{}) bool {
+	if worker.maxSyncRetries == 0 {
+		return false
+	}
+	return uint(worker.queue.NumRequeues(obj)) >= worker.maxSyncRetries
+}
+
+func (worker *NamespaceWorker) fatalRetryStatus(
+	status update.Status,
+) update.Status {
+	return update.MakeFatal("max sync retries %d exceeded, was: (%s) %s",
+		worker.maxSyncRetries, status.Reason(), status)
+}
+
 func (worker *NamespaceWorker) next() {
 	obj, quit := worker.queue.Get()
 	if quit {
@@ -283,9 +298,21 @@ func (worker *NamespaceWorker) next() {
 		worker.syncFailure(obj, msgPfx, status)
 		worker.queue.Forget(obj)
 	case update.Recoverable:
+		if worker.tooManyRetries(obj) {
+			worker.syncFailure(obj, msgPfx,
+				worker.fatalRetryStatus(status))
+			worker.queue.Forget(obj)
+			return
+		}
 		worker.syncFailure(obj, msgPfx, status)
 		worker.queue.AddRateLimited(obj)
 	case update.Incomplete:
+		if worker.tooManyRetries(obj) {
+			worker.syncFailure(obj, msgPfx,
+				worker.fatalRetryStatus(status))
+			worker.queue.Forget(obj)
+			return
+		}
 		worker.syncFailure(obj, msgPfx, status)
 		worker.queue.AddAfter(obj, worker.incomplRetryDelay)
 	}
@@ -321,16 +348,22 @@ type NamespaceQueues struct {
 	wg                *sync.WaitGroup
 	incomplRetryDelay time.Duration
 	devMode           bool
+	maxSyncRetries    uint
 }
 
 // NewNamespaceQueues creates a NamespaceQueues object.
 //
 //	log: logger initialized at startup
 //	ingClass: value of the ingress.class Ingress annotation
-//	vController: Varnish controller initialied at startup
+//	vController: Varnish controller initialized at startup
+//	hController: haproxy controller initialized at startup
 //	listers: client-go/lister instance for each resource type
 //	client: k8s API client initialized at startup
 //	recorder: Event broadcaster initialized at startup
+//	incomplRetryDelay: duration until SyncIncomplete errors are re-queued
+//	devMode: true if development mode was specified at invocation
+//	varnishImpl: name of the varnish implementation (varnish, klarlack)
+//	maxSyncRetries: max re-queues for SyncIncomplete or SyncRecoverable
 func NewNamespaceQueues(
 	log *logrus.Logger,
 	ingClass string,
@@ -342,6 +375,7 @@ func NewNamespaceQueues(
 	incomplRetryDelay time.Duration,
 	devMode bool,
 	varnishImpl string,
+	maxSyncRetries uint,
 ) *NamespaceQueues {
 
 	q := workqueue.NewNamedRateLimitingQueue(
@@ -360,6 +394,7 @@ func NewNamespaceQueues(
 		wg:                new(sync.WaitGroup),
 		incomplRetryDelay: incomplRetryDelay,
 		devMode:           devMode,
+		maxSyncRetries:    maxSyncRetries,
 	}
 }
 
@@ -413,6 +448,7 @@ func (qs *NamespaceQueues) next() {
 			recorder:          qs.recorder,
 			wg:                qs.wg,
 			incomplRetryDelay: qs.incomplRetryDelay,
+			maxSyncRetries:    qs.maxSyncRetries,
 		}
 		if qs.devMode {
 			worker.tcfg = qs.listers.tcfg.TemplateConfigs(ns)