Merge branch 'bcfg-tls-onload'

Makefile

@@ -70,7 +70,7 @@ k8s-ingress: vikingctrl
 check: build
 	go vet $(CODE_SUBDIRS)
 	golint $(CODE_SUBDIRS)
-	go test -v ./pkg/controller/... ./pkg/varnish/...
+	go test -v ./pkg/controller/... ./pkg/varnish/... ./pkg/net/...
 
 test: check
 

examples/self-sharding/verify.sh

@@ -11,6 +11,13 @@ LOCALPORT=${LOCALPORT:-8888}
 wait_until_ready app=varnish-ingress
 wait_until_configured app=varnish-ingress
 
+# Cached responses from previous tests occasionally interfere with the
+# current test, so issue a broad ban first.
+PODS=$(kubectl get pod -l app=varnish-ingress -o=name)
+for pod in ${PODS}; do
+    kubectl exec -t -c varnish ${pod} -- varnishadm -n /run/varnish-home ban obj.status == 200
+done
+
 kubectl port-forward svc/varnish-ingress ${LOCALPORT}:80 >/dev/null &
 trap 'kill $(jobs -p)' EXIT
 wait_for_port ${LOCALPORT}
@@ -22,5 +29,5 @@ wait_for_port ${LOCALPORT}
 # For now, wait longer to run the verification. Should investigate why
 # Varnish is not properly configured immediately after self-sharding is
 # deployed.
-sleep 10
+sleep 20
 varnishtest ${TESTOPTS} -Dlocalport=${LOCALPORT} cafe.vtc

examples/tls/hello/verify.sh

@@ -21,7 +21,7 @@ trap 'kill $(jobs -p)' EXIT
 wait_for_port ${LOCALPORT}
 set +e
 
-sleep 1
+sleep 10
 # The test may be skipped (exit status 77) if haproxy is not installed.
 varnishtest ${TESTOPTS} -Dlocalport=${LOCALPORT} cafe.vtc
 ret=$?

examples/tls/onload/verify.sh

@@ -14,6 +14,6 @@ wait_for_port ${LOCALPORT}
 
 # Give VMOD dynamic and DNS time to determine the IP address of the
 # ExternalName Service.
-sleep 10
+sleep 20
 
 varnishtest ${TESTOPTS} -Dlocalport=${LOCALPORT} cafe.vtc

pkg/controller/backendconfig.go

@@ -38,19 +38,17 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 )
 
-func (worker *NamespaceWorker) enqueueIngsForBackendSvcs(svcs []string,
-	namespace, name string) update.Status {
-
-	svc2ing := make(map[string]*net_v1.Ingress)
+func (worker *NamespaceWorker) svc2IngMap(
+	svcs []string,
+) (map[string]*net_v1.Ingress, error) {
 	ings, err := worker.ing.List(labels.Everything())
 	if errors.IsNotFound(err) {
-		return update.MakeNoop(
-			"BackendConfig %s/%s: no Ingresses found in workspace %s",
-			namespace, name, worker.namespace)
+		return nil, nil
 	}
 	if err != nil {
-		return update.MakeRecoverable("%v", err)
+		return nil, err
 	}
+	svc2ing := make(map[string]*net_v1.Ingress)
 	for _, ing := range ings {
 		if ing.Spec.DefaultBackend != nil {
 			svc2ing[ing.Spec.DefaultBackend.Service.Name] = ing
@@ -64,7 +62,22 @@ func (worker *NamespaceWorker) enqueueIngsForBackendSvcs(svcs []string,
 			}
 		}
 	}
+	return svc2ing, nil
+}
 
+func (worker *NamespaceWorker) enqueueIngsForBackendSvcs(
+	svcs []string,
+	namespace, name string,
+) update.Status {
+	svc2ing, err := worker.svc2IngMap(svcs)
+	if err != nil {
+		return update.MakeRecoverable("%v", err)
+	}
+	if svc2ing == nil {
+		return update.MakeNoop(
+			"BackendConfig %s/%s: no Ingresses found in workspace %s",
+			namespace, name, worker.namespace)
+	}
 	svcSet := make(map[string]struct{})
 	for _, svc := range svcs {
 		if _, exists := svcSet[svc]; exists {
@@ -83,6 +96,44 @@ func (worker *NamespaceWorker) enqueueIngsForBackendSvcs(svcs []string,
 		"BackendConfig %s/%s: re-queued Ingress(es)", namespace, name)
 }
 
+func (worker *NamespaceWorker) bcfgDeleteOnload(
+	svc2ing map[string]*net_v1.Ingress,
+	namespace, name string,
+) error {
+	if svc2ing == nil || len(svc2ing) == 0 {
+		worker.log.Infof("BackendConfig %s/%s: no Ingresses found "+
+			"in namespace %s specifying Services with TLS onload",
+			namespace, name, worker.namespace)
+		return nil
+	}
+	for svc, ing := range svc2ing {
+		vSvc, err := worker.getVarnishSvcForIng(ing)
+		if err != nil {
+			if errors.IsNotFound(err) {
+				worker.log.Infof("BackendConfig %s/%s: "+
+					"no viking Service found for "+
+					"Ingress %s/%s, ignoring",
+					namespace, name, ing.Namespace,
+					ing.Name)
+				continue
+			}
+			return err
+		}
+		worker.log.Infof("BackendConfig %s/%s: delete onloader at %s",
+			namespace, name, svc)
+		err = worker.hController.DeleteOnldr(svc, namespace+"/"+name)
+		if err != nil {
+			worker.log.Errorf(
+				"BackendConfig %s/%s: error deleting onload "+
+					"service %s at %s/%s: %v",
+				namespace, name, svc, vSvc.Namespace, vSvc.Name,
+				err)
+			return err
+		}
+	}
+	return nil
+}
+
 func (worker *NamespaceWorker) syncBcfg(key string) update.Status {
 	worker.log.Infof("Syncing BackendConfig: %s/%s", worker.namespace, key)
 	bcfg, err := worker.bcfg.Get(key)
@@ -128,6 +179,21 @@ func (worker *NamespaceWorker) deleteBcfg(obj interface{}) update.Status {
 	}
 	worker.log.Infof("Deleting BackendConfig: %s/%s", bcfg.Namespace,
 		bcfg.Name)
+	if bcfg.Spec.TLS != nil && worker.varnishImpl == "klarlack" {
+		worker.log.Infof("BackendConfig %s/%s: delete TLS onload",
+			bcfg.Namespace, bcfg.Name)
+		svc2ing, err := worker.svc2IngMap(bcfg.Spec.Services)
+		if err != nil {
+			return update.MakeRecoverable("%v", err)
+		}
+		err = worker.bcfgDeleteOnload(svc2ing, bcfg.Namespace,
+			bcfg.Name)
+		if err != nil {
+			return update.MakeRecoverable("%v", err)
+		}
+		worker.log.Infof("BackendConfig %s/%s: TLS onload deleted",
+			bcfg.Namespace, bcfg.Name)
+	}
 	return worker.enqueueIngsForBackendSvcs(bcfg.Spec.Services,
 		bcfg.Namespace, bcfg.Name)
 }

pkg/controller/ingress.go

@@ -980,6 +980,7 @@ func (worker *NamespaceWorker) ings2OffloaderSpec(
 		Namespace: svc.Namespace,
 		Name:      svc.Name,
 		Secrets:   make([]haproxy.SecretSpec, 0),
+		Onload:    make(map[string]haproxy.OnloadSpec),
 	}
 	for _, ing := range ings {
 		for _, tls := range ing.Spec.TLS {
@@ -1176,10 +1177,15 @@ func (worker *NamespaceWorker) addOrUpdateIng(
 			return update.MakeRecoverable("%v", err)
 		}
 		if worker.hController.HasOffloader(svcKey) {
+			worker.log.Infof("Deleting haproxy config for %s",
+				svcKey)
 			if status := worker.hController.
 				DeleteOffldSvc(svcKey); status.IsError() {
 				return status
 			}
+		} else {
+			worker.log.Infof("No haproxy config found for %s",
+				svcKey)
 		}
 		return update.MakeNoop("")
 	}
@@ -1249,16 +1255,8 @@ func (worker *NamespaceWorker) addOrUpdateIng(
 		return update.MakeFatal("%v", err)
 	}
 	offldrSpec.Defaults = &haproxyDefSpec
-	if len(onlds) > 0 {
-		if len(onlds) > 1 {
-			// XXX
-			return update.MakeFatal(
-				"Multiple TLS onload configs currently not " +
-					"supported")
-		} // else {
-		for _, v := range onlds {
-			offldrSpec.Onload = v
-		}
+	for n, v := range onlds {
+		offldrSpec.Onload[n] = *v
 	}
 	if offldrSpec.Name == "" && len(onlds) == 0 {
 		worker.log.Infof("No TLS config found for Ingresses: %v",

pkg/haproxy/dataplane.go

@@ -36,10 +36,13 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
+	"code.uplex.de/uplex-varnish/k8s-ingress/pkg/net"
+
 	"github.com/go-openapi/strfmt"
 	models "github.com/haproxytech/models/v2"
 
@@ -57,22 +60,23 @@ const (
 	versionHdr       = "Configuration-Version"
 	reloadIDHdr      = "Reload-ID"
 	varnishSock      = "unix@/varnish.sock"
-	backendSock      = "unix@/run/offload/onload.sock"
+	backendSockPfx   = "unix@"
 	frontend         = "offloader"
-	onloader         = "onloader"
+	onloaderPfx      = "onldr_"
 	varnish          = "varnish"
 	server           = varnish
-	ingBackends      = "ingressBackends"
+	ingBackendsPfx   = "ingBackends_"
 	frontendSitePath = sitesPath + "/" + frontend
-	backendSitePath  = sitesPath + "/" + onloader
 	crtPath          = "/etc/ssl/private"
 	caBundlePath     = "/run/haproxy/ca-bundle.crt"
+	encodingBase     = 36
 )
 
 var (
-	port       = int64(4443)
-	sslPort    = int64(443)
-	roundRobin = "roundrobin"
+	port              = int64(4443)
+	sslPort           = int64(443)
+	roundRobin        = "roundrobin"
+	nonProxyNameChars = regexp.MustCompile(`[^A-Za-z0-9_.:-]+`)
 )
 
 // ReloadStatus classifies the current state of a dataplane reload.
@@ -132,6 +136,10 @@ type ReloadState struct {
 type DataplaneError struct {
 	// Err encapsulates the dataplane API's error object.
 	Err *models.Error
+	// Method is the HTTP request method that led to the error response.
+	Method string
+	// URI is the request URI that let to the error response.
+	URI string
 	// Status is the HTTP response code.
 	Status int
 	// Version is the configuration-version returned in the response.
@@ -143,7 +151,11 @@ func (err *DataplaneError) Error() string {
 	sb.WriteString(http.StatusText(err.Status))
 	sb.WriteString(" (")
 	sb.WriteString(strconv.Itoa(err.Status))
-	sb.WriteString(") version=")
+	sb.WriteString(") ")
+	sb.WriteString(err.Method)
+	sb.WriteRune(' ')
+	sb.WriteString(err.URI)
+	sb.WriteString(" version=")
 	sb.WriteString(strconv.Itoa(err.Version))
 	if err.Err.Code != nil {
 		sb.WriteString(" models.Error.Code=")
@@ -232,16 +244,42 @@ func getSite() ([]byte, error) {
 	return offldSite.MarshalBinary()
 }
 
-func getOnldSite(spec *OnloadSpec) ([]byte, error) {
+// XXX DRY with replNonFileChars() in package net.
+func encodeChars(s string) string {
+	var sb strings.Builder
+	sb.WriteRune('_')
+	for _, r := range s {
+		sb.WriteString(strconv.FormatUint(uint64(r), encodingBase))
+	}
+	sb.WriteRune('_')
+	return sb.String()
+}
+
+// toProxyName converts a string to a proxy name that is legal for haproxy.cfg.
+// From haproxy configuration manual section 4:
+// "All proxy names must be formed from upper and lower case letters, digits,
+// '-' (dash), '_' (underscore) , '.' (dot) and ':' (colon)."
+func toProxyName(s string) string {
+	return nonProxyNameChars.ReplaceAllStringFunc(s, encodeChars)
+}
+
+func onldSitePath(s string) string {
+	return sitesPath + "/" + onloaderPfx + toProxyName(s)
+}
+
+func getOnldSite(name string, spec *OnloadSpec) ([]byte, error) {
 	// XXX: stick-table configuration
 	maxConn := int64(spec.MaxConn)
+	proxyName := onloaderPfx + toProxyName(name)
+	sockPath := backendSockPfx + net.UDSPath(name)
+	backendName := ingBackendsPfx + toProxyName(name)
 	site := &models.Site{
-		Name: onloader,
+		Name: proxyName,
 		Service: &models.SiteService{
 			Listeners: []*models.Bind{
 				{
-					Name:        onloader,
-					Address:     backendSock,
+					Name:        proxyName,
+					Address:     sockPath,
 					Mode:        "660",
 					AcceptProxy: true,
 				},
@@ -251,7 +289,7 @@ func getOnldSite(spec *OnloadSpec) ([]byte, error) {
 		},
 		Farms: []*models.SiteFarm{
 			{
-				Name:  ingBackends,
+				Name:  backendName,
 				UseAs: "default",
 				Mode:  "tcp",
 				Balance: &models.Balance{
@@ -352,6 +390,8 @@ func getTx(body []byte) (tx *models.Transaction, err error) {
 
 func getError(resp *http.Response, body []byte) error {
 	dplaneErr := &DataplaneError{
+		Method: resp.Request.Method,
+		URI:    resp.Request.URL.RequestURI(),
 		Err:    &models.Error{},
 		Status: resp.StatusCode,
 	}
@@ -447,7 +487,7 @@ func (client *DataplaneClient) FinishTx(
 
 func (client *DataplaneClient) configTLS(
 	tx *models.Transaction,
-	path, method string,
+	path, method, svc string,
 	onldSpec *OnloadSpec,
 ) error {
 	var rdr *bytes.Reader
@@ -458,7 +498,7 @@ func (client *DataplaneClient) configTLS(
 		if onldSpec == nil {
 			siteBytes, err = getSite()
 		} else {
-			siteBytes, err = getOnldSite(onldSpec)
+			siteBytes, err = getOnldSite(svc, onldSpec)
 		}
 		if err != nil {
 			return err
@@ -567,7 +607,7 @@ func (client *DataplaneClient) configDefaults(
 //
 // A non-nil error return may wrap a DataplaneError.
 func (client *DataplaneClient) AddOffldr(tx *models.Transaction) error {
-	return client.configTLS(tx, sitesPath, http.MethodPost, nil)
+	return client.configTLS(tx, sitesPath, http.MethodPost, "", nil)
 }
 
 // UpdateOffldr modifies the offloader configuration for haproxy, in
@@ -578,7 +618,7 @@ func (client *DataplaneClient) AddOffldr(tx *models.Transaction) error {
 //
 // A non-nil error return may wrap a DataplaneError.
 func (client *DataplaneClient) UpdateOffldr(tx *models.Transaction) error {
-	return client.configTLS(tx, frontendSitePath, http.MethodPut, nil)
+	return client.configTLS(tx, frontendSitePath, http.MethodPut, "", nil)
 }
 
 // DeleteOffldr removes the haproxy offloader configuration, in the
@@ -586,12 +626,12 @@ func (client *DataplaneClient) UpdateOffldr(tx *models.Transaction) error {
 //
 // A non-nil error return may wrap a DataplaneError.
 func (client *DataplaneClient) DeleteOffldr(tx *models.Transaction) error {
-	return client.configTLS(tx, frontendSitePath, http.MethodDelete, nil)
+	return client.configTLS(tx, frontendSitePath, http.MethodDelete, "",
+		nil)
 }
 
 // AddOnldr adds the onloader configuration for haproxy, in the
-// dataplane transaction tx. instances specifies the number of servers
-// in the haproxy backend.
+// dataplane transaction tx.
 //
 // AddOnldr MUST be used if the onloader was not configured previously
 // since the haproxy container was started, or after deletion.
@@ -599,14 +639,14 @@ func (client *DataplaneClient) DeleteOffldr(tx *models.Transaction) error {
 // A non-nil error return may wrap a DataplaneError.
 func (client *DataplaneClient) AddOnldr(
 	tx *models.Transaction,
+	svc string,
 	onldSpec *OnloadSpec,
 ) error {
-	return client.configTLS(tx, sitesPath, http.MethodPost, onldSpec)
+	return client.configTLS(tx, sitesPath, http.MethodPost, svc, onldSpec)
 }
 
 // UpdateOnldr modifies the onloader configuration for haproxy, in the
-// dataplane transaction tx. instances specifies the number of servers
-// in the haproxy backend.
+// dataplane transaction tx.
 //
 // UpdateOnldr MUST be used if the onloader was previously added with
 // AddOnldr, and not removed with DeleteOnldr.
@@ -614,17 +654,23 @@ func (client *DataplaneClient) AddOnldr(
 // A non-nil error return may wrap a DataplaneError.
 func (client *DataplaneClient) UpdateOnldr(
 	tx *models.Transaction,
+	svc string,
 	onldSpec *OnloadSpec,
 ) error {
-	return client.configTLS(tx, backendSitePath, http.MethodPut, onldSpec)
+	return client.configTLS(tx, onldSitePath(svc), http.MethodPut, svc,
+		onldSpec)
 }
 
 // DeleteOnldr removes the haproxy onloader configuration, in the
 // dataplane transaction tx.
 //
 // A non-nil error return may wrap a DataplaneError.
-func (client *DataplaneClient) DeleteOnldr(tx *models.Transaction) error {
-	return client.configTLS(tx, backendSitePath, http.MethodDelete, nil)
+func (client *DataplaneClient) DeleteOnldr(
+	tx *models.Transaction,
+	svc string,
+) error {
+	return client.configTLS(tx, onldSitePath(svc), http.MethodDelete, "",
+		nil)
 }
 
 // UpdateDefaults modifies default haproxy configuration (valid for both
@@ -748,7 +794,7 @@ func (client *DataplaneClient) LoaderStatus() (
 	for _, site := range []*models.Site(sb.Sites) {
 		if site.Name == frontend {
 			offLoaded = true
-		} else if site.Name == onloader {
+		} else if strings.HasPrefix(site.Name, onloaderPfx) {
 			onLoaded = true
 		}
 		if offLoaded && onLoaded {

pkg/haproxy/haproxy.go

@@ -87,15 +87,18 @@ type DefaultsSpec struct {
 	Timeouts DefaultTimeoutsSpec
 }
 
-// Spec specifies the configuration of TLS offload for haproxy. It
-// includes the namespace and name of the Varnish admin Service (the
-// headless k8s Service specifying ports for remote administration),
-// and a list of specs for Ingress TLS Secrets.
+// Spec specifies the configuration of TLS off- and onload for haproxy,
+// including:
+//   - the namespace and name of the Varnish admin Service (the
+//     headless k8s Service specifying ports for remote administration),
+//   - a list of specs for Ingress TLS Secrets,
+//   - a (potentially empty) set of specifications for onload,
+//   - optional specifications for global timeouts.
 type Spec struct {
 	Namespace string
 	Name      string
 	Secrets   []SecretSpec
-	Onload    *OnloadSpec
+	Onload    map[string]OnloadSpec
 	Defaults  *DefaultsSpec
 }
 
@@ -348,6 +351,13 @@ func (hc *Controller) updateLoadStatus(inst *haproxyInst) error {
 	return nil
 }
 
+type onldSvcMap map[string]OnloadSpec
+
+func (onldMap onldSvcMap) hasOnldSvc(svc string) bool {
+	_, exists := onldMap[svc]
+	return exists
+}
+
 func (hc *Controller) updateInstance(inst *haproxyInst, spec *Spec) error {
 	var err error
 
@@ -367,7 +377,7 @@ func (hc *Controller) updateInstance(inst *haproxyInst, spec *Spec) error {
 	hc.wg.Add(1)
 	defer hc.wg.Done()
 
-	if len(spec.Secrets) == 0 && spec.Onload == nil {
+	if len(spec.Secrets) == 0 && len(spec.Onload) == 0 {
 		return update.MakeIncomplete(
 			"haproxy instance %s: no offload certificates or "+
 				"onload config specified", inst.name)
@@ -413,23 +423,36 @@ func (hc *Controller) updateInstance(inst *haproxyInst, spec *Spec) error {
 			return err
 		}
 	}
-	if spec.Onload != nil && !inst.status.onLoaded {
-		if tx.Version <= 1 || inst.spec == nil ||
-			inst.spec.Onload == nil {
+	for svcName, onld := range spec.Onload {
+		if !inst.status.onLoaded {
 			hc.log.Debugf("Onloader instance %s: "+
-				"adding TLS config %+v", inst.name,
-				*spec.Onload)
-			err = inst.dplane.AddOnldr(tx, spec.Onload)
+				"adding TLS config %+v for svc %s",
+				inst.name, onld, svcName)
+			err = inst.dplane.AddOnldr(tx, svcName, &onld)
 		} else {
 			hc.log.Debugf("Onloader instance %s: "+
-				"updating TLS config %+v", inst.name,
-				*spec.Onload)
-			err = inst.dplane.UpdateOnldr(tx, spec.Onload)
+				"updating TLS config %+v for svc %s", inst.name,
+				onld, svcName)
+			err = inst.dplane.UpdateOnldr(tx, svcName, &onld)
 		}
 		if err != nil {
 			return err
 		}
 	}
+	if inst.spec != nil {
+		for svcName, onld := range inst.spec.Onload {
+			if _, exists := spec.Onload[svcName]; exists {
+				continue
+			}
+			hc.log.Debugf("Onloader instance %s: "+
+				"deleting TLS config %+v for svc %s", inst.name,
+				onld, svcName)
+			if err = inst.dplane.DeleteOnldr(tx,
+				svcName); err != nil {
+				return err
+			}
+		}
+	}
 	if spec.Defaults == nil {
 		hc.log.Debugf("Instance %s: global defaults not set", inst.name)
 	} else {
@@ -466,25 +489,16 @@ func (hc *Controller) updateInstance(inst *haproxyInst, spec *Spec) error {
 		Namespace: spec.Namespace,
 		Name:      spec.Name,
 		Secrets:   make([]SecretSpec, len(spec.Secrets)),
-		Onload:    nil,
+		Onload:    make(map[string]OnloadSpec),
 		Defaults:  spec.Defaults,
 	}
 	for i, s := range spec.Secrets {
 		inst.spec.Secrets[i] = s
 	}
-	if spec.Onload != nil {
-		inst.spec.Onload = &OnloadSpec{
-			Verify:     spec.Onload.Verify,
-			Authority:  spec.Onload.Authority,
-			Instances:  spec.Onload.Instances,
-			StickTblSz: spec.Onload.StickTblSz,
-			MaxConn:    spec.Onload.MaxConn,
-		}
+	for svcName, onld := range spec.Onload {
+		inst.spec.Onload[svcName] = onld
 	}
 
-	// XXX where does this go?
-	// defer hc.dataplane.DeleteTx(tx)
-
 	switch state.Status {
 	case Succeeded:
 		inst.status.dplaneState = state
@@ -494,7 +508,7 @@ func (hc *Controller) updateInstance(inst *haproxyInst, spec *Spec) error {
 		if len(spec.Secrets) > 0 {
 			inst.status.offLoaded = true
 		}
-		if spec.Onload != nil {
+		if len(spec.Onload) > 0 {
 			inst.status.onLoaded = true
 		}
 		return nil
@@ -518,7 +532,7 @@ func (hc *Controller) updateInstance(inst *haproxyInst, spec *Spec) error {
 			if len(spec.Secrets) > 0 {
 				inst.status.offLoaded = true
 			}
-			if spec.Onload != nil {
+			if len(spec.Onload) > 0 {
 				inst.status.onLoaded = true
 			}
 			return nil
@@ -590,7 +604,13 @@ func (hc *Controller) removeOffldrInstances(
 				"the instance is already deleted", inst.name)
 			continue
 		}
-		version := inst.status.version + 1
+		hc.log.Debugf("haproxy svc %s: get current status", inst.name)
+		if err := hc.getOffldStatus(inst); err != nil {
+			offldrErr := inst.mkError(err)
+			errs = append(errs, offldrErr)
+			continue
+		}
+		version := inst.status.version
 		hc.log.Debugf("haproxy instance %s: starting tx for version "+
 			"%d", inst.name, version)
 		tx, err := inst.dplane.StartTx(version)
@@ -612,6 +632,16 @@ func (hc *Controller) removeOffldrInstances(
 			continue
 		}
 
+		for onldSvc := range inst.spec.Onload {
+			hc.log.Debugf("haproxy instance %s: deleting onload "+
+				"config for %s", inst.name, onldSvc)
+			err = inst.dplane.DeleteOnldr(tx, onldSvc)
+			if err != nil {
+				errs = append(errs, inst.mkError(err))
+				continue
+			}
+		}
+
 		hc.log.Debugf("haproxy instance %s: finishing tx for "+
 			"version %d: %+v", inst.name, tx.Version, tx)
 		state, err := inst.dplane.FinishTx(tx)
@@ -972,6 +1002,30 @@ func (hc *Controller) DeleteTLSSecret(
 			"for TLS Secret %s", svcKey, secret)
 }
 
+func (hc *Controller) DeleteOnldr(svcKey, onldSvcKey string) error {
+	svc, exists := hc.svcs[svcKey]
+	if !exists {
+		hc.log.Infof("haproxy service %s: not found, ignoring "+
+			"onloader deletion", svcKey)
+		return nil
+	}
+	if svc.spec == nil {
+		hc.log.Infof("haproxy service %s: no current spec, ignoring "+
+			"onloader deletion", svcKey)
+		return nil
+	}
+	onldMap := svc.spec.Onload
+	if _, exists := onldMap[onldSvcKey]; !exists {
+		hc.log.Infof("haproxy service %s: onload service %s not found,"+
+			" ignoring onloader deletion", svcKey, onldSvcKey)
+		return nil
+	}
+	delete(onldMap, onldSvcKey)
+	hc.log.Infof("haproxy service %s: updating for onload service "+
+		"%s deletion", svcKey, onldSvcKey)
+	return hc.updateOffldSvc(svcKey)
+}
+
 // SetDataplaneSecret stores the secret to be used as the Basic Auth
 // password used in requests to a dataplane API, under the name given
 // in key (from the namespace/name of a k8s Secret).

pkg/net/net.go

@@ -31,11 +31,29 @@
 package net
 
 import (
+	"crypto/sha256"
+	"encoding/binary"
 	"net"
+	"regexp"
+	"strconv"
+	"strings"
+	"syscall"
 
 	"github.com/sirupsen/logrus"
 )
 
+const (
+	udsPfx  = "/run/offload/onld_"
+	udsSfx  = ".sock"
+	encBase = 36
+)
+
+var (
+	udsAddr      = syscall.RawSockaddrUnix{}
+	udsPathLen   = len(udsAddr.Path)
+	nonFileChars = regexp.MustCompile(`[^A-Za-z0-9_.]+`)
+)
+
 // IsNonTimeoutNetErr returns true for network errors that are not
 // timeouts. On syncs for deletion, the Pod may be already gone.
 func IsNonTimeoutNetErr(err error, log *logrus.Logger) bool {
@@ -47,3 +65,37 @@ func IsNonTimeoutNetErr(err error, log *logrus.Logger) bool {
 	log.Warnf("Non-timeout network error: %+v", err)
 	return true
 }
+
+func replNonFileChars(s string) string {
+	var sb strings.Builder
+	sb.WriteRune('_')
+	for _, r := range s {
+		sb.WriteString(strconv.FormatUint(uint64(r), encBase))
+	}
+	sb.WriteRune('_')
+	return sb.String()
+}
+
+func toFileChars(s string) string {
+	return nonFileChars.ReplaceAllStringFunc(s, replNonFileChars)
+}
+
+// UDSPath returns the path accessed by both Varnish and haproxy for
+// the Unix domain socket used for TLS onload to the service
+// designated by name.
+func UDSPath(name string) string {
+	var sb strings.Builder
+	sb.WriteString(udsPfx)
+	fileStr := toFileChars(name)
+	if len(udsPfx)+len(fileStr)+len(udsSfx) < udsPathLen {
+		sb.WriteString(fileStr)
+	} else {
+		hash := sha256.Sum256([]byte(name))
+		for i := 0; i < sha256.Size; i += 8 {
+			n := binary.NativeEndian.Uint64((hash[i : i+8]))
+			sb.WriteString(strconv.FormatUint(n, encBase))
+		}
+	}
+	sb.WriteString(udsSfx)
+	return sb.String()
+}

pkg/net/net_test.go

+/*
+ * Copyright (c) 2024 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+package net
+
+import "testing"
+
+func chkUDSPath(t *testing.T, name string) {
+	p := UDSPath(name)
+	if len(p) >= udsPathLen {
+		t.Errorf("UDSPath %s for %s too long: want len < %d, got %d",
+			p, name, udsPathLen, len(p))
+	}
+}
+
+func TestUDSPath(t *testing.T) {
+	chkUDSPath(t, "default/svc")
+	chkUDSPath(t, "ns!$%&()/name;,#+)(%!")
+	chkUDSPath(t, "veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongNS/"+
+		"veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongName")
+}

pkg/varnish/vcl/testdata/via_authority_backend.golden

@@ -7,7 +7,9 @@ import dynamic;
 import selector;
 
 include "bogo_backend.vcl";
-backend vk8s_via { .path = "/run/offload/onload.sock"; }
+backend vk8s_via_tea-svc {
+	.path = "/run/offload/onld_tea_19_svc.sock";
+}
 
 backend vk8s_default_coffee-6b9f5c47d7-bdt68_80 {
 	.host = "192.0.2.4";
@@ -20,19 +22,19 @@ backend vk8s_default_coffee-6b9f5c47d7-l5zvl_80 {
 backend vk8s_default_tea-5798f99dc5-5wj8n_80 {
 	.host = "192.0.2.1";
 	.port = "80";
-	.via = vk8s_via;
+	.via = vk8s_via_tea-svc;
 	.authority = "www.tea.org";
 }
 backend vk8s_default_tea-5798f99dc5-hn27l_80 {
 	.host = "192.0.2.2";
 	.port = "80";
-	.via = vk8s_via;
+	.via = vk8s_via_tea-svc;
 	.authority = "www.tea.org";
 }
 backend vk8s_default_tea-5798f99dc5-5wj8n_80 {
 	.host = "192.0.2.3";
 	.port = "80";
-	.via = vk8s_via;
+	.via = vk8s_via_tea-svc;
 	.authority = "www.tea.org";
 }
 

pkg/varnish/vcl/testdata/via_backend.golden

@@ -7,7 +7,9 @@ import dynamic;
 import selector;
 
 include "bogo_backend.vcl";
-backend vk8s_via { .path = "/run/offload/onload.sock"; }
+backend vk8s_via_tea-svc {
+	.path = "/run/offload/onld_tea_19_svc.sock";
+}
 
 backend vk8s_default_coffee-6b9f5c47d7-bdt68_80 {
 	.host = "192.0.2.4";
@@ -20,17 +22,17 @@ backend vk8s_default_coffee-6b9f5c47d7-l5zvl_80 {
 backend vk8s_default_tea-5798f99dc5-5wj8n_80 {
 	.host = "192.0.2.1";
 	.port = "80";
-	.via = vk8s_via;
+	.via = vk8s_via_tea-svc;
 }
 backend vk8s_default_tea-5798f99dc5-hn27l_80 {
 	.host = "192.0.2.2";
 	.port = "80";
-	.via = vk8s_via;
+	.via = vk8s_via_tea-svc;
 }
 backend vk8s_default_tea-5798f99dc5-5wj8n_80 {
 	.host = "192.0.2.3";
 	.port = "80";
-	.via = vk8s_via;
+	.via = vk8s_via_tea-svc;
 }
 
 

pkg/varnish/vcl/vcl_tmpl.go

@@ -32,6 +32,8 @@ import (
 	"sort"
 	"strings"
 	"text/template"
+
+	"code.uplex.de/uplex-varnish/k8s-ingress/pkg/net"
 )
 
 const ingTmplSrc = `vcl 4.1;
@@ -83,9 +85,18 @@ probe {{probeName $name}} {
 {{- end}}
 {{- template "ProbeDef" .IntSvcs -}}
 {{- template "ProbeDef" .ExtSvcs}}
-{{- if needsVia .IntSvcs .ExtSvcs }}
-backend vk8s_via { .path = "/run/offload/onload.sock"; }
-{{- end }}
+
+{{- define "ViaDef"}}
+{{- range $name, $svc := .}}
+{{- if $svc.Via}}
+backend {{viaBackendName $name}} {
+	.path = "{{onldUDSPath $name}}";
+}
+{{- end}}
+{{- end}}
+{{- end}}
+{{- template "ViaDef" .IntSvcs -}}
+{{- template "ViaDef" .ExtSvcs}}
 
 {{range $name, $svc := .IntSvcs -}}
 {{range $addr := $svc.Addresses -}}
@@ -115,7 +126,7 @@ backend {{backendName $svc $addr}} {
 	.probe = {{probeName $name}};
 {{- end}}
 {{- if .Via}}
-	.via = vk8s_via;
+	.via = {{viaBackendName $name}};
 {{- end}}
 {{- if .Authority}}
 	.authority = "{{.Authority}}";
@@ -203,7 +214,7 @@ sub vcl_init {
 		, probe = {{probeName $name}}
 		{{- end}}
 		{{- if .Via}}
-		, via = vk8s_via
+		, via = {{viaBackendName $name}}
 		{{- end}}
 		{{- end}}
 	);
@@ -483,6 +494,12 @@ var vclFuncs = template.FuncMap{
 	"resolverName": func(svc Service) string {
 		return mangle(svc.Name + "_resolver")
 	},
+	"viaBackendName": func(name string) string {
+		return mangle("via_" + name)
+	},
+	"onldUDSPath": func(name string) string {
+		return net.UDSPath(name)
+	},
 }
 
 var ingressTmpl = template.Must(template.New(ingTmplName).