Support multiple TLS Secrets/certs per Ingress.

We take advantage of the fact that if a directory is specified for
haproxy as the location for certificates, then haproxy reads all
of the certificates found there, and automatically associates the
SNI of incoming connections with the appropriate certificate.

container/haproxy.cfg

@@ -7,9 +7,7 @@ global
 	group varnish
         master-worker
 
-	# Default SSL material locations
 	ca-base /etc/ssl/certs
-	crt-base /etc/ssl/private
 
 	# See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
         ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384

pkg/controller/ingress.go

@@ -855,9 +855,14 @@ func (worker *NamespaceWorker) configReqDisps(spec *vcl.Spec,
 }
 
 func (worker *NamespaceWorker) ings2OffloaderSpec(
-	ings []*extensions.Ingress) (haproxy.Spec, error) {
-
-	offldrSpec := haproxy.Spec{}
+	svc *api_v1.Service,
+	ings []*extensions.Ingress,
+) (haproxy.Spec, error) {
+	offldrSpec := haproxy.Spec{
+		Namespace: svc.Namespace,
+		Name:      svc.Name,
+		Secrets:   make([]haproxy.SecretSpec, 0),
+	}
 	for _, ing := range ings {
 		for _, tls := range ing.Spec.TLS {
 			if len(tls.Hosts) > 0 {
@@ -872,21 +877,11 @@ func (worker *NamespaceWorker) ings2OffloaderSpec(
 					ing.ObjectMeta.Namespace,
 					ing.ObjectMeta.Name)
 			}
-			if offldrSpec.Name != "" {
-				if offldrSpec.Namespace == ing.ObjectMeta.Namespace &&
-					offldrSpec.Name == tls.SecretName {
-					continue
-				}
-				return offldrSpec, fmt.Errorf("Multiple TLS "+
-					"Secrets not supported: Secret %s in "+
-					"Ingress %s/%s, Secret %s/%s specified"+
-					" in another Ingress", tls.SecretName,
-					ing.ObjectMeta.Namespace,
-					ing.ObjectMeta.Name,
-					offldrSpec.Namespace, offldrSpec.Name)
+			secr := haproxy.SecretSpec{
+				Namespace: ing.ObjectMeta.Namespace,
+				Name:      tls.SecretName,
 			}
-			offldrSpec.Name = tls.SecretName
-			offldrSpec.Namespace = ing.ObjectMeta.Namespace
+			offldrSpec.Secrets = append(offldrSpec.Secrets, secr)
 		}
 	}
 	return offldrSpec, nil
@@ -983,7 +978,7 @@ func (worker *NamespaceWorker) addOrUpdateIng(ing *extensions.Ingress) error {
 	// readiness check to the Varnish readiness check, so if we're
 	// adding a new service, the offloader becomes ready when
 	// Varnish is ready.
-	offldrSpec, err := worker.ings2OffloaderSpec(ings)
+	offldrSpec, err := worker.ings2OffloaderSpec(svc, ings)
 	if err != nil {
 		return err
 	}
@@ -991,13 +986,15 @@ func (worker *NamespaceWorker) addOrUpdateIng(ing *extensions.Ingress) error {
 		worker.log.Infof("No TLS config found for Ingresses: %v",
 			ingNames)
 	} else {
-		// XXX first check if the PEM Secret is already set
-		if err = worker.updateCertSecret(&offldrSpec); err != nil {
-			return err
+		for _, tlsSecr := range offldrSpec.Secrets {
+			// XXX first check if the PEM Secret is already set
+			if err = worker.updateCertSecret(&tlsSecr); err != nil {
+				return err
+			}
+			worker.log.Infof("Ingress TLS Secret %s/%s: added "+
+				"certificate %s", tlsSecr.Namespace,
+				tlsSecr.Name, tlsSecr.CertName())
 		}
-		worker.log.Infof("Ingress TLS Secret %s/%s: added certificate "+
-			"%s", offldrSpec.Namespace, offldrSpec.Name,
-			offldrSpec.CertName())
 		if secrKey, dSecr, err := worker.getDplaneSecret(); err != nil {
 			return err
 		} else if dSecr == nil {
@@ -1012,10 +1009,12 @@ func (worker *NamespaceWorker) addOrUpdateIng(ing *extensions.Ingress) error {
 				return err
 			}
 		}
-		// XXX check if already loaded
-		if err = worker.hController.Update(svcKey, offldrSpec); err != nil {
-			return err
-		}
+	}
+	if len(offldrSpec.Secrets) == 0 {
+		worker.log.Infof("Service %s: no TLS certificates specified",
+			svcKey)
+	} else if err = worker.hController.Update(svcKey, offldrSpec); err != nil {
+		return err
 	}
 
 	ingsMeta := make(map[string]varnish.Meta)

pkg/controller/secret.go

@@ -50,7 +50,7 @@ const (
 
 // XXX client...Update(secret) returns Secret
 // XXX client...Create(secret) if it's new?
-func (worker *NamespaceWorker) updateCertSecret(spec *haproxy.Spec) error {
+func (worker *NamespaceWorker) updateCertSecret(spec *haproxy.SecretSpec) error {
 	nsLister := worker.listers.secr.Secrets(spec.Namespace)
 	tlsSecret, err := nsLister.Get(spec.Name)
 	if err != nil {
@@ -111,7 +111,7 @@ func (worker *NamespaceWorker) deleteTLSSecret(secret *api_v1.Secret) error {
 		return err
 	}
 
-	spec := haproxy.Spec{
+	spec := haproxy.SecretSpec{
 		Namespace: secret.ObjectMeta.Namespace,
 		Name:      secret.ObjectMeta.Name,
 	}

pkg/haproxy/dataplane.go

@@ -57,6 +57,7 @@ const (
 	backend          = "varnish"
 	server           = backend
 	frontendSitePath = sitesPath + "/" + frontend
+	crtPath          = "/etc/ssl/private"
 )
 
 var port = int64(443)
@@ -156,35 +157,36 @@ func (client *DataplaneClient) getHost() string {
 	return client.baseURL.Host
 }
 
-func getSite(certName string) ([]byte, error) {
-	site := &models.Site{
-		Name: frontend,
-		Service: &models.SiteService{
-			Listeners: []*models.Bind{
-				&models.Bind{
-					Name:           frontend,
-					Port:           &port,
-					Ssl:            true,
-					SslCertificate: certName,
-				},
+var offldSite = &models.Site{
+	Name: frontend,
+	Service: &models.SiteService{
+		Listeners: []*models.Bind{
+			&models.Bind{
+				Name:           frontend,
+				Port:           &port,
+				Ssl:            true,
+				SslCertificate: crtPath,
 			},
 		},
-		Farms: []*models.SiteFarm{
-			&models.SiteFarm{
-				Name:  backend,
-				UseAs: "default",
-				Servers: []*models.Server{
-					&models.Server{
-						Name:        server,
-						Address:     varnishSock,
-						Check:       "enabled",
-						SendProxyV2: "enabled",
-					},
+	},
+	Farms: []*models.SiteFarm{
+		&models.SiteFarm{
+			Name:  backend,
+			UseAs: "default",
+			Servers: []*models.Server{
+				&models.Server{
+					Name:        server,
+					Address:     varnishSock,
+					Check:       "enabled",
+					SendProxyV2: "enabled",
 				},
 			},
 		},
-	}
-	return site.MarshalBinary()
+	},
+}
+
+func getSite() ([]byte, error) {
+	return offldSite.MarshalBinary()
 }
 
 func drainAndClose(body io.ReadCloser) {
@@ -413,7 +415,7 @@ func (client *DataplaneClient) configTLS(
 	// }
 	var rdr *bytes.Reader
 	if method != http.MethodDelete {
-		site, err := getSite(spec.CertName())
+		site, err := getSite()
 		if err != nil {
 			return err
 		}

pkg/haproxy/faccess.go

@@ -61,7 +61,7 @@ func (client *FaccessClient) getHost() string {
 	return client.baseURL.Host
 }
 
-func (client *FaccessClient) PEMExists(spec Spec) (bool, error) {
+func (client *FaccessClient) PEMExists(spec SecretSpec) (bool, error) {
 	relPath := &url.URL{Path: spec.CertName()}
 	url := client.baseURL.ResolveReference(relPath)
 	req, err := http.NewRequest(http.MethodHead, url.String(), nil)

pkg/haproxy/haproxy.go

@@ -49,18 +49,28 @@ import (
 
 const linux_name_max = 255
 
-type Spec struct {
+type SecretSpec struct {
 	Namespace       string
 	Name            string
 	UID             string
 	ResourceVersion string
 }
 
+func (spec SecretSpec) String() string {
+	return spec.Namespace + "/" + spec.Name
+}
+
+type Spec struct {
+	Namespace string
+	Name      string
+	Secrets   []SecretSpec
+}
+
 func (spec Spec) String() string {
 	return spec.Namespace + "/" + spec.Name
 }
 
-func (spec Spec) hashedName() string {
+func (spec SecretSpec) hashedName() string {
 	hash := sha512.New512_256()
 	hash.Write([]byte(spec.Namespace))
 	hash.Write([]byte(spec.Name))
@@ -76,7 +86,7 @@ func (spec Spec) hashedName() string {
 // Guaranteed to be shorter than NAME_MAX on recent common Linux
 // distributions; so it may be an opaque hash string, if the Namespace
 // and/or Name are too long.
-func (spec Spec) CertName() string {
+func (spec SecretSpec) CertName() string {
 	name := spec.Namespace + "_" + spec.Name
 	if len(name) > linux_name_max {
 		name = spec.hashedName()
@@ -120,7 +130,7 @@ func (offldrErrs OffldrErrors) Error() string {
 type configStatus struct {
 	dplaneState ReloadState
 	version     int64
-	pemExists   bool
+	pemExists   map[SecretSpec]bool
 	loaded      bool
 }
 
@@ -181,23 +191,29 @@ func (hc *Controller) updateLoadStatus(inst *haproxyInst) error {
 	hc.wg.Add(1)
 	defer hc.wg.Done()
 
-	if !inst.status.pemExists {
+	for _, s := range inst.spec.Secrets {
+		pemExists, ok := inst.status.pemExists[s]
+		if ok && pemExists {
+			continue
+		}
+		if !ok {
+			inst.status.pemExists[s] = false
+		}
 		hc.log.Debugf("Offloader instance %s, checking for PEM file %s",
-			inst.name, inst.spec.CertName())
-		pemExists, err := inst.faccess.PEMExists(*inst.spec)
+			inst.name, s.CertName())
+		pemExists, err := inst.faccess.PEMExists(s)
 		if err != nil {
 			// XXX decode?
 			return err
 		}
-		inst.status.pemExists = pemExists
+		inst.status.pemExists[s] = pemExists
 		if !pemExists {
 			// XXX SyncIncomplete
 			return fmt.Errorf("Offloader instance %s: certificate "+
-				"PEM %s not found", inst.name,
-				inst.spec.CertName())
+				"PEM %s not found", inst.name, s.CertName())
 		}
 		hc.log.Infof("Offloader instance %s: PEM file %s exists",
-			inst.name, inst.spec.CertName())
+			inst.name, s.CertName())
 		// XXX initiate load
 	}
 
@@ -245,21 +261,28 @@ func (hc *Controller) updateInstance(inst *haproxyInst, spec *Spec) error {
 	hc.wg.Add(1)
 	defer hc.wg.Done()
 
-	hc.log.Debugf("Offloader instance %s, checking for PEM file %s",
-		inst.name, spec.CertName())
-	pemExists, err := inst.faccess.PEMExists(*spec)
-	if err != nil {
-		// XXX decode?
-		return err
+	if len(spec.Secrets) == 0 {
+		hc.log.Warnf("Offloader instance %s: no certificates specified",
+			inst.name)
+		return nil
 	}
-	inst.status.pemExists = pemExists
-	if !pemExists {
-		// XXX SyncIncomplete
-		return fmt.Errorf("Offloader instance %s: certificate PEM %s "+
-			"not found", inst.name, spec.CertName())
+	for _, s := range spec.Secrets {
+		hc.log.Debugf("Offloader instance %s, checking for PEM file %s",
+			inst.name, s.CertName())
+		pemExists, err := inst.faccess.PEMExists(s)
+		if err != nil {
+			// XXX decode?
+			return err
+		}
+		inst.status.pemExists[s] = pemExists
+		if !pemExists {
+			// XXX SyncIncomplete
+			return fmt.Errorf("Offloader instance %s: certificate "+
+				"PEM %s not found", inst.name, s.CertName())
+		}
+		hc.log.Infof("Offloader instance %s: PEM file %s exists",
+			inst.name, s.CertName())
 	}
-	hc.log.Infof("Offloader instance %s: PEM file %s exists", inst.name,
-		spec.CertName())
 
 	version := inst.status.version
 	if version == 0 {
@@ -301,10 +324,12 @@ func (hc *Controller) updateInstance(inst *haproxyInst, spec *Spec) error {
 	inst.status.version = tx.Version
 	inst.status.dplaneState = state
 	inst.spec = &Spec{
-		Namespace:       spec.Namespace,
-		Name:            spec.Name,
-		UID:             spec.UID,
-		ResourceVersion: spec.ResourceVersion,
+		Namespace: spec.Namespace,
+		Name:      spec.Name,
+		Secrets:   make([]SecretSpec, len(spec.Secrets)),
+	}
+	for i, s := range spec.Secrets {
+		inst.spec.Secrets[i] = s
 	}
 
 	// XXX where does this go?
@@ -502,13 +527,15 @@ func offldAddr2haproxyInst(addr OffldAddr, dplanePasswd *string) *haproxyInst {
 	dplaneClient := NewDataplaneClient(dplaneAddr, passwd)
 	faccessAddr := addr.IP + ":" + strconv.Itoa(int(addr.FaccessPort))
 	faccessClient := NewFaccessClient(faccessAddr)
-	return &haproxyInst{
+	inst := &haproxyInst{
 		dplane:       dplaneClient,
 		faccess:      faccessClient,
 		name:         addr.PodNamespace + "/" + addr.PodName,
 		dplanePasswd: dplanePasswd,
 		admMtx:       &sync.Mutex{},
 	}
+	inst.status.pemExists = make(map[SecretSpec]bool)
+	return inst
 }
 
 // Used as a map key in updateOffldrAddrs().
@@ -749,8 +776,9 @@ func (hc *Controller) SetDataplaneSecret(key string, secret []byte) {
 func (hc *Controller) SetOffldSecret(svcKey, secretKey string) error {
 	svc, ok := hc.svcs[svcKey]
 	if !ok {
-		return fmt.Errorf("Cannot set secret %s for offloader %s: "+
+		hc.log.Warnf("Cannot set secret %s for offloader %s: "+
 			"offloader not found", secretKey, svcKey)
+		return nil
 	}
 	svc.secrName = secretKey
 	if secret, ok := hc.secrets[secretKey]; ok {