Move to a passthrough route to support mTLS

pkg/operator2/deployment.go

@@ -28,6 +28,7 @@ func (c *authOperator) getGeneration() int64 {
 func defaultDeployment(
 	operatorConfig *operatorv1.Authentication,
 	syncData *configSyncData,
+	routerSecret *corev1.Secret,
 	resourceVersions ...string,
 ) *appsv1.Deployment {
 	replicas := int32(1) // TODO configurable?
@@ -63,6 +64,12 @@ func defaultDeployment(
 			path:      serviceCAMount,
 			keys:      []string{serviceCAKey},
 		},
+		{
+			name:      routerCertsLocalName,
+			configmap: false,
+			path:      routerCertsLocalMount,
+			keys:      sets.StringKeySet(routerSecret.Data).List(),
+		},
 	} {
 		v, m := data.split()
 		volumes = append(volumes, v)

pkg/operator2/oauth.go

@@ -20,6 +20,7 @@ import (
 func (c *authOperator) handleOAuthConfig(
 	operatorConfig *operatorv1.Authentication,
 	route *routev1.Route,
+	routerSecret *corev1.Secret,
 	service *corev1.Service,
 	consoleConfig *configv1.Console,
 	infrastructureConfig *configv1.Infrastructure,
@@ -86,13 +87,14 @@ func (c *authOperator) handleOAuthConfig(
 				ServingInfo: configv1.ServingInfo{
 					BindAddress: fmt.Sprintf("0.0.0.0:%d", containerPort),
 					BindNetwork: "tcp4",
-					// we have valid serving certs provided by service-ca so that we can use reencrypt routes
+					// we have valid serving certs provided by service-ca
+					// this is our main server cert which is used if SNI does not match
 					CertInfo: configv1.CertInfo{
 						CertFile: servingCertPathCert,
 						KeyFile:  servingCertPathKey,
 					},
 					ClientCA:          "", // I think this can be left unset
-					NamedCertificates: nil,
+					NamedCertificates: routerSecretToSNI(routerSecret),
 					MinTLSVersion:     crypto.TLSVersionToNameOrDie(crypto.DefaultTLSVersion()),
 					CipherSuites:      crypto.CipherSuitesToNamesOrDie(crypto.DefaultCiphers()),
 				},

pkg/operator2/operator.go

@@ -69,6 +69,10 @@ const (
 
 	oauthMetadataName = systemConfigPrefix + "metadata"
 
+	routerCertsSharedName = "router-certs"
+	routerCertsLocalName  = systemConfigPrefix + routerCertsSharedName
+	routerCertsLocalMount = systemConfigPathSecrets + "/" + routerCertsLocalName
+
 	userConfigPath = "/var/config/user"
 
 	servicePort   = 443
@@ -185,11 +189,11 @@ func (c *authOperator) handleSync(operatorConfig *operatorv1.Authentication) err
 	// BLOCK 1: Metadata
 	// ==================================
 
-	route, err := c.handleRoute()
+	route, routerSecret, err := c.handleRoute()
 	if err != nil {
 		return err
 	}
-	resourceVersions = append(resourceVersions, route.GetResourceVersion())
+	resourceVersions = append(resourceVersions, route.GetResourceVersion(), routerSecret.GetResourceVersion())
 
 	// make sure API server sees our metadata as soon as we've got a route with a host
 	metadata, _, err := resourceapply.ApplyConfigMap(c.configMaps, c.recorder, getMetadataConfigMap(route))
@@ -241,7 +245,7 @@ func (c *authOperator) handleSync(operatorConfig *operatorv1.Authentication) err
 	infrastructureConfig := c.handleInfrastructureConfig()
 	resourceVersions = append(resourceVersions, infrastructureConfig.GetResourceVersion())
 
-	oauthConfig, expectedCLIconfig, syncData, err := c.handleOAuthConfig(operatorConfig, route, service, consoleConfig, infrastructureConfig)
+	oauthConfig, expectedCLIconfig, syncData, err := c.handleOAuthConfig(operatorConfig, route, routerSecret, service, consoleConfig, infrastructureConfig)
 	if err != nil {
 		return err
 	}
@@ -274,6 +278,7 @@ func (c *authOperator) handleSync(operatorConfig *operatorv1.Authentication) err
 	expectedDeployment := defaultDeployment(
 		operatorConfig,
 		syncData,
+		routerSecret,
 		resourceVersions...,
 	)
 	// TODO add support for spec.operandSpecs.unsupportedResourcePatches, like:

pkg/operator2/route.go

@@ -5,24 +5,26 @@ import (
 
 	"github.com/golang/glog"
 
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
+	configv1 "github.com/openshift/api/config/v1"
 	routev1 "github.com/openshift/api/route/v1"
 )
 
-func (c *authOperator) handleRoute() (*routev1.Route, error) {
+func (c *authOperator) handleRoute() (*routev1.Route, *corev1.Secret, error) {
 	route, err := c.route.Get(targetName, metav1.GetOptions{})
 	if errors.IsNotFound(err) {
 		route, err = c.route.Create(defaultRoute())
 	}
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if len(route.Spec.Host) == 0 {
-		return nil, fmt.Errorf("route has no host: %#v", route)
+		return nil, nil, fmt.Errorf("route has no host: %#v", route)
 	}
 
 	if err := isValidRoute(route); err != nil {
@@ -32,14 +34,23 @@ func (c *authOperator) handleRoute() (*routev1.Route, error) {
 		if err := c.route.Delete(route.Name, opts); err != nil && !errors.IsNotFound(err) {
 			glog.Infof("failed to delete invalid route: %v", err)
 		}
-		return nil, err
+		return nil, nil, err
 	}
 
-	return route, nil
+	routerSecret, err := c.secrets.Secrets(targetName).Get(routerCertsLocalName, metav1.GetOptions{})
+	if err != nil {
+		return nil, nil, err
+	}
+	if len(routerSecret.Data) == 0 {
+		return nil, nil, fmt.Errorf("router secret is empty: %#v", routerSecret)
+	}
+
+	return route, routerSecret, nil
 }
 
 func isValidRoute(route *routev1.Route) error {
 	// TODO: return all errors at once
+	// TODO error when fields that should be empty are set
 
 	// get the expected settings from the default route
 	expectedRoute := defaultRoute()
@@ -64,7 +75,7 @@ func isValidRoute(route *routev1.Route) error {
 		return fmt.Errorf("route contains wrong TLS termination - '%s' is required: %#v", expTLSTermination, route)
 	}
 
-	if route.Spec.TLS.InsecureEdgeTerminationPolicy != routev1.InsecureEdgeTerminationPolicyRedirect {
+	if route.Spec.TLS.InsecureEdgeTerminationPolicy != expInsecureEdgeTerminationPolicy {
 		return fmt.Errorf("route contains wrong insecure termination policy - '%s' is required: %#v", expInsecureEdgeTerminationPolicy, route)
 	}
 
@@ -83,9 +94,23 @@ func defaultRoute() *routev1.Route {
 				TargetPort: intstr.FromInt(containerPort),
 			},
 			TLS: &routev1.TLSConfig{
-				Termination:                   routev1.TLSTerminationReencrypt,
+				Termination:                   routev1.TLSTerminationPassthrough,
 				InsecureEdgeTerminationPolicy: routev1.InsecureEdgeTerminationPolicyRedirect,
 			},
 		},
 	}
 }
+
+func routerSecretToSNI(routerSecret *corev1.Secret) []configv1.NamedCertificate {
+	var out []configv1.NamedCertificate
+	for key := range routerSecret.Data {
+		out = append(out, configv1.NamedCertificate{
+			Names: []string{"*." + key}, // ingress domain is always a wildcard
+			CertInfo: configv1.CertInfo{ // the cert and key are appended together
+				CertFile: routerCertsLocalMount + "/" + key,
+				KeyFile:  routerCertsLocalMount + "/" + key,
+			},
+		})
+	}
+	return out
+}

pkg/operator2/starter.go

@@ -140,6 +140,14 @@ func RunOperator(ctx *controllercmd.ControllerContext) error {
 		return err
 	}
 
+	// add syncing for router certs for all cluster ingresses
+	if err := resourceSyncer.SyncSecret(
+		resourcesynccontroller.ResourceLocation{Namespace: targetName, Name: routerCertsLocalName},
+		resourcesynccontroller.ResourceLocation{Namespace: machineConfigNamespace, Name: routerCertsSharedName},
+	); err != nil {
+		return err
+	}
+
 	operator := NewAuthenticationOperator(
 		*operatorClient,
 		kubeInformersNamespaced,