Move to a passthrough route to support mTLS

This change moves us to a passthrough route that can support the
mTLS requirements of the request header identity provider.

1. Sync router certs from openshift-config-managed to
openshift-authentication namespace
2. Mount (copied) router certs into deployment
3. SNI: wire each key, value pair in router certs secret to a domain
and the mount path for the combined TLS cert and key data.

Signed-off-by: Monis Khan <[email protected]>

Author: enj

pkg/operator2/deployment.go

@@ -28,6 +28,7 @@ func (c *authOperator) getGeneration() int64 {
 func defaultDeployment(
 	operatorConfig *operatorv1.Authentication,
 	syncData *configSyncData,
+	routerSecret *corev1.Secret,
 	resourceVersions ...string,
 ) *appsv1.Deployment {
 	replicas := int32(1) // TODO configurable?
@@ -63,6 +64,12 @@ func defaultDeployment(
 			path:      serviceCAMount,
 			keys:      []string{serviceCAKey},
 		},
+		{
+			name:      routerCertsLocalName,
+			configmap: false,
+			path:      routerCertsLocalMount,
+			keys:      sets.StringKeySet(routerSecret.Data).List(),
+		},
 	} {
 		v, m := data.split()
 		volumes = append(volumes, v)

pkg/operator2/oauth.go

@@ -34,6 +34,7 @@ func init() {
 func (c *authOperator) handleOAuthConfig(
 	operatorConfig *operatorv1.Authentication,
 	route *routev1.Route,
+	routerSecret *corev1.Secret,
 	service *corev1.Service,
 	consoleConfig *configv1.Console,
 ) (
@@ -100,13 +101,14 @@ func (c *authOperator) handleOAuthConfig(
 				ServingInfo: configv1.ServingInfo{
 					BindAddress: fmt.Sprintf("0.0.0.0:%d", containerPort),
 					BindNetwork: "tcp4",
-					// we have valid serving certs provided by service-ca so that we can use reencrypt routes
+					// we have valid serving certs provided by service-ca
+					// this is our main server cert which is used if SNI does not match
 					CertInfo: configv1.CertInfo{
 						CertFile: servingCertPathCert,
 						KeyFile:  servingCertPathKey,
 					},
 					ClientCA:          "", // I think this can be left unset
-					NamedCertificates: nil,
+					NamedCertificates: routerSecretToSNI(routerSecret),
 					MinTLSVersion:     crypto.TLSVersionToNameOrDie(crypto.DefaultTLSVersion()),
 					CipherSuites:      crypto.CipherSuitesToNamesOrDie(crypto.DefaultCiphers()),
 				},

pkg/operator2/operator.go

@@ -67,6 +67,10 @@ const (
 	cliConfigMount      = systemConfigPathConfigMaps + "/" + cliConfigNameAndKey
 	cliConfigPath       = cliConfigMount + "/" + cliConfigNameAndKey
 
+	routerCertsSharedName = "router-ca"
+	routerCertsLocalName  = systemConfigPrefix + "router-certs"
+	routerCertsLocalMount = systemConfigPathSecrets + "/" + routerCertsLocalName
+
 	userConfigPath = "/var/config/user"
 
 	servicePort   = 443
@@ -174,11 +178,11 @@ func (c *authOperator) handleSync(operatorConfig *operatorv1.Authentication) err
 	// we get resource versions so that if either changes, we redeploy our payload
 	resourceVersions := []string{operatorConfig.GetResourceVersion()}
 
-	route, err := c.handleRoute()
+	route, routerSecret, err := c.handleRoute()
 	if err != nil {
 		return err
 	}
-	resourceVersions = append(resourceVersions, route.GetResourceVersion())
+	resourceVersions = append(resourceVersions, route.GetResourceVersion(), routerSecret.GetResourceVersion())
 
 	serviceCA, err := c.handleServiceCA()
 	if err != nil {
@@ -217,7 +221,7 @@ func (c *authOperator) handleSync(operatorConfig *operatorv1.Authentication) err
 	consoleConfig := c.handleConsoleConfig()
 	resourceVersions = append(resourceVersions, consoleConfig.GetResourceVersion())
 
-	oauthConfig, expectedCLIconfig, syncData, err := c.handleOAuthConfig(operatorConfig, route, service, consoleConfig)
+	oauthConfig, expectedCLIconfig, syncData, err := c.handleOAuthConfig(operatorConfig, route, routerSecret, service, consoleConfig)
 	if err != nil {
 		return err
 	}
@@ -241,6 +245,7 @@ func (c *authOperator) handleSync(operatorConfig *operatorv1.Authentication) err
 	expectedDeployment := defaultDeployment(
 		operatorConfig,
 		syncData,
+		routerSecret,
 		resourceVersions...,
 	)
 	// TODO add support for spec.operandSpecs.unsupportedResourcePatches, like:

pkg/operator2/route.go

@@ -2,27 +2,30 @@ package operator2
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/golang/glog"
 
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
+	configv1 "github.com/openshift/api/config/v1"
 	routev1 "github.com/openshift/api/route/v1"
 )
 
-func (c *authOperator) handleRoute() (*routev1.Route, error) {
+func (c *authOperator) handleRoute() (*routev1.Route, *corev1.Secret, error) {
 	route, err := c.route.Get(targetName, metav1.GetOptions{})
 	if errors.IsNotFound(err) {
 		route, err = c.route.Create(defaultRoute())
 	}
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if len(route.Spec.Host) == 0 {
-		return nil, fmt.Errorf("route has no host: %#v", route)
+		return nil, nil, fmt.Errorf("route has no host: %#v", route)
 	}
 
 	if err := isValidRoute(route); err != nil {
@@ -32,14 +35,20 @@ func (c *authOperator) handleRoute() (*routev1.Route, error) {
 		if err := c.route.Delete(route.Name, opts); err != nil && !errors.IsNotFound(err) {
 			glog.Infof("failed to delete invalid route: %v", err)
 		}
-		return nil, err
+		return nil, nil, err
 	}
 
-	return route, nil
+	routerSecret, err := c.secrets.Secrets(targetName).Get(routerCertsLocalName, metav1.GetOptions{})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return route, routerSecret, nil
 }
 
 func isValidRoute(route *routev1.Route) error {
 	// TODO: return all errors at once
+	// TODO error when fields that should be empty are set
 
 	// get the expected settings from the default route
 	expectedRoute := defaultRoute()
@@ -64,7 +73,7 @@ func isValidRoute(route *routev1.Route) error {
 		return fmt.Errorf("route contains wrong TLS termination - '%s' is required: %#v", expTLSTermination, route)
 	}
 
-	if route.Spec.TLS.InsecureEdgeTerminationPolicy != routev1.InsecureEdgeTerminationPolicyRedirect {
+	if route.Spec.TLS.InsecureEdgeTerminationPolicy != expInsecureEdgeTerminationPolicy {
 		return fmt.Errorf("route contains wrong insecure termination policy - '%s' is required: %#v", expInsecureEdgeTerminationPolicy, route)
 	}
 
@@ -83,9 +92,32 @@ func defaultRoute() *routev1.Route {
 				TargetPort: intstr.FromInt(containerPort),
 			},
 			TLS: &routev1.TLSConfig{
-				Termination:                   routev1.TLSTerminationReencrypt,
+				Termination:                   routev1.TLSTerminationPassthrough,
 				InsecureEdgeTerminationPolicy: routev1.InsecureEdgeTerminationPolicyRedirect,
 			},
 		},
 	}
 }
+
+func routerSecretToSNI(routerSecret *corev1.Secret) []configv1.NamedCertificate {
+	var out []configv1.NamedCertificate
+	for key := range routerSecret.Data {
+		out = append(out, configv1.NamedCertificate{
+			Names: []string{routerSecretKeyToName(key)}, // workaround the lack of *.domain.tld as valid key
+			CertInfo: configv1.CertInfo{ // the cert and key are appended together
+				CertFile: routerCertsLocalMount + "/" + key,
+				KeyFile:  routerCertsLocalMount + "/" + key,
+			},
+		})
+	}
+	return out
+}
+
+func routerSecretKeyToName(key string) string {
+	switch {
+	case strings.HasPrefix(key, "."): // .domain.tld means *.domain.tld
+		return "*" + key
+	default:
+		return key // domain.tld is good as-is
+	}
+}

pkg/operator2/starter.go

@@ -112,7 +112,7 @@ func RunOperator(ctx *controllercmd.ControllerContext) error {
 		v1helpers.EnsureOperatorConfigExists(dynamicClient, []byte(resource), gvr)
 	}
 
-	resourceSyncerInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient, targetName, userConfigNamespace)
+	resourceSyncerInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient, targetName, userConfigNamespace, machineConfigNamespace)
 
 	operatorClient := &OperatorClient{
 		authOperatorConfigInformers,
@@ -127,6 +127,13 @@ func RunOperator(ctx *controllercmd.ControllerContext) error {
 		ctx.EventRecorder,
 	)
 
+	if err := resourceSyncer.SyncSecret(
+		resourcesynccontroller.ResourceLocation{Namespace: targetName, Name: routerCertsLocalName},
+		resourcesynccontroller.ResourceLocation{Namespace: machineConfigNamespace, Name: routerCertsSharedName},
+	); err != nil {
+		return err
+	}
+
 	operator := NewAuthenticationOperator(
 		*operatorClient,
 		kubeInformersNamespaced,