ref: use binary flag to enable use of MeshRootCertificate (#4871)

Uses a binary flag to enable the use of MeshRootCertificate. Updates
the MRC id to be the MRC name. Passes vault token secret reference
options to the controller, injector, and bootstrap.

Signed-off-by: jaellio <[email protected]>

Author: jaellio

charts/osm/README.md

@@ -97,6 +97,7 @@ The following table lists the configurable parameters of the osm chart and their
 | osm.featureFlags.enableEgressPolicy | bool | `true` | Enable OSM's Egress policy API. When enabled, fine grained control over Egress (external) traffic is enforced |
 | osm.featureFlags.enableEnvoyActiveHealthChecks | bool | `false` | Enable Envoy active health checks |
 | osm.featureFlags.enableIngressBackendPolicy | bool | `true` | Enables OSM's IngressBackend policy API. When enabled, OSM will use the IngressBackend API allow ingress traffic to mesh backends |
+| osm.featureFlags.enableMeshRootCertificate | bool | `false` | Enable the MeshRootCertificate to configure the OSM certificate provider |
 | osm.featureFlags.enableRetryPolicy | bool | `false` | Enable Retry Policy for automatic request retries |
 | osm.featureFlags.enableSnapshotCacheMode | bool | `false` | Enables SnapshotCache feature for Envoy xDS server. |
 | osm.featureFlags.enableWASMStats | bool | `true` | Enable extra Envoy statistics generated by a custom WASM extension |
@@ -270,7 +271,7 @@ The following table lists the configurable parameters of the osm chart and their
 | osm.vault.port | int | `8200` | port to use to connect to Vault |
 | osm.vault.protocol | string | `"http"` | protocol to use to connect to Vault |
 | osm.vault.role | string | `"openservicemesh"` | Vault role to be used by Open Service Mesh |
-| osm.vault.secret | object | `{"key":"","name":""}` | The Kubernetes secret storing the Vault token used in OSM |
+| osm.vault.secret | object | `{"key":"","name":""}` | The Kubernetes secret storing the Vault token used in OSM. The secret must be located in the namespace of the OSM installation |
 | osm.vault.secret.key | string | `""` | The Kubernetes secret key with the value bring the Vault token |
 | osm.vault.secret.name | string | `""` | The Kubernetes secret name storing the Vault token used in OSM |
 | osm.vault.token | string | `""` | token that should be used to connect to Vault |

charts/osm/templates/osm-bootstrap-deployment.yaml

@@ -61,11 +61,14 @@ spec:
             "--osm-version", "{{ .Chart.AppVersion }}",
             "--ca-bundle-secret-name", "{{.Values.osm.caBundleSecretName}}",
             "--certificate-manager", "{{.Values.osm.certificateProvider.kind}}",
+            "--enable-mesh-root-certificate={{.Values.osm.featureFlags.enableMeshRootCertificate}}",
             {{ if eq .Values.osm.certificateProvider.kind "vault" }}
             "--vault-host", "{{.Values.osm.vault.host}}",
             "--vault-port", "{{.Values.osm.vault.port}}",
             "--vault-protocol", "{{.Values.osm.vault.protocol}}",
             "--vault-token", "{{.Values.osm.vault.token}}",
+            "--vault-token-secret-name",  "{{ .Values.osm.vault.secret.name }}",
+            "--vault-token-secret-key",  "{{ .Values.osm.vault.secret.key }}",
             {{- end }}
             "--cert-manager-issuer-name", "{{.Values.osm.certmanager.issuerName}}",
             "--cert-manager-issuer-kind", "{{.Values.osm.certmanager.issuerKind}}",

charts/osm/templates/osm-deployment.yaml

@@ -61,11 +61,18 @@ spec:
             "--validator-webhook-config", "{{ include "osm.validatorWebhookConfigName" . }}",
             "--ca-bundle-secret-name", "{{.Values.osm.caBundleSecretName}}",
             "--certificate-manager", "{{.Values.osm.certificateProvider.kind}}",
+            "--enable-mesh-root-certificate={{.Values.osm.featureFlags.enableMeshRootCertificate}}",
             {{ if eq .Values.osm.certificateProvider.kind "vault" }}
             "--vault-host", "{{ required "osm.vault.host is required when osm.certificateProvider.kind==vault" .Values.osm.vault.host }}",
             "--vault-port", "{{.Values.osm.vault.port}}",
             "--vault-protocol", "{{.Values.osm.vault.protocol}}",
-            "--vault-token", "{{ required "osm.vault.token is required when osm.certificateProvider.kind==vault" .Values.osm.vault.token }}",
+            {{ if and (empty .Values.osm.vault.secret.name) (empty .Values.osm.vault.secret.key) }}
+            "--vault-token", "{{ required "osm.vault.token is required when osm.certificateProvider.kind==vault and osm.vault.secret.name and osm.vault.secret.key are empty" .Values.osm.vault.token }}",
+            {{- end }}
+            {{ if empty .Values.osm.vault.token }}
+            "--vault-token-secret-name",  "{{ required "osm.vault.secret.name is required when osm.certificateProvider.kind==vault and osm.vault.token is empty" .Values.osm.vault.secret.name }}",
+            "--vault-token-secret-key",  "{{ required "osm.vault.secret.key is required when osm.certificateProvider.kind==vault and osm.vault.token is empty" .Values.osm.vault.secret.key }}",
+            {{- end }}
             {{- end }}
             "--cert-manager-issuer-name", "{{.Values.osm.certmanager.issuerName}}",
             "--cert-manager-issuer-kind", "{{.Values.osm.certmanager.issuerKind}}",

charts/osm/templates/osm-injector-deployment.yaml

@@ -58,11 +58,14 @@ spec:
             "--webhook-timeout", "{{.Values.osm.injector.webhookTimeoutSeconds}}",
             "--ca-bundle-secret-name", "{{.Values.osm.caBundleSecretName}}",
             "--certificate-manager", "{{.Values.osm.certificateProvider.kind}}",
+            "--enable-mesh-root-certificate={{.Values.osm.featureFlags.enableMeshRootCertificate}}",
             {{ if eq .Values.osm.certificateProvider.kind "vault" }}
             "--vault-host", "{{.Values.osm.vault.host}}",
             "--vault-port", "{{.Values.osm.vault.port}}",
             "--vault-protocol", "{{.Values.osm.vault.protocol}}",
             "--vault-token", "{{.Values.osm.vault.token}}",
+            "--vault-token-secret-name",  "{{ .Values.osm.vault.secret.name }}",
+            "--vault-token-secret-key",  "{{ .Values.osm.vault.secret.key }}",
             {{- end }}
             "--cert-manager-issuer-name", "{{.Values.osm.certmanager.issuerName}}",
             "--cert-manager-issuer-kind", "{{.Values.osm.certmanager.issuerKind}}",

charts/osm/templates/preset-mesh-root-certificate.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.osm.featureFlags.enableMeshRootCertificate }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -42,3 +43,4 @@ data:
         {{- end}}
       }
     }
+{{- end}}

charts/osm/values.schema.json

@@ -983,7 +983,8 @@
                         "enableIngressBackendPolicy",
                         "enableEnvoyActiveHealthChecks",
                         "enableSnapshotCacheMode",
-                        "enableRetryPolicy"
+                        "enableRetryPolicy",
+                        "enableMeshRootCertificate"
                     ],
                     "properties": {
                         "enableWASMStats": {
@@ -1048,6 +1049,15 @@
                             "examples": [
                                 true
                             ]
+                        },
+                        "enableMeshRootCertificate": {
+                            "$id": "#/properties/osm/properties/featureFlags/properties/enableMeshRootCertificate",
+                            "type": "boolean",
+                            "title": "Enable the MeshRootCertificate",
+                            "description": "Enable the MeshRootCertificate to configure the OSM certificate provider.",
+                            "examples": [
+                                false
+                            ]
                         }
                     },
                     "additionalProperties": false

charts/osm/values.yaml

@@ -191,7 +191,7 @@ osm:
     token: ""
     # -- Vault role to be used by Open Service Mesh
     role: openservicemesh
-    # -- The Kubernetes secret storing the Vault token used in OSM
+    # -- The Kubernetes secret storing the Vault token used in OSM. The secret must be located in the namespace of the OSM installation
     secret:
       # -- The Kubernetes secret name storing the Vault token used in OSM
       name: ""
@@ -482,6 +482,8 @@ osm:
     enableSnapshotCacheMode: false
     # -- Enable Retry Policy for automatic request retries
     enableRetryPolicy: false
+    # -- Enable the MeshRootCertificate to configure the OSM certificate provider
+    enableMeshRootCertificate: false
 
   # -- Node tolerations applied to control plane pods.
   # The specified tolerations allow pods to schedule onto nodes with matching taints.
@@ -547,7 +549,7 @@ osm:
 
   #
   # -- OSM's preinstall hook parameters
-  
+
   preinstall:
     ## Affinity settings for pod assignment
     ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
@@ -580,7 +582,7 @@ osm:
 
     ## Affinity settings for pod assignment
     ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
-    affinity: 
+    affinity:
       nodeAffinity:
         requiredDuringSchedulingIgnoredDuringExecution:
           nodeSelectorTerms:

cmd/cli/install_test.go

@@ -28,13 +28,15 @@ import (
 )
 
 const (
-	testRegistrySecret = "test-registry-secret"
-	testVaultHost      = "vault.osm.svc.cluster.local"
-	testVaultToken     = "token"
-	testChartPath      = "testdata/test-chart"
-	kubeVersionMajor   = 1
-	kubeVersionMinor   = 22
-	kubeVersionPatch   = 9
+	testRegistrySecret  = "test-registry-secret"
+	testVaultHost       = "vault.osm.svc.cluster.local"
+	testVaultToken      = "token"
+	testVaultSecretName = "secret"
+	testVaultSecretKey  = "key"
+	testChartPath       = "testdata/test-chart"
+	kubeVersionMajor    = 1
+	kubeVersionMinor    = 22
+	kubeVersionPatch    = 9
 )
 
 func helmCapabilities() *chartutil.Capabilities {
@@ -181,7 +183,7 @@ var _ = Describe("Running the install command", func() {
 		})
 	})
 
-	Describe("with the vault cert manager", func() {
+	Describe("with the vault cert manager using vault token", func() {
 		var (
 			out    *bytes.Buffer
 			store  *storage.Storage
@@ -258,6 +260,87 @@ var _ = Describe("Running the install command", func() {
 		})
 	})
 
+	Describe("with the vault cert manager using token secret ref", func() {
+		var (
+			out    *bytes.Buffer
+			store  *storage.Storage
+			config *helm.Configuration
+			err    error
+		)
+
+		BeforeEach(func() {
+			out = new(bytes.Buffer)
+			store = storage.Init(driver.NewMemory())
+			if mem, ok := store.Driver.(*driver.Memory); ok {
+				mem.SetNamespace(settings.Namespace())
+			}
+
+			config = &helm.Configuration{
+				Releases: store,
+				KubeClient: &kubefake.PrintingKubeClient{
+					Out: ioutil.Discard},
+				Capabilities: helmCapabilities(),
+				Log:          func(format string, v ...interface{}) {},
+			}
+
+			installCmd := getDefaultInstallCmd(out)
+
+			installCmd.setOptions = []string{
+				"osm.certificateProvider.kind=vault",
+				fmt.Sprintf("osm.vault.host=%s", testVaultHost),
+				"osm.vault.token=",
+				fmt.Sprintf("osm.vault.secret.name=%s", testVaultSecretName),
+				fmt.Sprintf("osm.vault.secret.key=%s", testVaultSecretKey),
+			}
+			err = installCmd.run(config)
+		})
+
+		It("should not error", func() {
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should give a message confirming the successful install", func() {
+			Expect(out.String()).To(Equal("OSM installed successfully in namespace [osm-system] with mesh name [osm]\n"))
+		})
+
+		Context("the Helm release", func() {
+			var (
+				rel *release.Release
+				err error
+			)
+
+			BeforeEach(func() {
+				rel, err = config.Releases.Get(defaultMeshName, 1)
+			})
+
+			It("should not error when retrieved", func() {
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("should have the correct values", func() {
+				expectedValues := getDefaultValues()
+				valuesConfig := []string{
+					fmt.Sprintf("osm.certificateProvider.kind=%s", "vault"),
+					fmt.Sprintf("osm.vault.host=%s", testVaultHost),
+					"osm.vault.token=",
+					fmt.Sprintf("osm.vault.secret.name=%s", testVaultSecretName),
+					fmt.Sprintf("osm.vault.secret.key=%s", testVaultSecretKey),
+				}
+				for _, val := range valuesConfig {
+					// parses Helm strvals line and merges into a map
+					err := strvals.ParseInto(val, expectedValues)
+					Expect(err).NotTo(HaveOccurred())
+				}
+
+				Expect(rel.Config).To(BeEquivalentTo(expectedValues))
+			})
+
+			It("should be installed in the correct namespace", func() {
+				Expect(rel.Namespace).To(Equal(settings.Namespace()))
+			})
+		})
+	})
+
 	Describe("without required vault parameters", func() {
 		var (
 			installCmd installCmd
@@ -291,11 +374,32 @@ var _ = Describe("Running the install command", func() {
 			Expect(err.Error()).To(ContainSubstring("osm.vault.host is required"))
 		})
 
-		It("should error when token isn't set", func() {
+		It("should error when token and token secret key are not set", func() {
+			installCmd.setOptions = append(installCmd.setOptions,
+				"osm.vault.host=my-host",
+				"osm.vault.secret.name=secret",
+			)
+			err := installCmd.run(config)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("osm.vault.secret.key is required"))
+		})
+
+		It("should error when token and token secret name are not set", func() {
+			installCmd.setOptions = append(installCmd.setOptions,
+				"osm.vault.host=my-host",
+				"osm.vault.secret.key=key",
+			)
+			err := installCmd.run(config)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("osm.vault.secret.name is required"))
+		})
+
+		It("should error when token and token secret name and key are not set", func() {
 			installCmd.setOptions = append(installCmd.setOptions,
 				"osm.vault.host=my-host",
 			)
 			err := installCmd.run(config)
+			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("osm.vault.token is required"))
 		})
 	})

cmd/osm-bootstrap/osm-bootstrap.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/kubectl/pkg/util"
 
 	configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
+	"github.com/openservicemesh/osm/pkg/certificate"
 	configClientset "github.com/openservicemesh/osm/pkg/gen/client/config/clientset/versioned"
 	"github.com/openservicemesh/osm/pkg/health"
 
@@ -63,7 +64,8 @@ var (
 	meshName           string
 	osmVersion         string
 
-	certProviderKind string
+	certProviderKind          string
+	enableMeshRootCertificate bool
 
 	tresorOptions      providers.TresorOptions
 	vaultOptions       providers.VaultOptions
@@ -94,6 +96,7 @@ func init() {
 
 	// Generic certificate manager/provider options
 	flags.StringVar(&certProviderKind, "certificate-manager", providers.TresorKind.String(), fmt.Sprintf("Certificate manager, one of [%v]", providers.ValidCertificateProviders))
+	flags.BoolVar(&enableMeshRootCertificate, "enable-mesh-root-certificate", false, "Enable unsupported MeshRootCertificate to create the OSM Certificate Manager")
 	flags.StringVar(&caBundleSecretName, "ca-bundle-secret-name", "", "Name of the Kubernetes Secret for the OSM CA bundle")
 
 	// Vault certificate manager/provider options
@@ -102,6 +105,8 @@ func init() {
 	flags.StringVar(&vaultOptions.VaultToken, "vault-token", "", "Secret token for the the Hashi Vault")
 	flags.StringVar(&vaultOptions.VaultRole, "vault-role", "openservicemesh", "Name of the Vault role dedicated to Open Service Mesh")
 	flags.IntVar(&vaultOptions.VaultPort, "vault-port", 8200, "Port of the Hashi Vault")
+	flags.StringVar(&vaultOptions.VaultTokenSecretName, "vault-token-secret-name", "", "Name of the secret storing the Vault token used in OSM")
+	flags.StringVar(&vaultOptions.VaultTokenSecretKey, "vault-token-secret-key", "", "Key for the vault token used in OSM")
 
 	// Cert-manager certificate manager/provider options
 	flags.StringVar(&certManagerOptions.IssuerName, "cert-manager-issuer-name", "osm-ca", "cert-manager issuer name")
@@ -122,6 +127,7 @@ func getCertOptions() (providers.Options, error) {
 		tresorOptions.SecretName = caBundleSecretName
 		return tresorOptions, nil
 	case providers.VaultKind:
+		vaultOptions.VaultTokenSecretNamespace = osmNamespace
 		return vaultOptions, nil
 	case providers.CertManagerKind:
 		return certManagerOptions, nil
@@ -171,10 +177,12 @@ func main() {
 		return
 	}
 
-	err = bootstrap.ensureMeshRootCertificate()
-	if err != nil {
-		log.Fatal().Err(err).Msgf("Error setting up default MeshRootCertificate %s from ConfigMap %s", meshRootCertificateName, presetMeshRootCertificateName)
-		return
+	if enableMeshRootCertificate {
+		err = bootstrap.ensureMeshRootCertificate()
+		if err != nil {
+			log.Fatal().Err(err).Msgf("Error setting up default MeshRootCertificate %s from ConfigMap %s", meshRootCertificateName, presetMeshRootCertificateName)
+			return
+		}
 	}
 
 	err = bootstrap.initiatilizeKubernetesEventsRecorder()
@@ -214,10 +222,19 @@ func main() {
 		log.Fatal().Err(err).Msg("Error getting certificate options")
 	}
 
-	certManager, err := providers.NewCertificateManager(ctx, kubeClient, kubeConfig, cfg, osmNamespace, certOpts, msgBroker, informerCollection, 5*time.Second)
-	if err != nil {
-		events.GenericEventRecorder().FatalEvent(err, events.InvalidCertificateManager,
-			"Error initializing certificate manager of kind %s", certProviderKind)
+	var certManager *certificate.Manager
+	if enableMeshRootCertificate {
+		certManager, err = providers.NewCertificateManagerFromMRC(ctx, kubeClient, kubeConfig, cfg, osmNamespace, certOpts, msgBroker, informerCollection, 5*time.Second)
+		if err != nil {
+			events.GenericEventRecorder().FatalEvent(err, events.InvalidCertificateManager,
+				"Error initializing certificate manager of kind %s from MRC", certProviderKind)
+		}
+	} else {
+		certManager, err = providers.NewCertificateManager(ctx, kubeClient, kubeConfig, cfg, osmNamespace, certOpts, msgBroker, 5*time.Second)
+		if err != nil {
+			events.GenericEventRecorder().FatalEvent(err, events.InvalidCertificateManager,
+				"Error initializing certificate manager of kind %s", certProviderKind)
+		}
 	}
 
 	// Initialize the crd conversion webhook server to support the conversion of OSM's CRDs
@@ -414,7 +431,7 @@ func (b *bootstrap) createMeshRootCertificate() error {
 
 	_, err = b.configClient.ConfigV1alpha2().MeshRootCertificates(b.namespace).UpdateStatus(context.Background(), createdMRC, metav1.UpdateOptions{})
 	if apierrors.IsAlreadyExists(err) {
-		log.Info().Msgf("MeshRootCertificate statys already exists in %s. Skip creating.", b.namespace)
+		log.Info().Msgf("MeshRootCertificate status already exists in %s. Skip creating.", b.namespace)
 	}
 
 	if err != nil {
@@ -440,9 +457,6 @@ func buildMeshRootCertificate(presetMeshRootCertificateConfigMap *corev1.ConfigM
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: meshRootCertificateName,
-			Annotations: map[string]string{
-				constants.MRCVersionAnnotation: "0",
-			},
 		},
 		Spec: presetMeshRootCertificateSpec,
 	}