Separate bootstrap building logic into the envoy/bootstrap package (#4838)

This PR separates the bootstrap config build logic into the bootstrap
package. There are 0 "logic" code changes, only moving code around,
exporting previously unexported functions, and prefacing 
methods/types with the new package name.

The main reason to do this is that there are changes coming with
respect to rotating the secrets with the bootstrap config. Because
of this, we will need to modify how we are creating and writing the
bootstrap config.

Signed-off-by: Sean Teeling <[email protected]>

Author: steeling

pkg/constants/constants.go

@@ -152,6 +152,33 @@ const (
 	CRDConversionWebhookPort = 9443
 )
 
+// HealthProbe constants
+const (
+	// LivenessProbePort is the port to use for liveness probe
+	LivenessProbePort = int32(15901)
+
+	// ReadinessProbePort is the port to use for readiness probe
+	ReadinessProbePort = int32(15902)
+
+	// StartupProbePort is the port to use for startup probe
+	StartupProbePort = int32(15903)
+
+	// HealthcheckPort is the port to use for healthcheck probe
+	HealthcheckPort = int32(15904)
+
+	// LivenessProbePath is the path to use for liveness probe
+	LivenessProbePath = "/osm-liveness-probe"
+
+	// ReadinessProbePath is the path to use for readiness probe
+	ReadinessProbePath = "/osm-readiness-probe"
+
+	// StartupProbePath is the path to use for startup probe
+	StartupProbePath = "/osm-startup-probe"
+
+	// HealthcheckPath is the path to use for healthcheck probe
+	HealthcheckPath = "/osm-healthcheck"
+)
+
 // Annotations used by the control plane
 const (
 	// SidecarInjectionAnnotation is the annotation used for sidecar injection

pkg/envoy/bootstrap/config.go

@@ -8,16 +8,19 @@ import (
 	xds_cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	xds_core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	xds_endpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	xds_listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	xds_accesslog_stream "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/stream/v3"
 	xds_transport_sockets "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	xds_upstream_http "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3"
 	xds_discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
 	"github.com/golang/protobuf/ptypes/any"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/openservicemesh/osm/pkg/configurator"
 	"github.com/openservicemesh/osm/pkg/constants"
 	"github.com/openservicemesh/osm/pkg/envoy"
 	"github.com/openservicemesh/osm/pkg/errcode"
+	"github.com/openservicemesh/osm/pkg/utils"
 )
 
 const (
@@ -282,3 +285,111 @@ func BuildFromConfig(config Config) (*xds_bootstrap.Bootstrap, error) {
 
 	return bootstrap, nil
 }
+
+// GenerateEnvoyConfig generates an envoy bootstrap configuration from the given metadata.
+func GenerateEnvoyConfig(config EnvoyBootstrapConfigMeta, cfg configurator.Configurator) (*xds_bootstrap.Bootstrap, error) {
+	bootstrapConfig, err := BuildFromConfig(Config{
+		NodeID:                config.NodeID,
+		AdminPort:             constants.EnvoyAdminPort,
+		XDSClusterName:        constants.OSMControllerName,
+		XDSHost:               config.XDSHost,
+		XDSPort:               config.XDSPort,
+		TLSMinProtocolVersion: config.TLSMinProtocolVersion,
+		TLSMaxProtocolVersion: config.TLSMaxProtocolVersion,
+		CipherSuites:          config.CipherSuites,
+		ECDHCurves:            config.ECDHCurves,
+	})
+	if err != nil {
+		log.Error().Err(err).Msgf("Error building Envoy boostrap config")
+		return nil, err
+	}
+
+	probeListeners, probeClusters, err := getProbeResources(config)
+	if err != nil {
+		return nil, err
+	}
+	bootstrapConfig.StaticResources.Listeners = append(bootstrapConfig.StaticResources.Listeners, probeListeners...)
+	bootstrapConfig.StaticResources.Clusters = append(bootstrapConfig.StaticResources.Clusters, probeClusters...)
+
+	return bootstrapConfig, nil
+}
+
+// GetTLSSDSConfigYAML returns the statically used TLS SDS config YAML.
+func GetTLSSDSConfigYAML() ([]byte, error) {
+	tlsSDSConfig, err := BuildTLSSecret()
+	if err != nil {
+		log.Error().Err(err).Msgf("Error building Envoy TLS Certificate SDS Config")
+		return nil, err
+	}
+
+	configYAML, err := utils.ProtoToYAML(tlsSDSConfig)
+	if err != nil {
+		log.Error().Err(err).Str(errcode.Kind, errcode.GetErrCodeWithMetric(errcode.ErrMarshallingProtoToYAML)).
+			Msgf("Failed to marshal Envoy TLS Certificate SDS Config to yaml")
+		return nil, err
+	}
+	return configYAML, nil
+}
+
+// GetValidationContextSDSConfigYAML returns the statically used validation context SDS config YAML.
+func GetValidationContextSDSConfigYAML() ([]byte, error) {
+	validationContextSDSConfig, err := BuildValidationSecret()
+	if err != nil {
+		log.Error().Err(err).Msgf("Error building Envoy Validation Context SDS Config")
+		return nil, err
+	}
+
+	configYAML, err := utils.ProtoToYAML(validationContextSDSConfig)
+	if err != nil {
+		log.Error().Err(err).Str(errcode.Kind, errcode.GetErrCodeWithMetric(errcode.ErrMarshallingProtoToYAML)).
+			Msgf("Failed to marshal Envoy Validation Context SDS Config to yaml")
+		return nil, err
+	}
+	return configYAML, nil
+}
+
+// getProbeResources returns the listener and cluster objects that are statically configured to serve
+// startup, readiness and liveness probes.
+// These will not change during the lifetime of the Pod.
+// If the original probe defined a TCPSocket action, listener and cluster objects are not configured
+// to serve that probe.
+func getProbeResources(config EnvoyBootstrapConfigMeta) ([]*xds_listener.Listener, []*xds_cluster.Cluster, error) {
+	// This slice is the list of listeners for liveness, readiness, startup IF these have been configured in the Pod Spec
+	var listeners []*xds_listener.Listener
+	var clusters []*xds_cluster.Cluster
+
+	// Is there a liveness probe in the Pod Spec?
+	if config.OriginalHealthProbes.Liveness != nil && !config.OriginalHealthProbes.Liveness.IsTCPSocket {
+		listener, err := getLivenessListener(config.OriginalHealthProbes.Liveness)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error getting liveness listener")
+			return nil, nil, err
+		}
+		listeners = append(listeners, listener)
+		clusters = append(clusters, getLivenessCluster(config.OriginalHealthProbes.Liveness))
+	}
+
+	// Is there a readiness probe in the Pod Spec?
+	if config.OriginalHealthProbes.Readiness != nil && !config.OriginalHealthProbes.Readiness.IsTCPSocket {
+		listener, err := getReadinessListener(config.OriginalHealthProbes.Readiness)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error getting readiness listener")
+			return nil, nil, err
+		}
+		listeners = append(listeners, listener)
+		clusters = append(clusters, getReadinessCluster(config.OriginalHealthProbes.Readiness))
+	}
+
+	// Is there a startup probe in the Pod Spec?
+	if config.OriginalHealthProbes.Startup != nil && !config.OriginalHealthProbes.Startup.IsTCPSocket {
+		listener, err := getStartupListener(config.OriginalHealthProbes.Startup)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error getting startup listener")
+			return nil, nil, err
+		}
+		listeners = append(listeners, listener)
+		clusters = append(clusters, getStartupCluster(config.OriginalHealthProbes.Startup))
+	}
+
+	return listeners, clusters, nil
+}

pkg/envoy/bootstrap/config_test.go

@@ -1,12 +1,27 @@
 package bootstrap
 
 import (
+	"fmt"
+	"io/ioutil"
+	"path"
+	"path/filepath"
+
+	xds_bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/golang/mock/gomock"
+
 	"testing"
 
+	"github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
+	"github.com/openservicemesh/osm/pkg/configurator"
+
 	tassert "github.com/stretchr/testify/assert"
 
 	tresorFake "github.com/openservicemesh/osm/pkg/certificate/providers/tresor/fake"
 	"github.com/openservicemesh/osm/pkg/constants"
+	"github.com/openservicemesh/osm/pkg/models"
 	"github.com/openservicemesh/osm/pkg/utils"
 )
 
@@ -103,3 +118,125 @@ static_resources:
 `
 	assert.Equal(expectedYAML, string(actualYAML))
 }
+
+var _ = Describe("Test functions creating Envoy bootstrap configuration", func() {
+	const (
+		// This file contains the Bootstrap YAML generated for all of the Envoy proxies in OSM.
+		// This is provisioned by the MutatingWebhook during the addition of a sidecar
+		// to every new Pod that is being created in a namespace participating in the service mesh.
+		// We deliberately leave this entire string literal here to document and visualize what the
+		// generated YAML looks like!
+		expectedEnvoyBootstrapConfigFileName        = "expected_envoy_bootstrap_config.yaml"
+		actualGeneratedEnvoyBootstrapConfigFileName = "actual_envoy_bootstrap_config.yaml"
+
+		// All the YAML files listed above are in this sub-directory
+		directoryForYAMLFiles = "test_fixtures"
+	)
+
+	meshConfig := v1alpha2.MeshConfig{
+		Spec: v1alpha2.MeshConfigSpec{
+			Sidecar: v1alpha2.SidecarSpec{
+				TLSMinProtocolVersion: "TLSv1_2",
+				TLSMaxProtocolVersion: "TLSv1_3",
+				CipherSuites:          []string{},
+			},
+		},
+	}
+
+	cert := tresorFake.NewFakeCertificate()
+	mockCtrl := gomock.NewController(GinkgoT())
+	mockConfigurator := configurator.NewMockConfigurator(mockCtrl)
+	mockConfigurator.EXPECT().GetMeshConfig().Return(meshConfig).AnyTimes()
+
+	getExpectedEnvoyYAML := func(filename string) string {
+		expectedEnvoyConfig, err := ioutil.ReadFile(filepath.Clean(path.Join(directoryForYAMLFiles, filename)))
+		if err != nil {
+			log.Error().Err(err).Msgf("Error reading expected Envoy bootstrap YAML from file %s", filename)
+		}
+		Expect(err).ToNot(HaveOccurred())
+		return string(expectedEnvoyConfig)
+	}
+
+	getExpectedEnvoyConfig := func(filename string) *xds_bootstrap.Bootstrap {
+		yaml := getExpectedEnvoyYAML(filename)
+		conf := xds_bootstrap.Bootstrap{}
+		err := utils.YAMLToProto([]byte(yaml), &conf)
+		Expect(err).ToNot(HaveOccurred())
+		return &conf
+	}
+
+	saveActualEnvoyConfig := func(filename string, actual *xds_bootstrap.Bootstrap) {
+		actualContent, err := utils.ProtoToYAML(actual)
+		Expect(err).ToNot(HaveOccurred())
+		err = ioutil.WriteFile(filepath.Clean(path.Join(directoryForYAMLFiles, filename)), actualContent, 0600)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error writing actual Envoy Cluster XDS YAML to file %s", filename)
+		}
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+	probes := models.HealthProbes{
+		Liveness:  &models.HealthProbe{Path: "/liveness", Port: 81, IsHTTP: true},
+		Readiness: &models.HealthProbe{Path: "/readiness", Port: 82, IsHTTP: true},
+		Startup:   &models.HealthProbe{Path: "/startup", Port: 83, IsHTTP: true},
+	}
+
+	config := EnvoyBootstrapConfigMeta{
+		NodeID: cert.GetCommonName().String(),
+
+		EnvoyAdminPort: 15000,
+
+		XDSClusterName: constants.OSMControllerName,
+		XDSHost:        "osm-controller.b.svc.cluster.local",
+		XDSPort:        15128,
+
+		OriginalHealthProbes:  probes,
+		TLSMinProtocolVersion: meshConfig.Spec.Sidecar.TLSMinProtocolVersion,
+		TLSMaxProtocolVersion: meshConfig.Spec.Sidecar.TLSMaxProtocolVersion,
+		CipherSuites:          meshConfig.Spec.Sidecar.CipherSuites,
+		ECDHCurves:            meshConfig.Spec.Sidecar.ECDHCurves,
+	}
+
+	Context("Test GenerateEnvoyConfig()", func() {
+		It("creates Envoy bootstrap config", func() {
+			config.OriginalHealthProbes = probes
+			actual, err := GenerateEnvoyConfig(config, mockConfigurator)
+			Expect(err).ToNot(HaveOccurred())
+			saveActualEnvoyConfig(actualGeneratedEnvoyBootstrapConfigFileName, actual)
+
+			expectedEnvoyConfig := getExpectedEnvoyConfig(expectedEnvoyBootstrapConfigFileName)
+
+			actualYaml, err := utils.ProtoToYAML(actual)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedYaml, err := utils.ProtoToYAML(expectedEnvoyConfig)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(actualYaml).To(Equal(expectedYaml),
+				fmt.Sprintf("	 %s and %s\nExpected:\n%s\nActual:\n%s\n",
+					expectedEnvoyBootstrapConfigFileName, actualGeneratedEnvoyBootstrapConfigFileName, expectedYaml, actualYaml))
+		})
+	})
+
+	Context("Test getProbeResources()", func() {
+		It("Should not create listeners and clusters when there are no probes", func() {
+			config.OriginalHealthProbes = models.HealthProbes{} // no probes
+			actualListeners, actualClusters, err := getProbeResources(config)
+			Expect(err).To(BeNil())
+			Expect(actualListeners).To(BeNil())
+			Expect(actualClusters).To(BeNil())
+		})
+
+		It("Should not create listeners and cluster for TCPSocket probes", func() {
+			config.OriginalHealthProbes = models.HealthProbes{
+				Liveness:  &models.HealthProbe{Port: 81, IsTCPSocket: true},
+				Readiness: &models.HealthProbe{Port: 82, IsTCPSocket: true},
+				Startup:   &models.HealthProbe{Port: 83, IsTCPSocket: true},
+			}
+			actualListeners, actualClusters, err := getProbeResources(config)
+			Expect(err).To(BeNil())
+			Expect(actualListeners).To(BeNil())
+			Expect(actualClusters).To(BeNil())
+		})
+	})
+})