Separate bootstrap building logic into the envoy/bootstrap package

pkg/constants/constants.go

@@ -152,6 +152,33 @@ const (
 	CRDConversionWebhookPort = 9443
 )
 
+// HealthProbe constants
+const (
+	// LivenessProbePort is the port to use for liveness probe
+	LivenessProbePort = int32(15901)
+
+	// ReadinessProbePort is the port to use for readiness probe
+	ReadinessProbePort = int32(15902)
+
+	// StartupProbePort is the port to use for startup probe
+	StartupProbePort = int32(15903)
+
+	// HealthcheckPort is the port to use for healthcheck probe
+	HealthcheckPort = int32(15904)
+
+	// LivenessProbePath is the path to use for liveness probe
+	LivenessProbePath = "/osm-liveness-probe"
+
+	// ReadinessProbePath is the path to use for readiness probe
+	ReadinessProbePath = "/osm-readiness-probe"
+
+	// StartupProbePath is the path to use for startup probe
+	StartupProbePath = "/osm-startup-probe"
+
+	// HealthcheckPath is the path to use for healthcheck probe
+	HealthcheckPath = "/osm-healthcheck"
+)
+
 // Annotations used by the control plane
 const (
 	// SidecarInjectionAnnotation is the annotation used for sidecar injection

pkg/envoy/bootstrap/config.go

@@ -8,16 +8,19 @@ import (
 	xds_cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	xds_core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	xds_endpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	xds_listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	xds_accesslog_stream "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/stream/v3"
 	xds_transport_sockets "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	xds_upstream_http "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3"
 	xds_discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
 	"github.com/golang/protobuf/ptypes/any"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/openservicemesh/osm/pkg/configurator"
 	"github.com/openservicemesh/osm/pkg/constants"
 	"github.com/openservicemesh/osm/pkg/envoy"
 	"github.com/openservicemesh/osm/pkg/errcode"
+	"github.com/openservicemesh/osm/pkg/utils"
 )
 
 const (
@@ -282,3 +285,111 @@ func BuildFromConfig(config Config) (*xds_bootstrap.Bootstrap, error) {
 
 	return bootstrap, nil
 }
+
+// GenerateEnvoyConfig generates an envoy bootstrap configuration from the given metadata.
+func GenerateEnvoyConfig(config EnvoyBootstrapConfigMeta, cfg configurator.Configurator) (*xds_bootstrap.Bootstrap, error) {
+	bootstrapConfig, err := BuildFromConfig(Config{
+		NodeID:                config.NodeID,
+		AdminPort:             constants.EnvoyAdminPort,
+		XDSClusterName:        constants.OSMControllerName,
+		XDSHost:               config.XDSHost,
+		XDSPort:               config.XDSPort,
+		TLSMinProtocolVersion: config.TLSMinProtocolVersion,
+		TLSMaxProtocolVersion: config.TLSMaxProtocolVersion,
+		CipherSuites:          config.CipherSuites,
+		ECDHCurves:            config.ECDHCurves,
+	})
+	if err != nil {
+		log.Error().Err(err).Msgf("Error building Envoy boostrap config")
+		return nil, err
+	}
+
+	probeListeners, probeClusters, err := getProbeResources(config)
+	if err != nil {
+		return nil, err
+	}
+	bootstrapConfig.StaticResources.Listeners = append(bootstrapConfig.StaticResources.Listeners, probeListeners...)
+	bootstrapConfig.StaticResources.Clusters = append(bootstrapConfig.StaticResources.Clusters, probeClusters...)
+
+	return bootstrapConfig, nil
+}
+
+// GetTLSSDSConfigYAML returns the statically used TLS SDS config YAML.
+func GetTLSSDSConfigYAML() ([]byte, error) {
+	tlsSDSConfig, err := BuildTLSSecret()
+	if err != nil {
+		log.Error().Err(err).Msgf("Error building Envoy TLS Certificate SDS Config")
+		return nil, err
+	}
+
+	configYAML, err := utils.ProtoToYAML(tlsSDSConfig)
+	if err != nil {
+		log.Error().Err(err).Str(errcode.Kind, errcode.GetErrCodeWithMetric(errcode.ErrMarshallingProtoToYAML)).
+			Msgf("Failed to marshal Envoy TLS Certificate SDS Config to yaml")
+		return nil, err
+	}
+	return configYAML, nil
+}
+
+// GetValidationContextSDSConfigYAML returns the statically used validation context SDS config YAML.
+func GetValidationContextSDSConfigYAML() ([]byte, error) {
+	validationContextSDSConfig, err := BuildValidationSecret()
+	if err != nil {
+		log.Error().Err(err).Msgf("Error building Envoy Validation Context SDS Config")
+		return nil, err
+	}
+
+	configYAML, err := utils.ProtoToYAML(validationContextSDSConfig)
+	if err != nil {
+		log.Error().Err(err).Str(errcode.Kind, errcode.GetErrCodeWithMetric(errcode.ErrMarshallingProtoToYAML)).
+			Msgf("Failed to marshal Envoy Validation Context SDS Config to yaml")
+		return nil, err
+	}
+	return configYAML, nil
+}
+
+// getProbeResources returns the listener and cluster objects that are statically configured to serve
+// startup, readiness and liveness probes.
+// These will not change during the lifetime of the Pod.
+// If the original probe defined a TCPSocket action, listener and cluster objects are not configured
+// to serve that probe.
+func getProbeResources(config EnvoyBootstrapConfigMeta) ([]*xds_listener.Listener, []*xds_cluster.Cluster, error) {
+	// This slice is the list of listeners for liveness, readiness, startup IF these have been configured in the Pod Spec
+	var listeners []*xds_listener.Listener
+	var clusters []*xds_cluster.Cluster
+
+	// Is there a liveness probe in the Pod Spec?
+	if config.OriginalHealthProbes.Liveness != nil && !config.OriginalHealthProbes.Liveness.IsTCPSocket {
+		listener, err := getLivenessListener(config.OriginalHealthProbes.Liveness)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error getting liveness listener")
+			return nil, nil, err
+		}
+		listeners = append(listeners, listener)
+		clusters = append(clusters, getLivenessCluster(config.OriginalHealthProbes.Liveness))
+	}
+
+	// Is there a readiness probe in the Pod Spec?
+	if config.OriginalHealthProbes.Readiness != nil && !config.OriginalHealthProbes.Readiness.IsTCPSocket {
+		listener, err := getReadinessListener(config.OriginalHealthProbes.Readiness)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error getting readiness listener")
+			return nil, nil, err
+		}
+		listeners = append(listeners, listener)
+		clusters = append(clusters, getReadinessCluster(config.OriginalHealthProbes.Readiness))
+	}
+
+	// Is there a startup probe in the Pod Spec?
+	if config.OriginalHealthProbes.Startup != nil && !config.OriginalHealthProbes.Startup.IsTCPSocket {
+		listener, err := getStartupListener(config.OriginalHealthProbes.Startup)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error getting startup listener")
+			return nil, nil, err
+		}
+		listeners = append(listeners, listener)
+		clusters = append(clusters, getStartupCluster(config.OriginalHealthProbes.Startup))
+	}
+
+	return listeners, clusters, nil
+}

pkg/envoy/bootstrap/config_test.go

@@ -1,12 +1,27 @@
 package bootstrap
 
 import (
+	"fmt"
+	"io/ioutil"
+	"path"
+	"path/filepath"
+
+	xds_bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/golang/mock/gomock"
+
 	"testing"
 
+	"github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
+	"github.com/openservicemesh/osm/pkg/configurator"
+
 	tassert "github.com/stretchr/testify/assert"
 
 	tresorFake "github.com/openservicemesh/osm/pkg/certificate/providers/tresor/fake"
 	"github.com/openservicemesh/osm/pkg/constants"
+	"github.com/openservicemesh/osm/pkg/models"
 	"github.com/openservicemesh/osm/pkg/utils"
 )
 
@@ -103,3 +118,125 @@ static_resources:
 `
 	assert.Equal(expectedYAML, string(actualYAML))
 }
+
+var _ = Describe("Test functions creating Envoy bootstrap configuration", func() {
+	const (
+		// This file contains the Bootstrap YAML generated for all of the Envoy proxies in OSM.
+		// This is provisioned by the MutatingWebhook during the addition of a sidecar
+		// to every new Pod that is being created in a namespace participating in the service mesh.
+		// We deliberately leave this entire string literal here to document and visualize what the
+		// generated YAML looks like!
+		expectedEnvoyBootstrapConfigFileName        = "expected_envoy_bootstrap_config.yaml"
+		actualGeneratedEnvoyBootstrapConfigFileName = "actual_envoy_bootstrap_config.yaml"
+
+		// All the YAML files listed above are in this sub-directory
+		directoryForYAMLFiles = "test_fixtures"
+	)
+
+	meshConfig := v1alpha2.MeshConfig{
+		Spec: v1alpha2.MeshConfigSpec{
+			Sidecar: v1alpha2.SidecarSpec{
+				TLSMinProtocolVersion: "TLSv1_2",
+				TLSMaxProtocolVersion: "TLSv1_3",
+				CipherSuites:          []string{},
+			},
+		},
+	}
+
+	cert := tresorFake.NewFakeCertificate()
+	mockCtrl := gomock.NewController(GinkgoT())
+	mockConfigurator := configurator.NewMockConfigurator(mockCtrl)
+	mockConfigurator.EXPECT().GetMeshConfig().Return(meshConfig).AnyTimes()
+
+	getExpectedEnvoyYAML := func(filename string) string {
+		expectedEnvoyConfig, err := ioutil.ReadFile(filepath.Clean(path.Join(directoryForYAMLFiles, filename)))
+		if err != nil {
+			log.Error().Err(err).Msgf("Error reading expected Envoy bootstrap YAML from file %s", filename)
+		}
+		Expect(err).ToNot(HaveOccurred())
+		return string(expectedEnvoyConfig)
+	}
+
+	getExpectedEnvoyConfig := func(filename string) *xds_bootstrap.Bootstrap {
+		yaml := getExpectedEnvoyYAML(filename)
+		conf := xds_bootstrap.Bootstrap{}
+		err := utils.YAMLToProto([]byte(yaml), &conf)
+		Expect(err).ToNot(HaveOccurred())
+		return &conf
+	}
+
+	saveActualEnvoyConfig := func(filename string, actual *xds_bootstrap.Bootstrap) {
+		actualContent, err := utils.ProtoToYAML(actual)
+		Expect(err).ToNot(HaveOccurred())
+		err = ioutil.WriteFile(filepath.Clean(path.Join(directoryForYAMLFiles, filename)), actualContent, 0600)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error writing actual Envoy Cluster XDS YAML to file %s", filename)
+		}
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+	probes := models.HealthProbes{
+		Liveness:  &models.HealthProbe{Path: "/liveness", Port: 81, IsHTTP: true},
+		Readiness: &models.HealthProbe{Path: "/readiness", Port: 82, IsHTTP: true},
+		Startup:   &models.HealthProbe{Path: "/startup", Port: 83, IsHTTP: true},
+	}
+
+	config := EnvoyBootstrapConfigMeta{
+		NodeID: cert.GetCommonName().String(),
+
+		EnvoyAdminPort: 15000,
+
+		XDSClusterName: constants.OSMControllerName,
+		XDSHost:        "osm-controller.b.svc.cluster.local",
+		XDSPort:        15128,
+
+		OriginalHealthProbes:  probes,
+		TLSMinProtocolVersion: meshConfig.Spec.Sidecar.TLSMinProtocolVersion,
+		TLSMaxProtocolVersion: meshConfig.Spec.Sidecar.TLSMaxProtocolVersion,
+		CipherSuites:          meshConfig.Spec.Sidecar.CipherSuites,
+		ECDHCurves:            meshConfig.Spec.Sidecar.ECDHCurves,
+	}
+
+	Context("Test GenerateEnvoyConfig()", func() {
+		It("creates Envoy bootstrap config", func() {
+			config.OriginalHealthProbes = probes
+			actual, err := GenerateEnvoyConfig(config, mockConfigurator)
+			Expect(err).ToNot(HaveOccurred())
+			saveActualEnvoyConfig(actualGeneratedEnvoyBootstrapConfigFileName, actual)
+
+			expectedEnvoyConfig := getExpectedEnvoyConfig(expectedEnvoyBootstrapConfigFileName)
+
+			actualYaml, err := utils.ProtoToYAML(actual)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedYaml, err := utils.ProtoToYAML(expectedEnvoyConfig)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(actualYaml).To(Equal(expectedYaml),
+				fmt.Sprintf("	 %s and %s\nExpected:\n%s\nActual:\n%s\n",
+					expectedEnvoyBootstrapConfigFileName, actualGeneratedEnvoyBootstrapConfigFileName, expectedYaml, actualYaml))
+		})
+	})
+
+	Context("Test getProbeResources()", func() {
+		It("Should not create listeners and clusters when there are no probes", func() {
+			config.OriginalHealthProbes = models.HealthProbes{} // no probes
+			actualListeners, actualClusters, err := getProbeResources(config)
+			Expect(err).To(BeNil())
+			Expect(actualListeners).To(BeNil())
+			Expect(actualClusters).To(BeNil())
+		})
+
+		It("Should not create listeners and cluster for TCPSocket probes", func() {
+			config.OriginalHealthProbes = models.HealthProbes{
+				Liveness:  &models.HealthProbe{Port: 81, IsTCPSocket: true},
+				Readiness: &models.HealthProbe{Port: 82, IsTCPSocket: true},
+				Startup:   &models.HealthProbe{Port: 83, IsTCPSocket: true},
+			}
+			actualListeners, actualClusters, err := getProbeResources(config)
+			Expect(err).To(BeNil())
+			Expect(actualListeners).To(BeNil())
+			Expect(actualClusters).To(BeNil())
+		})
+	})
+})