Skip to content
This repository was archived by the owner on Jul 11, 2023. It is now read-only.

Commit 0163584

Browse files
jaelliojaellio
authored
ref(injector): load bootstrap SDS configuration from filesystem (#4635)
Loads the SDS configuration for the envoy bootstrap config from the file system. Previously, the certificates were provided inline to the Envoy bootstrap configuration which was stored in k8s Secret volume mounted to the Pod. This change will allow OSM to update the certificates used when establishing a connection with xDS without having to recreate the Pod. This allows the envoy xDS certificates to be rotated without potential data plane downtime. This change updates the existing envoy bootstrap k8s secret to include the TLS and validation context SDS configs, xDS certificate, key, and ca cert. This change is a part of the automated root certificate rotation work (#4502) Signed-off-by: jaellio <[email protected]>
1 parent 4f204dd commit 0163584

16 files changed

+281
-123
lines changed

‎cmd/osm-controller/gateway.go

+8-14
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,15 @@ import (
2020
"github.com/openservicemesh/osm/pkg/utils"
2121
)
2222

23-
const (
24-
gatewayBootstrapSecretName = "osm-multicluster-gateway-bootstrap-config" // #nosec G101: Potential hardcoded credentials
25-
bootstrapConfigKey = "bootstrap.yaml"
26-
)
23+
const gatewayBootstrapSecretName = "osm-multicluster-gateway-bootstrap-config" // #nosec G101: Potential hardcoded credentials
2724

2825
func bootstrapOSMMulticlusterGateway(kubeClient kubernetes.Interface, certManager certificate.Manager, osmNamespace string) error {
2926
secret, err := kubeClient.CoreV1().Secrets(osmNamespace).Get(context.Background(), gatewayBootstrapSecretName, metav1.GetOptions{})
3027
if err != nil {
3128
return errors.Errorf("Error fetching OSM gateway's bootstrap config %s/%s", osmNamespace, gatewayBootstrapSecretName)
3229
}
3330

34-
if bootstrapData, ok := secret.Data[bootstrapConfigKey]; !ok {
31+
if bootstrapData, ok := secret.Data[bootstrap.EnvoyBootstrapConfigFile]; !ok {
3532
return errors.Errorf("Missing OSM gateway bootstrap config in %s/%s", osmNamespace, gatewayBootstrapSecretName)
3633
} else if isValidBootstrapData(bootstrapData) {
3734
// If there is a valid bootstrap config, it means we do not need to reconfigure it. It implies
@@ -47,14 +44,11 @@ func bootstrapOSMMulticlusterGateway(kubeClient kubernetes.Interface, certManage
4744
}
4845

4946
bootstrapConfig, err := bootstrap.BuildFromConfig(bootstrap.Config{
50-
NodeID: bootstrapCert.GetCommonName().String(),
51-
AdminPort: constants.EnvoyAdminPort,
52-
XDSClusterName: constants.OSMControllerName,
53-
XDSHost: fmt.Sprintf("%s.%s.svc.%s", constants.OSMControllerName, osmNamespace, identity.ClusterLocalTrustDomain),
54-
XDSPort: constants.ADSServerPort,
55-
TrustedCA: bootstrapCert.GetIssuingCA(),
56-
CertificateChain: bootstrapCert.GetCertificateChain(),
57-
PrivateKey: bootstrapCert.GetPrivateKey(),
47+
NodeID: bootstrapCert.GetCommonName().String(),
48+
AdminPort: constants.EnvoyAdminPort,
49+
XDSClusterName: constants.OSMControllerName,
50+
XDSHost: fmt.Sprintf("%s.%s.svc.%s", constants.OSMControllerName, osmNamespace, identity.ClusterLocalTrustDomain),
51+
XDSPort: constants.ADSServerPort,
5852
})
5953
if err != nil {
6054
return errors.Errorf("Error building OSM gateway's bootstrap config from %s/%s", osmNamespace, gatewayBootstrapSecretName)
@@ -71,7 +65,7 @@ func bootstrapOSMMulticlusterGateway(kubeClient kubernetes.Interface, certManage
7165
Namespace: osmNamespace,
7266
},
7367
Data: map[string][]byte{
74-
bootstrapConfigKey: bootstrapData,
68+
bootstrap.EnvoyBootstrapConfigFile: bootstrapData,
7569
},
7670
}
7771

‎cmd/osm-controller/gateway_test.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"k8s.io/client-go/kubernetes/fake"
1212

1313
tresorFake "github.com/openservicemesh/osm/pkg/certificate/providers/tresor/fake"
14+
"github.com/openservicemesh/osm/pkg/envoy/bootstrap"
1415
)
1516

1617
func TestBootstrapOSMMulticlusterGateway(t *testing.T) {
@@ -37,7 +38,7 @@ func TestBootstrapOSMMulticlusterGateway(t *testing.T) {
3738
Namespace: osmNamespace,
3839
},
3940
Data: map[string][]byte{
40-
bootstrapConfigKey: []byte("-- placeholder --"),
41+
bootstrap.EnvoyBootstrapConfigFile: []byte("-- placeholder --"),
4142
},
4243
},
4344
expectError: false,
@@ -50,7 +51,7 @@ func TestBootstrapOSMMulticlusterGateway(t *testing.T) {
5051
Namespace: osmNamespace,
5152
},
5253
Data: map[string][]byte{
53-
bootstrapConfigKey: []byte(`admin:
54+
bootstrap.EnvoyBootstrapConfigFile: []byte(`admin:
5455
access_log:
5556
- name: envoy.access_loggers.stream
5657
typed_config:

‎pkg/envoy/bootstrap/config.go

+106-14
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package bootstrap
22

33
import (
4+
"path/filepath"
5+
46
xds_accesslog_config "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3"
57
xds_bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
68
xds_cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
@@ -9,6 +11,7 @@ import (
911
xds_accesslog_stream "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/stream/v3"
1012
xds_transport_sockets "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
1113
xds_upstream_http "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3"
14+
xds_discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
1215
"github.com/golang/protobuf/ptypes/any"
1316
"google.golang.org/protobuf/types/known/anypb"
1417

@@ -17,6 +20,98 @@ import (
1720
"github.com/openservicemesh/osm/pkg/errcode"
1821
)
1922

23+
const (
24+
envoyTLSCertificateSecretName = "tls_sds"
25+
envoyValidationContextSecretName = "validation_context_sds"
26+
27+
// EnvoyBootstrapConfigFile is the name Envoy bootstrap configuration file
28+
EnvoyBootstrapConfigFile = "bootstrap.yaml"
29+
30+
// EnvoyTLSCertificateSDSSecretFile is the name of the Envoy TLS certificate SDS config file
31+
EnvoyTLSCertificateSDSSecretFile = "tls_certificate_sds_secret.yaml"
32+
33+
// EnvoyValidationContextSDSSecretFile is the name of the Envoy validation context SDS config file
34+
EnvoyValidationContextSDSSecretFile = "validation_context_sds_secret.yaml"
35+
36+
// EnvoyProxyConfigPath is the path where the Envoy bootstrap config info is located
37+
EnvoyProxyConfigPath = "/etc/envoy"
38+
39+
// EnvoyXDSCACertFile is the name of the Envoy XDS CA certificate file
40+
EnvoyXDSCACertFile = "cacert.pem"
41+
42+
// EnvoyXDSCertFile is the name of the Envoy XDS certificate file
43+
EnvoyXDSCertFile = "sds_cert.pem"
44+
45+
// EnvoyXDSKeyFile is the name of the Envoy XDS private key file
46+
EnvoyXDSKeyFile = "sds_key.pem"
47+
)
48+
49+
var (
50+
envoyTLSCertificateConfigPath = filepath.Join(EnvoyProxyConfigPath, EnvoyTLSCertificateSDSSecretFile)
51+
envoyValidationContextConfigPath = filepath.Join(EnvoyProxyConfigPath, EnvoyValidationContextSDSSecretFile)
52+
53+
envoyXDSCertPath = filepath.Join(EnvoyProxyConfigPath, EnvoyXDSCertFile)
54+
envoyXDSKeyPath = filepath.Join(EnvoyProxyConfigPath, EnvoyXDSKeyFile)
55+
envoyXDSCACertPath = filepath.Join(EnvoyProxyConfigPath, EnvoyXDSCACertFile)
56+
)
57+
58+
// BuildTLSSecret builds and returns an Envoy Discovery Response object for Envoy's xDS TLS
59+
// Certificate
60+
func BuildTLSSecret() (*xds_discovery.DiscoveryResponse, error) {
61+
secret := &xds_transport_sockets.Secret{
62+
Name: envoyTLSCertificateSecretName,
63+
Type: &xds_transport_sockets.Secret_TlsCertificate{
64+
TlsCertificate: &xds_transport_sockets.TlsCertificate{
65+
CertificateChain: &xds_core.DataSource{
66+
Specifier: &xds_core.DataSource_Filename{
67+
Filename: envoyXDSCertPath,
68+
},
69+
},
70+
PrivateKey: &xds_core.DataSource{
71+
Specifier: &xds_core.DataSource_Filename{
72+
Filename: envoyXDSKeyPath,
73+
},
74+
},
75+
},
76+
},
77+
}
78+
marshalledSecret, err := anypb.New(secret)
79+
if err != nil {
80+
log.Error().Err(err).Msg("Error marshalling Secret for Envoy's xDS TLS certificate resource")
81+
return nil, err
82+
}
83+
84+
return &xds_discovery.DiscoveryResponse{
85+
Resources: []*any.Any{marshalledSecret},
86+
}, nil
87+
}
88+
89+
// BuildValidationSecret builds and returns an Envoy Discovery Response object for Envoy's xDS
90+
// Validation Context
91+
func BuildValidationSecret() (*xds_discovery.DiscoveryResponse, error) {
92+
secret := &xds_transport_sockets.Secret{
93+
Name: envoyValidationContextSecretName,
94+
Type: &xds_transport_sockets.Secret_ValidationContext{
95+
ValidationContext: &xds_transport_sockets.CertificateValidationContext{
96+
TrustedCa: &xds_core.DataSource{
97+
Specifier: &xds_core.DataSource_Filename{
98+
Filename: envoyXDSCACertPath,
99+
},
100+
},
101+
},
102+
},
103+
}
104+
marshalledSecret, err := anypb.New(secret)
105+
if err != nil {
106+
log.Error().Err(err).Msg("Error marshalling Secret for Envoy's xDS Validation Context resource")
107+
return nil, err
108+
}
109+
110+
return &xds_discovery.DiscoveryResponse{
111+
Resources: []*any.Any{marshalledSecret},
112+
}, nil
113+
}
114+
20115
// BuildFromConfig builds and returns an Envoy Bootstrap object from the given config
21116
func BuildFromConfig(config Config) (*xds_bootstrap.Bootstrap, error) {
22117
httpProtocolOptions := &xds_upstream_http.HttpProtocolOptions{
@@ -51,11 +146,12 @@ func BuildFromConfig(config Config) (*xds_bootstrap.Bootstrap, error) {
51146
AlpnProtocols: []string{
52147
"h2",
53148
},
54-
ValidationContextType: &xds_transport_sockets.CommonTlsContext_ValidationContext{
55-
ValidationContext: &xds_transport_sockets.CertificateValidationContext{
56-
TrustedCa: &xds_core.DataSource{
57-
Specifier: &xds_core.DataSource_InlineBytes{
58-
InlineBytes: config.TrustedCA,
149+
ValidationContextType: &xds_transport_sockets.CommonTlsContext_ValidationContextSdsSecretConfig{
150+
ValidationContextSdsSecretConfig: &xds_transport_sockets.SdsSecretConfig{
151+
Name: envoyValidationContextSecretName,
152+
SdsConfig: &xds_core.ConfigSource{
153+
ConfigSourceSpecifier: &xds_core.ConfigSource_Path{
154+
Path: envoyValidationContextConfigPath,
59155
},
60156
},
61157
},
@@ -66,16 +162,12 @@ func BuildFromConfig(config Config) (*xds_bootstrap.Bootstrap, error) {
66162
CipherSuites: config.CipherSuites,
67163
EcdhCurves: config.ECDHCurves,
68164
},
69-
TlsCertificates: []*xds_transport_sockets.TlsCertificate{
165+
TlsCertificateSdsSecretConfigs: []*xds_transport_sockets.SdsSecretConfig{
70166
{
71-
CertificateChain: &xds_core.DataSource{
72-
Specifier: &xds_core.DataSource_InlineBytes{
73-
InlineBytes: config.CertificateChain,
74-
},
75-
},
76-
PrivateKey: &xds_core.DataSource{
77-
Specifier: &xds_core.DataSource_InlineBytes{
78-
InlineBytes: config.PrivateKey,
167+
Name: envoyTLSCertificateSecretName,
168+
SdsConfig: &xds_core.ConfigSource{
169+
ConfigSourceSpecifier: &xds_core.ConfigSource_Path{
170+
Path: envoyTLSCertificateConfigPath,
79171
},
80172
},
81173
},

‎pkg/envoy/bootstrap/config_test.go

+8-11
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,6 @@ func TestBuildFromConfig(t *testing.T) {
1818
NodeID: cert.GetCommonName().String(),
1919
AdminPort: 15000,
2020
XDSClusterName: constants.OSMControllerName,
21-
TrustedCA: cert.GetIssuingCA(),
22-
CertificateChain: cert.GetCertificateChain(),
23-
PrivateKey: cert.GetPrivateKey(),
2421
XDSHost: "osm-controller.osm-system.svc.cluster.local",
2522
XDSPort: 15128,
2623
TLSMinProtocolVersion: "TLSv1_0",
@@ -80,11 +77,10 @@ static_resources:
8077
common_tls_context:
8178
alpn_protocols:
8279
- h2
83-
tls_certificates:
84-
- certificate_chain:
85-
inline_bytes: eHg=
86-
private_key:
87-
inline_bytes: eXk=
80+
tls_certificate_sds_secret_configs:
81+
- name: tls_sds
82+
sds_config:
83+
path: /etc/envoy/tls_certificate_sds_secret.yaml
8884
tls_params:
8985
cipher_suites:
9086
- abc
@@ -94,9 +90,10 @@ static_resources:
9490
- XYZ
9591
tls_maximum_protocol_version: TLSv1_2
9692
tls_minimum_protocol_version: TLSv1_0
97-
validation_context:
98-
trusted_ca:
99-
inline_bytes: eHg=
93+
validation_context_sds_secret_config:
94+
name: validation_context_sds
95+
sds_config:
96+
path: /etc/envoy/validation_context_sds_secret.yaml
10097
type: LOGICAL_DNS
10198
typed_extension_protocol_options:
10299
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:

‎pkg/envoy/bootstrap/types.go

-10
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,6 @@ type Config struct {
2424
// NodeID is the proxy's node ID
2525
NodeID string
2626

27-
// TrustedCA is the trusted certificate authority used to validate the certificate
28-
// presented by the XDS cluster during a TLS handshake
29-
TrustedCA []byte
30-
31-
// CertificateChain is the certificate used by the proxy to connect to the XDS cluster
32-
CertificateChain []byte
33-
34-
// PrivateKey is the private key for the certificate used by the proxy to connect to the XDS cluster
35-
PrivateKey []byte
36-
3727
// TLSMinProtocolVersion is the minimum supported TLS protocol version
3828
TLSMinProtocolVersion string
3929

‎pkg/injector/envoy_config.go

+51-9
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,6 @@ func getEnvoyConfigYAML(config envoyBootstrapConfigMeta, cfg configurator.Config
2424
NodeID: config.NodeID,
2525
AdminPort: constants.EnvoyAdminPort,
2626
XDSClusterName: constants.OSMControllerName,
27-
TrustedCA: config.RootCert,
28-
CertificateChain: config.Cert,
29-
PrivateKey: config.Key,
3027
XDSHost: config.XDSHost,
3128
XDSPort: config.XDSPort,
3229
TLSMinProtocolVersion: config.TLSMinProtocolVersion,
@@ -55,6 +52,38 @@ func getEnvoyConfigYAML(config envoyBootstrapConfigMeta, cfg configurator.Config
5552
return configYAML, nil
5653
}
5754

55+
func getTLSSDSConfigYAML() ([]byte, error) {
56+
tlsSDSConfig, err := bootstrap.BuildTLSSecret()
57+
if err != nil {
58+
log.Error().Err(err).Msgf("Error building Envoy TLS Certificate SDS Config")
59+
return nil, err
60+
}
61+
62+
configYAML, err := utils.ProtoToYAML(tlsSDSConfig)
63+
if err != nil {
64+
log.Error().Err(err).Str(errcode.Kind, errcode.GetErrCodeWithMetric(errcode.ErrMarshallingProtoToYAML)).
65+
Msgf("Failed to marshal Envoy TLS Certificate SDS Config to yaml")
66+
return nil, err
67+
}
68+
return configYAML, nil
69+
}
70+
71+
func getValidationContextSDSConfigYAML() ([]byte, error) {
72+
validationContextSDSConfig, err := bootstrap.BuildValidationSecret()
73+
if err != nil {
74+
log.Error().Err(err).Msgf("Error building Envoy Validation Context SDS Config")
75+
return nil, err
76+
}
77+
78+
configYAML, err := utils.ProtoToYAML(validationContextSDSConfig)
79+
if err != nil {
80+
log.Error().Err(err).Str(errcode.Kind, errcode.GetErrCodeWithMetric(errcode.ErrMarshallingProtoToYAML)).
81+
Msgf("Failed to marshal Envoy Validation Context SDS Config to yaml")
82+
return nil, err
83+
}
84+
return configYAML, nil
85+
}
86+
5887
// getProbeResources returns the listener and cluster objects that are statically configured to serve
5988
// startup, readiness and liveness probes.
6089
// These will not change during the lifetime of the Pod.
@@ -107,10 +136,6 @@ func (wh *mutatingWebhook) createEnvoyBootstrapConfig(name, namespace, osmNamesp
107136
XDSClusterName: constants.OSMControllerName,
108137
NodeID: cert.GetCommonName().String(),
109138

110-
RootCert: cert.GetIssuingCA(),
111-
Cert: cert.GetCertificateChain(),
112-
Key: cert.GetPrivateKey(),
113-
114139
XDSHost: fmt.Sprintf("%s.%s.svc.cluster.local", constants.OSMControllerName, osmNamespace),
115140
XDSPort: constants.ADSServerPort,
116141

@@ -123,12 +148,24 @@ func (wh *mutatingWebhook) createEnvoyBootstrapConfig(name, namespace, osmNamesp
123148
CipherSuites: wh.configurator.GetMeshConfig().Spec.Sidecar.CipherSuites,
124149
ECDHCurves: wh.configurator.GetMeshConfig().Spec.Sidecar.ECDHCurves,
125150
}
126-
yamlContent, err := getEnvoyConfigYAML(configMeta, wh.configurator)
151+
configYamlContent, err := getEnvoyConfigYAML(configMeta, wh.configurator)
127152
if err != nil {
128153
log.Error().Err(err).Msg("Error creating Envoy bootstrap YAML")
129154
return nil, err
130155
}
131156

157+
tlsYamlContent, err := getTLSSDSConfigYAML()
158+
if err != nil {
159+
log.Error().Err(err).Msg("Error creating Envoy TLS Certificate SDS Config YAML")
160+
return nil, err
161+
}
162+
163+
validationYamlContent, err := getValidationContextSDSConfigYAML()
164+
if err != nil {
165+
log.Error().Err(err).Msg("Error creating Envoy Validation Context SDS Config YAML")
166+
return nil, err
167+
}
168+
132169
secret := &corev1.Secret{
133170
ObjectMeta: metav1.ObjectMeta{
134171
Name: name,
@@ -139,7 +176,12 @@ func (wh *mutatingWebhook) createEnvoyBootstrapConfig(name, namespace, osmNamesp
139176
},
140177
},
141178
Data: map[string][]byte{
142-
envoyBootstrapConfigFile: yamlContent,
179+
bootstrap.EnvoyBootstrapConfigFile: configYamlContent,
180+
bootstrap.EnvoyTLSCertificateSDSSecretFile: tlsYamlContent,
181+
bootstrap.EnvoyValidationContextSDSSecretFile: validationYamlContent,
182+
bootstrap.EnvoyXDSCACertFile: cert.GetIssuingCA(),
183+
bootstrap.EnvoyXDSCertFile: cert.GetCertificateChain(),
184+
bootstrap.EnvoyXDSKeyFile: cert.GetPrivateKey(),
143185
},
144186
}
145187
if existing, err := wh.kubeClient.CoreV1().Secrets(namespace).Get(context.Background(), name, metav1.GetOptions{}); err == nil {

0 commit comments

Comments
 (0)