Skip to content
This repository was archived by the owner on Jul 11, 2023. It is now read-only.

Commit 1a63975

Browse files
steelingsteeling
committed
Remove certificate common name from the proxy struct.
The CN is no longer static for the duration of the proxy, and is not required to key the proxy. Signed-off-by: Sean Teeling <[email protected]>
1 parent e49058f commit 1a63975

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+422
-657
lines changed

‎pkg/catalog/fake/fake.go

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,9 @@ import (
2727

2828
// NewFakeMeshCatalog creates a new struct implementing catalog.MeshCataloger interface used for testing.
2929
func NewFakeMeshCatalog(kubeClient kubernetes.Interface, meshConfigClient configClientset.Interface) *catalog.MeshCatalog {
30-
var (
31-
mockCtrl *gomock.Controller
32-
mockKubeController *k8s.MockController
33-
mockPolicyController *policy.MockController
34-
)
35-
36-
mockCtrl = gomock.NewController(ginkgo.GinkgoT())
37-
mockKubeController = k8s.NewMockController(mockCtrl)
38-
mockPolicyController = policy.NewMockController(mockCtrl)
30+
mockCtrl := gomock.NewController(ginkgo.GinkgoT())
31+
mockKubeController := k8s.NewMockController(mockCtrl)
32+
mockPolicyController := policy.NewMockController(mockCtrl)
3933

4034
meshSpec := smiFake.NewFakeMeshSpecClient()
4135

‎pkg/catalog/traffictarget.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,8 +139,7 @@ func trafficTargetIdentityToSvcAccount(identitySubject smiAccess.IdentityBinding
139139

140140
// trafficTargetIdentityToServiceIdentity returns an identity of the form <namespace>/<service-account>
141141
func trafficTargetIdentityToServiceIdentity(identitySubject smiAccess.IdentityBindingSubject) identity.ServiceIdentity {
142-
svcAccount := trafficTargetIdentityToSvcAccount(identitySubject)
143-
return identity.GetKubernetesServiceIdentity(svcAccount, identity.ClusterLocalTrustDomain)
142+
return trafficTargetIdentityToSvcAccount(identitySubject).ToServiceIdentity()
144143
}
145144

146145
// trafficTargetIdentitiesToSvcAccounts returns a list of Service Accounts from the given list of identities from a Traffic Target

‎pkg/debugger/proxy.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ func (ds DebugConfig) printProxies(w http.ResponseWriter) {
5151
ts := proxy.GetConnectedAt()
5252
proxyURL := fmt.Sprintf("/debug/proxy?%s=%s", uuidQueryKey, proxy.UUID)
5353
configDumpURL := fmt.Sprintf("%s&%s=%t", proxyURL, proxyConfigQueryKey, true)
54-
_, _ = fmt.Fprintf(w, `<tr><td>%d:</td><td>%s</td><td>%+v</td><td>(%+v ago)</td><td><a href="%s">certs</a></td><td><a href="%s">cfg</a></td></tr>`,
55-
idx+1, proxy.Identity, ts, time.Since(ts), proxyURL, configDumpURL)
54+
_, _ = fmt.Fprintf(w, `<tr><td>%d:</td><td>%s</td><td>%s</td><td>%+v</td><td>(%+v ago)</td><td><a href="%s">certs</a></td><td><a href="%s">cfg</a></td></tr>`,
55+
idx+1, proxy.Identity, proxy.UUID, ts, time.Since(ts), proxyURL, configDumpURL)
5656
}
5757
_, _ = fmt.Fprint(w, `</table>`)
5858
}
@@ -67,9 +67,9 @@ func (ds DebugConfig) getConfigDump(uuid string, w http.ResponseWriter) {
6767
}
6868
return
6969
}
70-
pod, err := envoy.GetPodFromCertificate(proxy.GetCertificateCommonName(), ds.kubeController)
70+
pod, err := ds.kubeController.GetPodForProxy(proxy)
7171
if err != nil {
72-
msg := fmt.Sprintf("Error getting Pod from certificate with CN=%s", proxy.GetCertificateCommonName())
72+
msg := fmt.Sprintf("Error getting Pod from proxy %s", proxy.GetName())
7373
log.Error().Err(err).Msg(msg)
7474
if _, err := w.Write([]byte(msg)); err != nil {
7575
log.Error().Err(err).Msg("Error writing debugger response")
@@ -91,9 +91,9 @@ func (ds DebugConfig) getProxy(uuid string, w http.ResponseWriter) {
9191
}
9292
return
9393
}
94-
pod, err := envoy.GetPodFromCertificate(proxy.GetCertificateCommonName(), ds.kubeController)
94+
pod, err := ds.kubeController.GetPodForProxy(proxy)
9595
if err != nil {
96-
msg := fmt.Sprintf("Error getting Pod from certificate with CN=%s", proxy.GetCertificateCommonName())
96+
msg := fmt.Sprintf("Error getting Pod from proxy %s", proxy.GetName())
9797
log.Error().Err(err).Msg(msg)
9898
if _, err := w.Write([]byte(msg)); err != nil {
9999
log.Error().Err(err).Msg("Error writing debugger response")

‎pkg/envoy/ads/cache_stream.go

Lines changed: 10 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@ import (
1212
v1 "k8s.io/api/core/v1"
1313

1414
"github.com/openservicemesh/osm/pkg/announcements"
15-
"github.com/openservicemesh/osm/pkg/certificate"
1615
"github.com/openservicemesh/osm/pkg/constants"
1716
"github.com/openservicemesh/osm/pkg/envoy"
1817
"github.com/openservicemesh/osm/pkg/errcode"
18+
"github.com/openservicemesh/osm/pkg/identity"
1919
)
2020

2121
// Routine which fulfills listening to proxy broadcasts
@@ -61,9 +61,6 @@ func (s *Server) allPodUpdater() {
6161
// All verticals use the proxy structure to infer the pod later, so the actual only mandatory
6262
// data for the verticals to be functional is the common name, which links proxy <-> pod
6363
func GetProxyFromPod(pod *v1.Pod) (*envoy.Proxy, error) {
64-
var serviceAccount string
65-
var namespace string
66-
6764
uuidString, uuidFound := pod.Labels[constants.EnvoyUniqueIDLabelName]
6865
if !uuidFound {
6966
return nil, errors.Errorf("UUID not found for pod %s/%s, not a mesh pod", pod.Namespace, pod.Name)
@@ -73,27 +70,18 @@ func GetProxyFromPod(pod *v1.Pod) (*envoy.Proxy, error) {
7370
return nil, errors.Errorf("Could not parse UUID label into UUID type (%s): %v", uuidString, err)
7471
}
7572

76-
serviceAccount = pod.Spec.ServiceAccountName
77-
namespace = pod.Namespace
78-
79-
// construct CN for this pod/proxy
80-
// TODO: Infer proxy type from Pod
81-
commonName := envoy.NewXDSCertCommonName(proxyUUID, envoy.KindSidecar, serviceAccount, namespace)
82-
tempProxy, err := envoy.NewProxy(certificate.CommonName(commonName), "NoSerial", &net.IPAddr{IP: net.IPv4zero})
73+
sa := pod.Spec.ServiceAccountName
74+
namespace := pod.Namespace
8375

84-
return tempProxy, err
76+
return envoy.NewProxy(envoy.KindSidecar, proxyUUID, identity.New(sa, namespace), &net.IPAddr{IP: net.IPv4zero}), nil
8577
}
8678

8779
// RecordFullSnapshot stores a group of resources as a new Snapshot with a new version in the cache.
8880
// It also runs a consistency check on the snapshot (will warn if there are missing resources referenced in
8981
// the snapshot)
9082
func (s *Server) RecordFullSnapshot(proxy *envoy.Proxy, snapshotResources map[string][]types.Resource) error {
91-
s.configVerMutex.Lock()
92-
s.configVersion[proxy.GetCertificateCommonName().String()]++
93-
s.configVerMutex.Unlock()
94-
9583
snapshot, err := cache.NewSnapshot(
96-
fmt.Sprintf("%d", s.configVersion[proxy.GetCertificateCommonName().String()]),
84+
fmt.Sprintf("%d", s.configVersion[proxy.UUID.String()]),
9785
snapshotResources,
9886
)
9987
if err != nil {
@@ -104,5 +92,9 @@ func (s *Server) RecordFullSnapshot(proxy *envoy.Proxy, snapshotResources map[st
10492
log.Warn().Err(err).Str("proxy", proxy.String()).Msgf("Snapshot for proxy not consistent")
10593
}
10694

107-
return s.ch.SetSnapshot(context.TODO(), proxy.GetCertificateCommonName().String(), snapshot)
95+
s.configVerMutex.Lock()
96+
defer s.configVerMutex.Unlock()
97+
s.configVersion[proxy.UUID.String()]++
98+
99+
return s.ch.SetSnapshot(context.TODO(), proxy.UUID.String(), snapshot)
108100
}

‎pkg/envoy/ads/cache_test.go

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,18 @@ import (
88
v1 "k8s.io/api/core/v1"
99
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1010

11-
"github.com/openservicemesh/osm/pkg/certificate"
1211
"github.com/openservicemesh/osm/pkg/constants"
13-
"github.com/openservicemesh/osm/pkg/envoy"
1412
)
1513

1614
func TestGetProxyFromPod(t *testing.T) {
1715
assert := tassert.New(t)
1816

1917
var (
2018
// Default fixtures for various test variables
21-
podName = "pod"
22-
namespace = "namespace"
23-
serviceAccount = "serviceAccount"
24-
validUUID = uuid.New()
25-
validCommonName = envoy.NewXDSCertCommonName(validUUID, envoy.KindSidecar, serviceAccount, namespace)
19+
podName = "pod"
20+
namespace = "namespace"
21+
serviceAccount = "serviceAccount"
22+
validUUID = uuid.New()
2623
)
2724

2825
testCases := []struct {
@@ -32,8 +29,7 @@ func TestGetProxyFromPod(t *testing.T) {
3229
pod *v1.Pod
3330

3431
// Output check
35-
expectErr bool
36-
commonName certificate.CommonName
32+
expectErr bool
3733
}{
3834
{
3935
testCaseName: "Pod with no label",
@@ -80,7 +76,6 @@ func TestGetProxyFromPod(t *testing.T) {
8076
ServiceAccountName: serviceAccount,
8177
},
8278
},
83-
commonName: validCommonName,
8479
},
8580
}
8681

@@ -90,8 +85,8 @@ func TestGetProxyFromPod(t *testing.T) {
9085
if testCase.expectErr {
9186
assert.Error(err)
9287
} else {
93-
assert.Equal(proxyResult.GetCertificateCommonName(), testCase.commonName,
94-
"%s: Did not return equal common name")
88+
assert.NotNil(proxyResult)
89+
assert.Equal(proxyResult.UUID, validUUID)
9590
}
9691
}
9792
}

‎pkg/envoy/ads/errors.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@ var errGrpcClosed = errors.New("grpc closed")
1010
var errTooManyConnections = errors.New("too many connections")
1111
var errServiceAccountMismatch = errors.New("service account mismatch in nodeid vs xds certificate common name")
1212
var errUnsuportedXDSRequest = errors.New("Unsupported XDS server connection type")
13+
var errInvalidCertificateCN = errors.New("invalid cn")

‎pkg/envoy/ads/jobs.go

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,5 @@ func (proxyJob *proxyResponseJob) Run() {
3737

3838
// JobName implementation for this job, for logging purposes
3939
func (proxyJob *proxyResponseJob) JobName() string {
40-
return fmt.Sprintf("sendJob-%s", proxyJob.proxy.GetCertificateSerialNumber())
41-
}
42-
43-
// Hash implementation for this job to hash into the worker queues
44-
func (proxyJob *proxyResponseJob) Hash() uint64 {
45-
// Uses proxy hash to always serialize work for the same proxy to the same worker,
46-
// this avoid out-of-order mishandling of envoy updates by multiple workers
47-
return proxyJob.proxy.GetHash()
40+
return fmt.Sprintf("sendJob-%s", proxyJob.proxy.GetName())
4841
}

‎pkg/envoy/ads/profile_test.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package ads
22

33
import (
4-
"fmt"
54
"testing"
65

76
xds_cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
@@ -10,15 +9,14 @@ import (
109
"github.com/google/uuid"
1110
tassert "github.com/stretchr/testify/assert"
1211

13-
"github.com/openservicemesh/osm/pkg/certificate"
1412
"github.com/openservicemesh/osm/pkg/envoy"
13+
"github.com/openservicemesh/osm/pkg/identity"
1514
"github.com/openservicemesh/osm/pkg/tests"
1615
)
1716

1817
func TestValidateResourcesRequestResponse(t *testing.T) {
1918
assert := tassert.New(t)
20-
proxy, err := envoy.NewProxy(certificate.CommonName(fmt.Sprintf("%s.sidecar.foo.bar", uuid.New())), certificate.SerialNumber("123"), tests.NewMockAddress("1.2.3.4"))
21-
assert.Nil(err)
19+
proxy := envoy.NewProxy(envoy.KindSidecar, uuid.New(), identity.New("foo", "bar"), tests.NewMockAddress("1.2.3.4"))
2220

2321
testCases := []struct {
2422
request *xds_discovery.DiscoveryRequest

‎pkg/envoy/ads/response.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ func (s *Server) SendDiscoveryResponse(proxy *envoy.Proxy, request *xds_discover
132132
proto, err := anypb.New(res.(proto.Message))
133133
if err != nil {
134134
log.Error().Err(err).Str(errcode.Kind, errcode.GetErrCodeWithMetric(errcode.ErrMarshallingXDSResource)).
135-
Msgf("Error marshalling resource %s for proxy %s", typeURI, proxy.GetCertificateSerialNumber())
135+
Msgf("Error marshalling resource %s for proxy %s", typeURI, proxy.GetName())
136136
continue
137137
}
138138
// Add resource to response
@@ -161,15 +161,15 @@ func (s *Server) SendDiscoveryResponse(proxy *envoy.Proxy, request *xds_discover
161161

162162
// Send the response
163163
if err := (*server).Send(response); err != nil {
164-
metricsstore.DefaultMetricsStore.ProxyResponseSendErrorCount.WithLabelValues(proxy.GetCertificateCommonName().String(), string(typeURI)).Inc()
164+
metricsstore.DefaultMetricsStore.ProxyResponseSendErrorCount.WithLabelValues(proxy.GetName(), string(typeURI)).Inc()
165165
log.Error().Err(err).Str(errcode.Kind, errcode.GetErrCodeWithMetric(errcode.ErrSendingDiscoveryResponse)).
166166
Str("proxy", proxy.String()).Msgf("Error sending response for typeURI %s to proxy", typeURI.Short())
167167
return err
168168
}
169169

170170
// Sending discovery response succeeded, record last resources sent
171171
proxy.SetLastResourcesSent(typeURI, resourcesSent)
172-
metricsstore.DefaultMetricsStore.ProxyResponseSendSuccessCount.WithLabelValues(proxy.GetCertificateCommonName().String(), string(typeURI)).Inc()
172+
metricsstore.DefaultMetricsStore.ProxyResponseSendSuccessCount.WithLabelValues(proxy.GetName(), string(typeURI)).Inc()
173173

174174
return nil
175175
}

‎pkg/envoy/ads/response_test.go

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -87,21 +87,17 @@ var _ = Describe("Test ADS response functions", func() {
8787
GinkgoT().Fatalf("Error creating new Bookstire Apex service: %s", err.Error())
8888
}
8989

90-
certCommonName := envoy.NewXDSCertCommonName(proxyUUID, envoy.KindSidecar, proxySvcAccount.Name, proxySvcAccount.Namespace)
91-
certSerialNumber := certificate.SerialNumber("123456")
92-
proxy, err := envoy.NewProxy(certCommonName, certSerialNumber, nil)
90+
proxy := envoy.NewProxy(envoy.KindSidecar, proxyUUID, proxySvcAccount.ToServiceIdentity(), nil)
9391

9492
Context("Proxy is valid", func() {
9593
Expect(proxy).ToNot((BeNil()))
96-
Expect(err).ToNot(HaveOccurred())
9794
})
9895

9996
Context("Test sendAllResponses()", func() {
10097

10198
certManager := tresorFake.NewFake(nil)
102-
certCommonName := certificate.CommonName(fmt.Sprintf("%s.%s.cluster.local", proxySvcAccount.Name, proxySvcAccount.Namespace))
10399
certDuration := 1 * time.Hour
104-
certPEM, _ := certManager.IssueCertificate(certCommonName, certDuration)
100+
certPEM, _ := certManager.IssueCertificate(certificate.CommonName(proxySvcAccount.ToServiceIdentity().String()), certDuration)
105101
cert, _ := certificate.DecodePEMCertificate(certPEM.GetCertificateChain())
106102
server, actualResponses := tests.NewFakeXDSServer(cert, nil, nil)
107103
kubectrlMock := k8s.NewMockController(mockCtrl)
@@ -118,6 +114,8 @@ var _ = Describe("Test ADS response functions", func() {
118114
}).AnyTimes()
119115
mockConfigurator.EXPECT().GetMeshConfig().AnyTimes()
120116

117+
mc.GetKubeController().(*k8s.MockController).EXPECT().GetPodForProxy(proxy).Return(&pod, nil).AnyTimes()
118+
121119
metricsstore.DefaultMetricsStore.Start(metricsstore.DefaultMetricsStore.ProxyResponseSendSuccessCount)
122120

123121
It("returns Aggregated Discovery Service response", func() {
@@ -174,7 +172,7 @@ var _ = Describe("Test ADS response functions", func() {
174172
CertType: secrets.ServiceCertType,
175173
}.String()))
176174

177-
Expect(metricsstore.DefaultMetricsStore.Contains(fmt.Sprintf("osm_proxy_response_send_success_count{common_name=%q,type=%q} 1\n", proxy.GetCertificateCommonName(), envoy.TypeCDS))).To(BeTrue())
175+
Expect(metricsstore.DefaultMetricsStore.Contains(fmt.Sprintf("osm_proxy_response_send_success_count{proxy_name=%q,type=%q} 1\n", proxy.GetName(), envoy.TypeCDS))).To(BeTrue())
178176
})
179177
})
180178

‎pkg/envoy/ads/stream.go

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
mapset "github.com/deckarep/golang-set"
1010
xds_discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
11+
"github.com/google/uuid"
1112
"github.com/pkg/errors"
1213

1314
"github.com/openservicemesh/osm/pkg/announcements"
@@ -33,24 +34,23 @@ func (s *Server) StreamAggregatedResources(server xds_discovery.AggregatedDiscov
3334
}
3435

3536
// If maxDataPlaneConnections is enabled i.e. not 0, then check that the number of Envoy connections is less than maxDataPlaneConnections
36-
if s.cfg.GetMaxDataPlaneConnections() > 0 && s.proxyRegistry.GetConnectedProxyCount() >= s.cfg.GetMaxDataPlaneConnections() {
37+
if s.cfg.GetMaxDataPlaneConnections() != 0 && s.proxyRegistry.GetConnectedProxyCount() >= s.cfg.GetMaxDataPlaneConnections() {
3738
metricsstore.DefaultMetricsStore.ProxyMaxConnectionsRejected.Inc()
3839
return errTooManyConnections
3940
}
4041

4142
log.Trace().Msgf("Envoy with certificate SerialNumber=%s connected", certSerialNumber)
4243
metricsstore.DefaultMetricsStore.ProxyConnectCount.Inc()
4344

45+
kind, uuid, si, err := getCertificateCommonNameMeta(certCommonName)
46+
if err != nil {
47+
return fmt.Errorf("error parsing certificate common name %s: %w", certCommonName, err)
48+
}
49+
4450
// This is the Envoy proxy that just connected to the control plane.
4551
// NOTE: This is step 1 of the registration. At this point we do not yet have context on the Pod.
4652
// Details on which Pod this Envoy is fronting will arrive via xDS in the NODE_ID string.
47-
// When this arrives we will call RegisterProxy() a second time - this time with Pod context!
48-
proxy, err := envoy.NewProxy(certCommonName, certSerialNumber, utils.GetIPFromContext(server.Context()))
49-
if err != nil {
50-
log.Error().Err(err).Str(errcode.Kind, errcode.GetErrCodeWithMetric(errcode.ErrInitializingProxy)).
51-
Msgf("Error initializing proxy with certificate SerialNumber=%s", certSerialNumber)
52-
return err
53-
}
53+
proxy := envoy.NewProxy(kind, uuid, si, utils.GetIPFromContext(server.Context()))
5454

5555
if err := s.recordPodMetadata(proxy); err == errServiceAccountMismatch {
5656
// Service Account mismatch
@@ -332,6 +332,22 @@ func isCNforProxy(proxy *envoy.Proxy, cn certificate.CommonName) bool {
332332
return identityForCN == proxy.Identity.ToK8sServiceAccount()
333333
}
334334

335+
func getCertificateCommonNameMeta(cn certificate.CommonName) (envoy.ProxyKind, uuid.UUID, identity.ServiceIdentity, error) {
336+
// XDS cert CN is of the form <proxy-UUID>.<kind>.<proxy-identity>.<namespace>.<trust-domain>
337+
chunks := strings.SplitN(cn.String(), constants.DomainDelimiter, 5)
338+
if len(chunks) < 3 {
339+
return "", uuid.UUID{}, "", errInvalidCertificateCN
340+
}
341+
proxyUUID, err := uuid.Parse(chunks[0])
342+
if err != nil {
343+
log.Error().Err(err).Str(errcode.Kind, errcode.GetErrCodeWithMetric(errcode.ErrParsingXDSCertCN)).
344+
Msgf("Error parsing %s into uuid.UUID", chunks[0])
345+
return "", uuid.UUID{}, "", err
346+
}
347+
348+
return envoy.ProxyKind(chunks[1]), proxyUUID, identity.New(chunks[2], chunks[3]), nil
349+
}
350+
335351
// recordPodMetadata records pod metadata and verifies the certificate issued for this pod
336352
// is for the same service account as seen on the pod's service account
337353
func (s *Server) recordPodMetadata(p *envoy.Proxy) error {
@@ -341,7 +357,7 @@ func (s *Server) recordPodMetadata(p *envoy.Proxy) error {
341357
return nil
342358
}
343359

344-
pod, err := envoy.GetPodFromCertificate(p.GetCertificateCommonName(), s.kubecontroller)
360+
pod, err := s.kubecontroller.GetPodForProxy(p)
345361
if err != nil {
346362
log.Warn().Str("proxy", p.String()).Msg("Could not find pod for connecting proxy. No metadata was recorded.")
347363
return nil

0 commit comments

Comments
 (0)