Skip to content
This repository was archived by the owner on Jul 11, 2023. It is now read-only.

Commit 30885c9

Browse files
authored
cert: Use MRCs on startup (#4816)
By default, reads MRCs from the cluster in order to build out the certificate manager. From there, allow the certificate manager to watch for changes to the MRCs in the cluster Signed-off-by: Keith Mattix II <[email protected]>
1 parent e3700d6 commit 30885c9

28 files changed

+507
-129
lines changed

cmd/osm-bootstrap/osm-bootstrap.go

+6-7
Original file line numberDiff line numberDiff line change
@@ -182,9 +182,8 @@ func main() {
182182
log.Fatal().Err(err).Msg("Error initializing Kubernetes events recorder")
183183
}
184184

185-
stop := signals.RegisterExitHandlers()
186-
_, cancel := context.WithCancel(context.Background())
187-
defer cancel()
185+
ctx, cancel := context.WithCancel(context.Background())
186+
stop := signals.RegisterExitHandlers(cancel)
188187

189188
// Start the default metrics store
190189
metricsstore.DefaultMetricsStore.Start(
@@ -214,14 +213,11 @@ func main() {
214213
log.Fatal().Err(err).Msg("Error getting certificate options")
215214
}
216215

217-
// Intitialize certificate manager/provider
218-
certManager, err := providers.NewCertificateManager(kubeClient, kubeConfig, cfg, osmNamespace, certOpts, msgBroker)
216+
certManager, err := providers.NewCertificateManager(ctx, kubeClient, kubeConfig, cfg, osmNamespace, certOpts, msgBroker, informerCollection, 5*time.Second)
219217
if err != nil {
220218
events.GenericEventRecorder().FatalEvent(err, events.InvalidCertificateManager,
221219
"Error initializing certificate manager of kind %s", certProviderKind)
222220
}
223-
// watch for certificate rotation
224-
certManager.Start(5*time.Second, stop)
225221

226222
// Initialize the crd conversion webhook server to support the conversion of OSM's CRDs
227223
crdConverterConfig.ListenPort = constants.CRDConversionWebhookPort
@@ -427,6 +423,9 @@ func buildMeshRootCertificate(presetMeshRootCertificateConfigMap *corev1.ConfigM
427423
},
428424
ObjectMeta: metav1.ObjectMeta{
429425
Name: meshRootCertificateName,
426+
Annotations: map[string]string{
427+
constants.MRCVersionAnnotation: "0",
428+
},
430429
},
431430
Spec: presetMeshRootCertificateSpec,
432431
Status: configv1alpha2.MeshRootCertificateStatus{

cmd/osm-controller/osm-controller.go

+3-6
Original file line numberDiff line numberDiff line change
@@ -172,9 +172,8 @@ func main() {
172172
events.GenericEventRecorder().FatalEvent(err, events.InvalidCLIParameters, "Error validating CLI parameters")
173173
}
174174

175-
stop := signals.RegisterExitHandlers()
176175
ctx, cancel := context.WithCancel(context.Background())
177-
defer cancel()
176+
stop := signals.RegisterExitHandlers(cancel)
178177

179178
// Start the default metrics store
180179
startMetricsStore()
@@ -208,15 +207,13 @@ func main() {
208207
}
209208

210209
// Intitialize certificate manager/provider
211-
certManager, err := providers.NewCertificateManager(kubeClient, kubeConfig, cfg, osmNamespace,
212-
certOpts, msgBroker)
210+
certManager, err := providers.NewCertificateManager(ctx, kubeClient, kubeConfig, cfg, osmNamespace,
211+
certOpts, msgBroker, informerCollection, 5*time.Second)
213212

214213
if err != nil {
215214
events.GenericEventRecorder().FatalEvent(err, events.InvalidCertificateManager,
216215
"Error fetching certificate manager of kind %s", certProviderKind)
217216
}
218-
// watch for certificate rotation
219-
certManager.Start(5*time.Second, stop)
220217

221218
kubeProvider := kube.NewClient(k8sClient, cfg)
222219

cmd/osm-injector/osm-injector.go

+4-7
Original file line numberDiff line numberDiff line change
@@ -160,9 +160,8 @@ func main() {
160160
events.GenericEventRecorder().FatalEvent(err, events.InvalidCLIParameters, "Error validating CLI parameters")
161161
}
162162

163-
stop := signals.RegisterExitHandlers()
164-
_, cancel := context.WithCancel(context.Background())
165-
defer cancel()
163+
ctx, cancel := context.WithCancel(context.Background())
164+
stop := signals.RegisterExitHandlers(cancel)
166165

167166
// Start the default metrics store
168167
metricsstore.DefaultMetricsStore.Start(
@@ -203,14 +202,12 @@ func main() {
203202
log.Fatal().Err(err).Msg("Error getting certificate options")
204203
}
205204
// Intitialize certificate manager/provider
206-
certManager, err := providers.NewCertificateManager(kubeClient, kubeConfig, cfg, osmNamespace,
207-
certOpts, msgBroker)
205+
certManager, err := providers.NewCertificateManager(ctx, kubeClient, kubeConfig, cfg, osmNamespace,
206+
certOpts, msgBroker, informerCollection, 5*time.Second)
208207
if err != nil {
209208
events.GenericEventRecorder().FatalEvent(err, events.InvalidCertificateManager,
210209
"Error initializing certificate manager of kind %s", certProviderKind)
211210
}
212-
// watch for certificate rotation
213-
certManager.Start(5*time.Second, stop)
214211

215212
// Initialize the sidecar injector webhook
216213
if err := injector.NewMutatingWebhook(injectorConfig, kubeClient, certManager, kubeController, meshName, osmNamespace, webhookConfigName, osmVersion, webhookTimeout, enableReconciler, stop, cfg, corev1.PullPolicy(osmContainerPullPolicy)); err != nil {

pkg/catalog/fake/fake.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package fake
22

33
import (
44
"context"
5+
"time"
56

67
"github.com/golang/mock/gomock"
78
"github.com/onsi/ginkgo"
@@ -54,7 +55,7 @@ func NewFakeMeshCatalog(kubeClient kubernetes.Interface, meshConfigClient config
5455

5556
cfg := configurator.NewConfigurator(ic, osmNamespace, osmMeshConfigName, nil)
5657

57-
certManager := tresorFake.NewFake(nil)
58+
certManager := tresorFake.NewFake(nil, 1*time.Hour)
5859

5960
// #1683 tracks potential improvements to the following dynamic mocks
6061
mockKubeController.EXPECT().ListServices().DoAndReturn(func() []*corev1.Service {

pkg/catalog/helpers_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ func newFakeMeshCatalogForRoutes(t *testing.T, testParams testParams) *MeshCatal
5050

5151
stop := make(chan struct{})
5252

53-
certManager := tresorFake.NewFake(nil)
53+
certManager := tresorFake.NewFake(nil, 1*time.Hour)
5454

5555
// Create a bookstoreV1 pod
5656
bookstoreV1Pod := tests.NewPodFixture(tests.BookstoreV1Service.Namespace, tests.BookstoreV1Service.Name, tests.BookstoreServiceAccountName, tests.PodLabels)

pkg/certificate/fake_manager.go

+45-1
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,17 @@
11
package certificate
22

33
import (
4+
"context"
45
"fmt"
5-
time "time"
6+
"time"
7+
8+
v1 "k8s.io/api/core/v1"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
610

711
"github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
12+
813
"github.com/openservicemesh/osm/pkg/certificate/pem"
14+
"github.com/openservicemesh/osm/pkg/constants"
915
)
1016

1117
var (
@@ -28,6 +34,42 @@ func (c *fakeMRCClient) List() ([]*v1alpha2.MeshRootCertificate, error) {
2834
}}, nil
2935
}
3036

37+
func (c *fakeMRCClient) Watch(ctx context.Context) (<-chan MRCEvent, error) {
38+
ch := make(chan MRCEvent)
39+
go func() {
40+
ch <- MRCEvent{
41+
Type: MRCEventAdded,
42+
MRC: &v1alpha2.MeshRootCertificate{
43+
ObjectMeta: metav1.ObjectMeta{
44+
Name: "osm-mesh-root-certificate",
45+
Namespace: "osm-system",
46+
Annotations: map[string]string{
47+
constants.MRCVersionAnnotation: "0",
48+
},
49+
},
50+
Spec: v1alpha2.MeshRootCertificateSpec{
51+
Provider: v1alpha2.ProviderSpec{
52+
Tresor: &v1alpha2.TresorProviderSpec{
53+
CA: v1alpha2.TresorCASpec{
54+
SecretRef: v1.SecretReference{
55+
Name: "osm-ca-bundle",
56+
Namespace: "osm-system",
57+
},
58+
},
59+
},
60+
},
61+
},
62+
Status: v1alpha2.MeshRootCertificateStatus{
63+
State: constants.MRCStateActive,
64+
},
65+
},
66+
}
67+
close(ch)
68+
}()
69+
70+
return ch, nil
71+
}
72+
3173
type fakeIssuer struct {
3274
err bool
3375
id string
@@ -51,9 +93,11 @@ func (i *fakeIssuer) IssueCertificate(cn CommonName, validityPeriod time.Duratio
5193
// FakeCertManager is a testing helper that returns a *certificate.Manager
5294
func FakeCertManager() (*Manager, error) {
5395
cm, err := NewManager(
96+
context.Background(),
5497
&fakeMRCClient{},
5598
validity,
5699
nil,
100+
1*time.Hour,
57101
)
58102
if err != nil {
59103
return nil, fmt.Errorf("error creating fakeCertManager, err: %w", err)

pkg/certificate/manager.go

+110-22
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,43 @@
11
package certificate
22

33
import (
4+
"context"
5+
"errors"
6+
"sync"
47
"time"
58

69
"github.com/rs/zerolog/log"
710

811
"github.com/openservicemesh/osm/pkg/announcements"
12+
"github.com/openservicemesh/osm/pkg/constants"
913
"github.com/openservicemesh/osm/pkg/errcode"
1014
"github.com/openservicemesh/osm/pkg/k8s/events"
1115
"github.com/openservicemesh/osm/pkg/messaging"
1216
)
1317

14-
// NewManager creates a new CertManager with the passed CA and CA Private Key
15-
func NewManager(mrcClient MRCClient, serviceCertValidityDuration time.Duration, msgBroker *messaging.Broker) (*Manager, error) {
16-
// TODO(#4502): transition this call to a watch function that knows how to handle multiple MRC and can react to changes.
17-
mrcs, err := mrcClient.List()
18-
if err != nil {
19-
return nil, err
18+
// NewManager creates a new CertificateManager with the passed MRCClient and options
19+
func NewManager(ctx context.Context, mrcClient MRCClient, serviceCertValidityDuration time.Duration, msgBroker *messaging.Broker, checkInterval time.Duration) (*Manager, error) {
20+
m := &Manager{
21+
serviceCertValidityDuration: serviceCertValidityDuration,
22+
msgBroker: msgBroker,
2023
}
2124

22-
client, ca, clientID, err := mrcClient.GetCertIssuerForMRC(mrcs[0])
25+
err := m.start(ctx, mrcClient)
2326
if err != nil {
2427
return nil, err
2528
}
2629

27-
c := &issuer{Issuer: client, ID: clientID, CertificateAuthority: ca, TrustDomain: mrcs[0].Spec.TrustDomain}
28-
29-
m := &Manager{
30-
// The signingIssuer is responsible for signing all newly issued certificates
31-
// The validatingIssuer is the issuer that issued existing certificates.
32-
// its underlying cert is still in the validating trust store
33-
signingIssuer: c,
34-
validatingIssuer: c,
35-
serviceCertValidityDuration: serviceCertValidityDuration,
36-
msgBroker: msgBroker,
37-
}
30+
m.startRotationTicker(ctx, checkInterval)
3831
return m, nil
3932
}
4033

41-
// Start takes an interval to check if the certificate
42-
// needs to be rotated
43-
func (m *Manager) Start(checkInterval time.Duration, stop <-chan struct{}) {
34+
func (m *Manager) startRotationTicker(ctx context.Context, checkInterval time.Duration) {
4435
ticker := time.NewTicker(checkInterval)
4536
go func() {
4637
m.checkAndRotate()
4738
for {
4839
select {
49-
case <-stop:
40+
case <-ctx.Done():
5041
ticker.Stop()
5142
return
5243
case <-ticker.C:
@@ -56,6 +47,103 @@ func (m *Manager) Start(checkInterval time.Duration, stop <-chan struct{}) {
5647
}()
5748
}
5849

50+
func (m *Manager) start(ctx context.Context, mrcClient MRCClient) error {
51+
// start a watch and we wait until the manager is initialized so that
52+
// the caller gets a manager that's ready to be used
53+
var once sync.Once
54+
var wg sync.WaitGroup
55+
mrcEvents, err := mrcClient.Watch(ctx)
56+
if err != nil {
57+
return err
58+
}
59+
60+
wg.Add(1)
61+
62+
go func(wg *sync.WaitGroup, once *sync.Once) {
63+
for {
64+
select {
65+
case <-ctx.Done():
66+
if err := ctx.Err(); err != nil {
67+
log.Error().Err(err).Msg("context canceled with error. stopping MRC watch...")
68+
return
69+
}
70+
71+
log.Info().Msg("context canceled. stopping MRC watch...")
72+
return
73+
case event, open := <-mrcEvents:
74+
if !open {
75+
// channel was closed; return
76+
log.Info().Msg("stopping MRC watch...")
77+
return
78+
}
79+
80+
err = m.handleMRCEvent(mrcClient, event)
81+
if err != nil {
82+
log.Error().Err(err).Msgf("error encountered processing MRCEvent")
83+
continue
84+
}
85+
}
86+
87+
if m.signingIssuer != nil && m.validatingIssuer != nil {
88+
once.Do(func() {
89+
wg.Done()
90+
})
91+
}
92+
}
93+
}(&wg, &once)
94+
95+
done := make(chan struct{})
96+
97+
// Wait for WaitGroup to finish and notify select when it does
98+
go func() {
99+
wg.Wait()
100+
close(done)
101+
}()
102+
103+
select {
104+
case <-time.After(10 * time.Second):
105+
// We timed out
106+
return errors.New("manager initialization timed out. Make sure your MeshRootCertificate(s) are valid")
107+
case <-done:
108+
}
109+
110+
return nil
111+
}
112+
113+
func (m *Manager) handleMRCEvent(mrcClient MRCClient, event MRCEvent) error {
114+
switch event.Type {
115+
case MRCEventAdded:
116+
mrc := event.MRC
117+
if mrc.Status.State == constants.MRCStateError {
118+
log.Debug().Msgf("skipping MRC with error state %s", mrc.GetName())
119+
return nil
120+
}
121+
122+
client, ca, clientID, err := mrcClient.GetCertIssuerForMRC(mrc)
123+
if err != nil {
124+
return err
125+
}
126+
127+
c := &issuer{Issuer: client, ID: clientID, CertificateAuthority: ca}
128+
switch {
129+
case mrc.Status.State == constants.MRCStateActive:
130+
m.signingIssuer = c
131+
m.validatingIssuer = c
132+
case mrc.Status.State == constants.MRCStateIssuingRollback || mrc.Status.State == constants.MRCStateIssuingRollout:
133+
m.signingIssuer = c
134+
case mrc.Status.State == constants.MRCStateValidatingRollback || mrc.Status.State == constants.MRCStateValidatingRollout:
135+
m.validatingIssuer = c
136+
default:
137+
m.signingIssuer = c
138+
m.validatingIssuer = c
139+
}
140+
case MRCEventUpdated:
141+
// TODO
142+
}
143+
144+
return nil
145+
}
146+
59147
// GetTrustDomain returns the trust domain from the configured signingkey issuer.
60148
// Note that the CRD uses a default, so this value will always be set.
61149
func (m *Manager) GetTrustDomain() string {

0 commit comments

Comments
 (0)