Skip to content
This repository was archived by the owner on Jul 11, 2023. It is now read-only.

Commit 4a3d57d

Browse files
keithmattixkeithmattix
authored
Fix flaky e2e tests (#4844)
* Add a GinkgoRecover to start debugging flaky tests Signed-off-by: Keith Mattix II <[email protected]> * Trigger CI Signed-off-by: Keith Mattix II <[email protected]> * Add another ginko recover Signed-off-by: Keith Mattix II <[email protected]> * Add locks that are missing Signed-off-by: Keith Mattix II <[email protected]> * Re-run CI Signed-off-by: Keith Mattix II <[email protected]> * Re-run CI Signed-off-by: Keith Mattix II <[email protected]> * Fix test flakiness by only reacting to MeshConfigs and MRCs in target namespaces Signed-off-by: Keith Mattix II <[email protected]> * More elegant solution for namespacing Signed-off-by: Keith Mattix II <[email protected]> * VSCode didn't save my file...thanks Go Language servier Signed-off-by: Keith Mattix II <[email protected]> * Trigger CI Signed-off-by: Keith Mattix II <[email protected]> * Trigger CI Signed-off-by: Keith Mattix II <[email protected]> * Address PR comments Signed-off-by: Keith Mattix II <[email protected]> * Add more info to logs Signed-off-by: Keith Mattix II <[email protected]>
1 parent f3966a3 commit 4a3d57d

File tree

13 files changed

+32
-15
lines changed

13 files changed

+32
-15
lines changed

‎cmd/osm-bootstrap/osm-bootstrap.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ func main() {
198198

199199
informerCollection, err := informers.NewInformerCollection(meshName, stop,
200200
informers.WithKubeClient(kubeClient),
201-
informers.WithConfigClient(configClient),
201+
informers.WithConfigClient(configClient, osmMeshConfigName, osmNamespace),
202202
)
203203

204204
if err != nil {

‎cmd/osm-controller/osm-controller.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ func main() {
187187
informerCollection, err := informers.NewInformerCollection(meshName, stop,
188188
informers.WithKubeClient(kubeClient),
189189
informers.WithSMIClients(smiTrafficSplitClientSet, smiTrafficSpecClientSet, smiTrafficTargetClientSet),
190-
informers.WithConfigClient(configClient),
190+
informers.WithConfigClient(configClient, osmMeshConfigName, osmNamespace),
191191
informers.WithPolicyClient(policyClient),
192192
)
193193
if err != nil {

‎cmd/osm-injector/osm-injector.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ func main() {
183183
informerCollection, err := informers.NewInformerCollection(meshName, stop,
184184
informers.WithKubeClient(kubeClient),
185185
informers.WithSMIClients(smiTrafficSplitClientSet, smiTrafficSpecClientSet, smiTrafficTargetClientSet),
186-
informers.WithConfigClient(configClient),
186+
informers.WithConfigClient(configClient, osmMeshConfigName, osmNamespace),
187187
informers.WithPolicyClient(policyClient),
188188
)
189189

‎pkg/catalog/fake/fake.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ func NewFakeMeshCatalog(kubeClient kubernetes.Interface, meshConfigClient config
4848

4949
osmNamespace := "-test-osm-namespace-"
5050
osmMeshConfigName := "-test-osm-mesh-config-"
51-
ic, err := informers.NewInformerCollection("osm", stop, informers.WithKubeClient(kubeClient), informers.WithConfigClient(meshConfigClient))
51+
ic, err := informers.NewInformerCollection("osm", stop, informers.WithKubeClient(kubeClient), informers.WithConfigClient(meshConfigClient, osmMeshConfigName, osmNamespace))
5252
if err != nil {
5353
return nil
5454
}

‎pkg/certificate/manager.go

+8
Original file line numberDiff line numberDiff line change
@@ -129,15 +129,23 @@ func (m *Manager) handleMRCEvent(mrcClient MRCClient, event MRCEvent) error {
129129
c := &issuer{Issuer: client, ID: clientID, CertificateAuthority: ca}
130130
switch {
131131
case mrc.Status.State == constants.MRCStateActive:
132+
m.mu.Lock()
132133
m.signingIssuer = c
133134
m.validatingIssuer = c
135+
m.mu.Unlock()
134136
case mrc.Status.State == constants.MRCStateIssuingRollback || mrc.Status.State == constants.MRCStateIssuingRollout:
137+
m.mu.Lock()
135138
m.signingIssuer = c
139+
m.mu.Unlock()
136140
case mrc.Status.State == constants.MRCStateValidatingRollback || mrc.Status.State == constants.MRCStateValidatingRollout:
141+
m.mu.Lock()
137142
m.validatingIssuer = c
143+
m.mu.Unlock()
138144
default:
145+
m.mu.Lock()
139146
m.signingIssuer = c
140147
m.validatingIssuer = c
148+
m.mu.Unlock()
141149
}
142150
case MRCEventUpdated:
143151
// TODO

‎pkg/certificate/providers/config_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ func TestGetCertificateManager(t *testing.T) {
139139
},
140140
}),
141141
informerCollectionFunc: func(tc testCase) (*informers.InformerCollection, error) {
142-
ic, err := informers.NewInformerCollection("osm", nil, informers.WithKubeClient(tc.kubeClient), informers.WithConfigClient(tc.configClient))
142+
ic, err := informers.NewInformerCollection("osm", nil, informers.WithKubeClient(tc.kubeClient), informers.WithConfigClient(tc.configClient, "", "osm-namespace"))
143143
if err != nil {
144144
return nil, err
145145
}

‎pkg/certificate/providers/mrc.go

+3
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package providers
33
import (
44
"context"
55

6+
"github.com/rs/zerolog/log"
67
"k8s.io/client-go/tools/cache"
78

89
"github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
@@ -46,6 +47,7 @@ func (m *MRCComposer) Watch(ctx context.Context) (<-chan certificate.MRCEvent, e
4647
m.informerCollection.AddEventHandler(informers.InformerKeyMeshRootCertificate, cache.ResourceEventHandlerFuncs{
4748
AddFunc: func(obj interface{}) {
4849
mrc := obj.(*v1alpha2.MeshRootCertificate)
50+
log.Debug().Msgf("received MRC add event for MRC %s/%s", mrc.GetNamespace(), mrc.GetName())
4951
eventChan <- certificate.MRCEvent{
5052
Type: certificate.MRCEventAdded,
5153
MRC: mrc,
@@ -55,6 +57,7 @@ func (m *MRCComposer) Watch(ctx context.Context) (<-chan certificate.MRCEvent, e
5557
// since the "state machine" of the MRC is well defined
5658
UpdateFunc: func(_, newObj interface{}) {
5759
mrc := newObj.(*v1alpha2.MeshRootCertificate)
60+
log.Debug().Msgf("received MRC update event for MRC %s/%s", mrc.GetNamespace(), mrc.GetName())
5861
eventChan <- certificate.MRCEvent{
5962
Type: certificate.MRCEventUpdated,
6063
MRC: mrc,

‎pkg/certificate/types.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ var (
147147
// MRCEventBroker describes any type that allows the caller to Watch() MRCEvents
148148
type MRCEventBroker interface {
149149
// Watch allows the caller to subscribe to events surrounding
150-
// MRCs that belong to this particular mesh. Watch returns a channel
151-
// that emits events, and an error if the subscription goes awry.
150+
// MRCs. Watch returns a channel that emits events, and
151+
// an error if the subscription goes awry.
152152
Watch(context.Context) (<-chan MRCEvent, error)
153153
}

‎pkg/configurator/client_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ func TestGetMeshConfig(t *testing.T) {
2424
meshConfigClient := fakeConfig.NewSimpleClientset()
2525
stop := make(chan struct{})
2626

27-
ic, err := informers.NewInformerCollection("osm", stop, informers.WithConfigClient(meshConfigClient))
27+
ic, err := informers.NewInformerCollection("osm", stop, informers.WithConfigClient(meshConfigClient, osmMeshConfigName, osmNamespace))
2828
a.Nil(err)
2929

3030
c := NewConfigurator(ic, osmNamespace, osmMeshConfigName, nil)

‎pkg/configurator/methods_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ func TestCreateUpdateConfig(t *testing.T) {
3131
meshConfigClientSet := testclient.NewSimpleClientset()
3232
stop := make(chan struct{})
3333

34-
ic, err := informers.NewInformerCollection("osm", stop, informers.WithConfigClient(meshConfigClientSet))
34+
ic, err := informers.NewInformerCollection("osm", stop, informers.WithConfigClient(meshConfigClientSet, osmMeshConfigName, osmNamespace))
3535
tassert.Nil(t, err)
3636

3737
cfg := NewConfigurator(ic, osmNamespace, osmMeshConfigName, nil)
@@ -465,7 +465,7 @@ func TestCreateUpdateConfig(t *testing.T) {
465465
stop := make(chan struct{})
466466
defer close(stop)
467467

468-
ic, err := informers.NewInformerCollection("osm", stop, informers.WithConfigClient(meshConfigClientSet))
468+
ic, err := informers.NewInformerCollection("osm", stop, informers.WithConfigClient(meshConfigClientSet, osmMeshConfigName, osmNamespace))
469469
assert.Nil(err)
470470

471471
cfg := NewConfigurator(ic, osmNamespace, osmMeshConfigName, nil)

‎pkg/envoy/sds/response_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ func TestNewResponse(t *testing.T) {
5858
podID := uuid.New()
5959

6060
proxy := envoy.NewProxy(envoy.KindSidecar, podID, proxySvcAccount.ToServiceIdentity(), nil)
61-
ic, err := informers.NewInformerCollection("osm", stop, informers.WithKubeClient(fakeKubeClient), informers.WithConfigClient(fakeConfigClient))
61+
ic, err := informers.NewInformerCollection("osm", stop, informers.WithKubeClient(fakeKubeClient), informers.WithConfigClient(fakeConfigClient, "-the-mesh-config-name-", "-osm-namespace-"))
6262
assert.Nil(err)
6363

6464
cfg := configurator.NewConfigurator(ic, "-osm-namespace-", "-the-mesh-config-name-", nil)

‎pkg/k8s/informers/informers.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -86,12 +86,16 @@ func WithSMIClients(smiTrafficSplitClient smiTrafficSplitClient.Interface, smiTr
8686
}
8787

8888
// WithConfigClient sets the config client for the InformerCollection
89-
func WithConfigClient(configClient configClientset.Interface) InformerCollectionOption {
89+
func WithConfigClient(configClient configClientset.Interface, meshConfigName, osmNamespace string) InformerCollectionOption {
9090
return func(ic *InformerCollection) {
91-
informerFactory := configInformers.NewSharedInformerFactory(configClient, DefaultKubeEventResyncInterval)
91+
listOption := configInformers.WithTweakListOptions(func(opt *metav1.ListOptions) {
92+
opt.FieldSelector = fields.OneTermEqualSelector(metav1.ObjectNameField, meshConfigName).String()
93+
})
94+
meshConfiginformerFactory := configInformers.NewSharedInformerFactoryWithOptions(configClient, DefaultKubeEventResyncInterval, configInformers.WithNamespace(osmNamespace), listOption)
95+
mrcInformerFactory := configInformers.NewSharedInformerFactoryWithOptions(configClient, DefaultKubeEventResyncInterval, configInformers.WithNamespace(osmNamespace))
9296

93-
ic.informers[InformerKeyMeshConfig] = informerFactory.Config().V1alpha2().MeshConfigs().Informer()
94-
ic.informers[InformerKeyMeshRootCertificate] = informerFactory.Config().V1alpha2().MeshRootCertificates().Informer()
97+
ic.informers[InformerKeyMeshConfig] = meshConfiginformerFactory.Config().V1alpha2().MeshConfigs().Informer()
98+
ic.informers[InformerKeyMeshRootCertificate] = mrcInformerFactory.Config().V1alpha2().MeshRootCertificates().Informer()
9599
}
96100
}
97101

‎tests/e2e/e2e_retry_policy_test.go

+2
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ var _ = OSMDescribe("Test Retry Policy",
125125

126126
By("A request that will be retried NumRetries times then fail")
127127
err = wait.Poll(time.Second*3, time.Second*30, func() (bool, error) {
128+
defer GinkgoRecover()
128129
result := Td.HTTPRequest(req)
129130

130131
stdout, stderr, err := Td.RunLocal(filepath.FromSlash("../../bin/osm"), "proxy", "get", "stats", clientPod.Name, "--namespace", client)
@@ -240,6 +241,7 @@ var _ = OSMDescribe("Test Retry Policy",
240241

241242
By("A request that will be retried 0 times and then fail")
242243
err = wait.Poll(time.Second*3, time.Second*30, func() (bool, error) {
244+
defer GinkgoRecover()
243245
result := Td.HTTPRequest(req)
244246

245247
stdout, stderr, err := Td.RunLocal(filepath.FromSlash("../../bin/osm"), "proxy", "get", "stats", clientPod.Name, "--namespace", client)

0 commit comments

Comments
 (0)