Skip to content

Commit b0ad571

Browse files
orsenthilorsenthil
authored
Ginkgo Test to exercise cross AZ testing, to be run on static canaries. (#2689)
* Test Pod Connectivity Across AZs, to exercise CNI and pod connectivity upon AZ failures. * Test API Server Connectivity from various AZs to ensure that Service Discovery (kube-proxy) and Service Name look up (Coredns) remain functional during AZ failures.
1 parent 60efe2e commit b0ad571

File tree

5 files changed

+484
-0
lines changed

5 files changed

+484
-0
lines changed

‎scripts/run-static-canary.sh

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
3+
# The script runs amazon-vpc-cni static canary tests
4+
# The tests in this suite are designed to exercise AZ failure scenarios.
5+
6+
set -e
7+
8+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
9+
GINKGO_TEST_BUILD="$SCRIPT_DIR/../test/build"
10+
# TEST_IMAGE_REGISTRY is the registry in test-infra-* accounts where e2e test images are stored
11+
TEST_IMAGE_REGISTRY=${TEST_IMAGE_REGISTRY:-"617930562442.dkr.ecr.us-west-2.amazonaws.com"}
12+
13+
source "$SCRIPT_DIR"/lib/cluster.sh
14+
source "$SCRIPT_DIR"/lib/canary.sh
15+
16+
function run_ginkgo_test() {
17+
local focus=$1
18+
echo "Running ginkgo tests with focus: $focus"
19+
20+
(CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --no-color --focus="$focus" -v --timeout 30m --fail-on-pending $GINKGO_TEST_BUILD/cni.test -- \
21+
--cluster-kubeconfig="$KUBE_CONFIG_PATH" \
22+
--cluster-name="$CLUSTER_NAME" \
23+
--aws-region="$REGION" \
24+
--aws-vpc-id="$VPC_ID" \
25+
--ng-name-label-key="kubernetes.io/os" \
26+
--ng-name-label-val="linux" \
27+
--test-image-registry=$TEST_IMAGE_REGISTRY)
28+
}
29+
30+
load_cluster_details
31+
32+
run_ginkgo_test "STATIC_CANARY"
33+
34+
echo "all tests ran successfully in $(($SECONDS / 60)) minutes and $(($SECONDS % 60)) seconds"

‎test/framework/resources/k8s/manifest/container.go

+7
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ type Container struct {
2828
probe *v1.Probe
2929
ports []v1.ContainerPort
3030
securityContext *v1.SecurityContext
31+
Env []v1.EnvVar
3132
}
3233

3334
func NewBusyBoxContainerBuilder(testImageRegistry string) *Container {
@@ -101,6 +102,11 @@ func (w *Container) Command(cmd []string) *Container {
101102
return w
102103
}
103104

105+
func (w *Container) EnvVar(env []v1.EnvVar) *Container {
106+
w.Env = env
107+
return w
108+
}
109+
104110
func (w *Container) Args(arg []string) *Container {
105111
w.args = arg
106112
return w
@@ -126,5 +132,6 @@ func (w *Container) Build() v1.Container {
126132
LivenessProbe: w.probe,
127133
Ports: w.ports,
128134
SecurityContext: w.securityContext,
135+
Env: w.Env,
129136
}
130137
}

‎test/framework/resources/k8s/manifest/daemonset.go

+123
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
// not use this file except in compliance with the License. A copy of the
5+
// License is located at
6+
//
7+
// http://aws.amazon.com/apache2.0/
8+
//
9+
// or in the "license" file accompanying this file. This file is distributed
10+
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
// express or implied. See the License for the specific language governing
12+
// permissions and limitations under the License.
13+
14+
package manifest
15+
16+
import (
17+
"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
18+
"github.com/aws/aws-sdk-go/aws"
19+
v1 "k8s.io/api/apps/v1"
20+
corev1 "k8s.io/api/core/v1"
21+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
22+
)
23+
24+
type DaemonsetBuilder struct {
25+
namespace string
26+
name string
27+
container corev1.Container
28+
labels map[string]string
29+
nodeSelector map[string]string
30+
terminationGracePeriod int
31+
hostNetwork bool
32+
volume []corev1.Volume
33+
volumeMount []corev1.VolumeMount
34+
}
35+
36+
func NewDefaultDaemonsetBuilder() *DaemonsetBuilder {
37+
return &DaemonsetBuilder{
38+
namespace: utils.DefaultTestNamespace,
39+
terminationGracePeriod: 1,
40+
labels: map[string]string{"role": "test"},
41+
nodeSelector: map[string]string{"kubernetes.io/os": "linux"},
42+
}
43+
}
44+
45+
func (d *DaemonsetBuilder) Labels(labels map[string]string) *DaemonsetBuilder {
46+
d.labels = labels
47+
return d
48+
}
49+
50+
func (d *DaemonsetBuilder) NodeSelector(labelKey string, labelVal string) *DaemonsetBuilder {
51+
if labelKey != "" {
52+
d.nodeSelector[labelKey] = labelVal
53+
}
54+
return d
55+
}
56+
57+
func (d *DaemonsetBuilder) Namespace(namespace string) *DaemonsetBuilder {
58+
d.namespace = namespace
59+
return d
60+
}
61+
62+
func (d *DaemonsetBuilder) TerminationGracePeriod(tg int) *DaemonsetBuilder {
63+
d.terminationGracePeriod = tg
64+
return d
65+
}
66+
67+
func (d *DaemonsetBuilder) Name(name string) *DaemonsetBuilder {
68+
d.name = name
69+
return d
70+
}
71+
72+
func (d *DaemonsetBuilder) Container(container corev1.Container) *DaemonsetBuilder {
73+
d.container = container
74+
return d
75+
}
76+
77+
func (d *DaemonsetBuilder) PodLabel(labelKey string, labelValue string) *DaemonsetBuilder {
78+
d.labels[labelKey] = labelValue
79+
return d
80+
}
81+
82+
func (d *DaemonsetBuilder) HostNetwork(hostNetwork bool) *DaemonsetBuilder {
83+
d.hostNetwork = hostNetwork
84+
return d
85+
}
86+
87+
func (d *DaemonsetBuilder) MountVolume(volume []corev1.Volume, volumeMount []corev1.VolumeMount) *DaemonsetBuilder {
88+
d.volume = volume
89+
d.volumeMount = volumeMount
90+
return d
91+
}
92+
93+
func (d *DaemonsetBuilder) Build() *v1.DaemonSet {
94+
deploymentSpec := &v1.DaemonSet{
95+
ObjectMeta: metav1.ObjectMeta{
96+
Name: d.name,
97+
Namespace: d.namespace,
98+
Labels: d.labels,
99+
},
100+
Spec: v1.DaemonSetSpec{
101+
Selector: &metav1.LabelSelector{
102+
MatchLabels: d.labels,
103+
},
104+
Template: corev1.PodTemplateSpec{
105+
ObjectMeta: metav1.ObjectMeta{
106+
Labels: d.labels,
107+
},
108+
Spec: corev1.PodSpec{
109+
HostNetwork: d.hostNetwork,
110+
NodeSelector: d.nodeSelector,
111+
Containers: []corev1.Container{d.container},
112+
TerminationGracePeriodSeconds: aws.Int64(int64(d.terminationGracePeriod)),
113+
},
114+
},
115+
},
116+
}
117+
118+
if len(d.volume) > 0 && len(d.volumeMount) > 0 {
119+
deploymentSpec.Spec.Template.Spec.Volumes = d.volume
120+
deploymentSpec.Spec.Template.Spec.Containers[0].VolumeMounts = d.volumeMount
121+
}
122+
return deploymentSpec
123+
}

‎test/framework/resources/k8s/resources/daemonset.go

+49
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,24 @@ package resources
1616
import (
1717
"context"
1818
"errors"
19+
"time"
1920

2021
"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
2122
v1 "k8s.io/api/apps/v1"
23+
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
2224
"k8s.io/apimachinery/pkg/types"
2325
"k8s.io/apimachinery/pkg/util/wait"
2426
"sigs.k8s.io/controller-runtime/pkg/client"
2527
)
2628

2729
type DaemonSetManager interface {
2830
GetDaemonSet(namespace string, name string) (*v1.DaemonSet, error)
31+
32+
CreateAndWaitTillDaemonSetIsReady(daemonSet *v1.DaemonSet, timeout time.Duration) (*v1.DaemonSet, error)
33+
2934
UpdateAndWaitTillDaemonSetReady(old *v1.DaemonSet, new *v1.DaemonSet) (*v1.DaemonSet, error)
3035
CheckIfDaemonSetIsReady(namespace string, name string) error
36+
DeleteAndWaitTillDaemonSetIsDeleted(daemonSet *v1.DaemonSet, timeout time.Duration) error
3137
}
3238

3339
type defaultDaemonSetManager struct {
@@ -38,6 +44,24 @@ func NewDefaultDaemonSetManager(k8sClient client.Client) DaemonSetManager {
3844
return &defaultDaemonSetManager{k8sClient: k8sClient}
3945
}
4046

47+
func (d *defaultDaemonSetManager) CreateAndWaitTillDaemonSetIsReady(daemonSet *v1.DaemonSet, timeout time.Duration) (*v1.DaemonSet, error) {
48+
ctx := context.Background()
49+
err := d.k8sClient.Create(ctx, daemonSet)
50+
if err != nil {
51+
return nil, err
52+
}
53+
54+
// Allow for the cache to sync
55+
time.Sleep(utils.PollIntervalShort)
56+
57+
err = d.CheckIfDaemonSetIsReady(daemonSet.Namespace, daemonSet.Name)
58+
if err != nil {
59+
return nil, err
60+
}
61+
62+
return daemonSet, nil
63+
}
64+
4165
func (d *defaultDaemonSetManager) GetDaemonSet(namespace string, name string) (*v1.DaemonSet, error) {
4266
ctx := context.Background()
4367
daemonSet := &v1.DaemonSet{}
@@ -94,3 +118,28 @@ func (d *defaultDaemonSetManager) CheckIfDaemonSetIsReady(namespace string, name
94118
}, ctx.Done())
95119

96120
}
121+
122+
func (d *defaultDaemonSetManager) DeleteAndWaitTillDaemonSetIsDeleted(daemonSet *v1.DaemonSet, timeout time.Duration) error {
123+
ctx := context.Background()
124+
125+
err := d.k8sClient.Delete(ctx, daemonSet)
126+
127+
if k8sErrors.IsNotFound(err) {
128+
return nil
129+
}
130+
131+
if err != nil {
132+
return err
133+
}
134+
observed := &v1.DaemonSet{}
135+
136+
return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
137+
if err := d.k8sClient.Get(ctx, utils.NamespacedName(daemonSet), observed); err != nil {
138+
if k8sErrors.IsNotFound(err) {
139+
return true, nil
140+
}
141+
return false, err
142+
}
143+
return false, nil
144+
}, ctx.Done())
145+
}

0 commit comments

Comments
 (0)