Skip to content

Commit 09e709f

Browse files
author
Joseph Chen
committed
POD_MTU
1 parent 502605c commit 09e709f

File tree

4 files changed

+159
-98
lines changed

4 files changed

+159
-98
lines changed

‎README.md

+9
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,15 @@ Default: empty
267267
Specify a comma-separated list of IPv4 CIDRs to exclude from SNAT. For every item in the list an `iptables` rule and off\-VPC
268268
IP rule will be applied. If an item is not a valid ipv4 range it will be skipped. This should be used when `AWS_VPC_K8S_CNI_EXTERNALSNAT=false`.
269269

270+
#### `POD_MTU` (v1.x.x+)
271+
272+
Type: Integer as a String
273+
274+
*Note*: The default value is set to AWS_VPC_ENI_MTU, which defaults to 9001 if unset.
275+
Default: 9001
276+
277+
Used to configure the MTU size for pod virtual interfaces. The valid range is from `576` to `9001`.
278+
270279
#### `WARM_ENI_TARGET`
271280

272281
Type: Integer as a String

‎cmd/aws-vpc-cni/main.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ const (
8888
envHostCniConfDirPath = "HOST_CNI_CONFDIR_PATH"
8989
envVethPrefix = "AWS_VPC_K8S_CNI_VETHPREFIX"
9090
envEniMTU = "AWS_VPC_ENI_MTU"
91+
envPodMTU = "POD_MTU"
9192
envEnablePodEni = "ENABLE_POD_ENI"
9293
envPodSGEnforcingMode = "POD_SECURITY_GROUP_ENFORCING_MODE"
9394
envPluginLogFile = "AWS_VPC_K8S_PLUGIN_LOG_FILE"
@@ -278,15 +279,18 @@ func generateJSON(jsonFile string, outFile string, getPrimaryIP func(ipv4 bool)
278279
}
279280
}
280281
vethPrefix := utils.GetEnv(envVethPrefix, defaultVethPrefix)
281-
mtu := utils.GetEnv(envEniMTU, defaultMTU)
282+
// Derive pod MTU from ENI MTU by default
283+
eniMTU := utils.GetEnv(envEniMTU, defaultMTU)
284+
// If pod MTU environment variable is set, overwrite ENI MTU.
285+
podMTU := utils.GetEnv(envPodMTU, eniMTU)
282286
podSGEnforcingMode := utils.GetEnv(envPodSGEnforcingMode, defaultPodSGEnforcingMode)
283287
pluginLogFile := utils.GetEnv(envPluginLogFile, defaultPluginLogFile)
284288
pluginLogLevel := utils.GetEnv(envPluginLogLevel, defaultPluginLogLevel)
285289
randomizeSNAT := utils.GetEnv(envRandomizeSNAT, defaultRandomizeSNAT)
286290

287291
netconf := string(byteValue)
288292
netconf = strings.Replace(netconf, "__VETHPREFIX__", vethPrefix, -1)
289-
netconf = strings.Replace(netconf, "__MTU__", mtu, -1)
293+
netconf = strings.Replace(netconf, "__MTU__", podMTU, -1)
290294
netconf = strings.Replace(netconf, "__PODSGENFORCINGMODE__", podSGEnforcingMode, -1)
291295
netconf = strings.Replace(netconf, "__PLUGINLOGFILE__", pluginLogFile, -1)
292296
netconf = strings.Replace(netconf, "__PLUGINLOGLEVEL__", pluginLogLevel, -1)

‎test/integration/cni/host_networking_test.go

+72-49
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,11 @@ import (
1717
"strconv"
1818
"time"
1919

20-
v1 "k8s.io/api/core/v1"
21-
2220
"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
2321
k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
2422
"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
2523
"github.com/aws/amazon-vpc-cni-k8s/test/integration/common"
24+
v1 "k8s.io/api/core/v1"
2625

2726
. "github.com/onsi/ginkgo/v2"
2827
. "github.com/onsi/gomega"
@@ -31,13 +30,15 @@ import (
3130
// TODO: Instead of passing the list of pods to the test helper, have the test helper get the pod on node
3231
const (
3332
NEW_MTU_VAL = 1300
33+
NEW_POD_MTU = 1280
3434
NEW_VETH_PREFIX = "veth"
35+
podLabelKey = "app"
36+
podLabelVal = "host-networking-test"
3537
)
3638

39+
var err error
40+
3741
var _ = Describe("test host networking", func() {
38-
var err error
39-
var podLabelKey = "app"
40-
var podLabelVal = "host-networking-test"
4142

4243
// For host networking tests, increase WARM_IP_TARGET to prevent long IPAMD warmup.
4344
BeforeEach(func() {
@@ -57,6 +58,10 @@ var _ = Describe("test host networking", func() {
5758
"AWS_VPC_ENI_MTU": DEFAULT_MTU_VAL,
5859
"AWS_VPC_K8S_CNI_VETHPREFIX": DEFAULT_VETH_PREFIX,
5960
})
61+
k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName,
62+
utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{
63+
"POD_MTU": {},
64+
})
6065
// After updating daemonset pod, we must wait until conflist is updated so that container-runtime calls CNI ADD with the latest VETH prefix and MTU.
6166
// Otherwise, the stale value can cause failures in future test cases.
6267
time.Sleep(utils.PollIntervalMedium)
@@ -104,51 +109,13 @@ var _ = Describe("test host networking", func() {
104109
common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f)
105110
})
106111

107-
It("Validate Host Networking setup after changing MTU and Veth Prefix", func() {
108-
deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
109-
Replicas(maxIPPerInterface*2).
110-
PodLabel(podLabelKey, podLabelVal).
111-
NodeName(primaryNode.Name).
112-
Build()
113-
114-
By("Configuring Veth Prefix and MTU value on aws-node daemonset")
115-
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
116-
"AWS_VPC_ENI_MTU": strconv.Itoa(NEW_MTU_VAL),
117-
"AWS_VPC_K8S_CNI_VETHPREFIX": NEW_VETH_PREFIX,
112+
Context("Validate Host Networking setup after changing Veth Prefix and", func() {
113+
It("ENI MTU", func() {
114+
mtuValidationTest(false, NEW_MTU_VAL)
115+
})
116+
It("POD MTU", func() {
117+
mtuValidationTest(true, NEW_POD_MTU)
118118
})
119-
// After updating daemonset pod, we must wait until conflist is updated so that container-runtime calls CNI ADD with the new VETH prefix and MTU.
120-
time.Sleep(utils.PollIntervalMedium)
121-
122-
By("creating a deployment to launch pods")
123-
deployment, err = f.K8sResourceManagers.DeploymentManager().
124-
CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
125-
Expect(err).ToNot(HaveOccurred())
126-
127-
By("getting the list of pods using IP from primary and secondary ENI")
128-
interfaceTypeToPodList :=
129-
common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f)
130-
131-
By("generating the pod networking validation input to be passed to tester")
132-
podNetworkingValidationInput := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs)
133-
podNetworkingValidationInput.VethPrefix = NEW_VETH_PREFIX
134-
podNetworkingValidationInput.ValidateMTU = true
135-
podNetworkingValidationInput.MTU = NEW_MTU_VAL
136-
input, err := podNetworkingValidationInput.Serialize()
137-
Expect(err).NotTo(HaveOccurred())
138-
139-
By("validating host networking setup is setup correctly with MTU check as well")
140-
common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f)
141-
142-
By("deleting the deployment to test teardown")
143-
err = f.K8sResourceManagers.DeploymentManager().
144-
DeleteAndWaitTillDeploymentIsDeleted(deployment)
145-
Expect(err).ToNot(HaveOccurred())
146-
147-
By("waiting to allow CNI to tear down networking for terminated pods")
148-
time.Sleep(time.Second * 60)
149-
150-
By("validating host networking is teared down correctly")
151-
common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f)
152119
})
153120
})
154121

@@ -205,3 +172,59 @@ var _ = Describe("test host networking", func() {
205172
})
206173
})
207174
})
175+
176+
func mtuValidationTest(usePodMTU bool, mtuVal int) {
177+
deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
178+
Replicas(maxIPPerInterface*2).
179+
PodLabel(podLabelKey, podLabelVal).
180+
NodeName(primaryNode.Name).
181+
Build()
182+
183+
if usePodMTU {
184+
By("Configuring Veth Prefix and Pod MTU value on aws-node daemonset")
185+
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
186+
"AWS_VPC_ENI_MTU": strconv.Itoa(NEW_MTU_VAL),
187+
"POD_MTU": strconv.Itoa(NEW_POD_MTU),
188+
"AWS_VPC_K8S_CNI_VETHPREFIX": NEW_VETH_PREFIX,
189+
})
190+
} else {
191+
By("Configuring Veth Prefix and ENI MTU value on aws-node daemonset")
192+
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
193+
"AWS_VPC_ENI_MTU": strconv.Itoa(NEW_MTU_VAL),
194+
"AWS_VPC_K8S_CNI_VETHPREFIX": NEW_VETH_PREFIX,
195+
})
196+
}
197+
// After updating daemonset pod, we must wait until conflist is updated so that container-runtime calls CNI ADD with the new VETH prefix and MTU.
198+
time.Sleep(utils.PollIntervalMedium)
199+
200+
By("creating a deployment to launch pods")
201+
deployment, err = f.K8sResourceManagers.DeploymentManager().
202+
CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
203+
Expect(err).ToNot(HaveOccurred())
204+
205+
By("getting the list of pods using IP from primary and secondary ENI")
206+
interfaceTypeToPodList :=
207+
common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f)
208+
209+
By("generating the pod networking validation input to be passed to tester")
210+
podNetworkingValidationInput := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs)
211+
podNetworkingValidationInput.VethPrefix = NEW_VETH_PREFIX
212+
podNetworkingValidationInput.ValidateMTU = true
213+
podNetworkingValidationInput.MTU = mtuVal
214+
input, err := podNetworkingValidationInput.Serialize()
215+
Expect(err).NotTo(HaveOccurred())
216+
217+
By("validating host networking setup is setup correctly with MTU check as well")
218+
common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f)
219+
220+
By("deleting the deployment to test teardown")
221+
err = f.K8sResourceManagers.DeploymentManager().
222+
DeleteAndWaitTillDeploymentIsDeleted(deployment)
223+
Expect(err).ToNot(HaveOccurred())
224+
225+
By("waiting to allow CNI to tear down networking for terminated pods")
226+
time.Sleep(time.Second * 60)
227+
228+
By("validating host networking is teared down correctly")
229+
common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f)
230+
}

‎test/integration/ipv6/ipv6_host_networking_test.go

+72-47
Original file line numberDiff line numberDiff line change
@@ -41,23 +41,30 @@ const (
4141
const (
4242
AWS_VPC_ENI_MTU = "AWS_VPC_ENI_MTU"
4343
AWS_VPC_K8S_CNI_VETHPREFIX = "AWS_VPC_K8S_CNI_VETHPREFIX"
44+
POD_MTU = "POD_MTU"
4445
NEW_MTU_VAL = 1300
46+
NEW_POD_MTU = 1280
4547
NEW_VETH_PREFIX = "veth"
4648
DEFAULT_MTU_VAL = "9001"
4749
DEFAULT_VETH_PREFIX = "eni"
50+
podLabelKey = "app"
51+
podLabelVal = "host-networking-test"
4852
)
4953

54+
var err error
55+
5056
var _ = Describe("[CANARY] test ipv6 host netns setup", func() {
51-
var err error
52-
var podLabelKey = "app"
53-
var podLabelVal = "host-networking-test"
5457

5558
Context("when pods using IP from primary ENI are created", func() {
5659
AfterEach(func() {
5760
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
5861
AWS_VPC_ENI_MTU: DEFAULT_MTU_VAL,
5962
AWS_VPC_K8S_CNI_VETHPREFIX: DEFAULT_VETH_PREFIX,
6063
})
64+
k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName,
65+
utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{
66+
"POD_MTU": {},
67+
})
6168
// After updating daemonset pod, we must wait until conflist is updated so that container-runtime calls CNI ADD with the latest VETH prefix and MTU.
6269
// Otherwise, the stale value can cause failures in future test cases.
6370
time.Sleep(utils.PollIntervalMedium)
@@ -98,51 +105,13 @@ var _ = Describe("[CANARY] test ipv6 host netns setup", func() {
98105
ValidateHostNetworking(NetworkingTearDownSucceeds, input)
99106
})
100107

101-
It("Validate host netns setup after changing MTU and Veth Prefix", func() {
102-
deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
103-
Replicas(2).
104-
PodLabel(podLabelKey, podLabelVal).
105-
NodeName(primaryNode.Name).
106-
Build()
107-
108-
By("Configuring Veth Prefix and MTU value on aws-node daemonset")
109-
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
110-
AWS_VPC_ENI_MTU: strconv.Itoa(NEW_MTU_VAL),
111-
AWS_VPC_K8S_CNI_VETHPREFIX: NEW_VETH_PREFIX,
108+
Context("Validate Host Networking setup after changing Veth Prefix and", func() {
109+
It("ENI MTU", func() {
110+
mtuValidationTest(false, NEW_MTU_VAL)
111+
})
112+
It("POD MTU", func() {
113+
mtuValidationTest(true, NEW_POD_MTU)
112114
})
113-
// After updating daemonset pod, we must wait until conflist is updated so that container-runtime calls CNI ADD with the new VETH prefix and MTU.
114-
time.Sleep(utils.PollIntervalMedium)
115-
116-
By("creating a deployment to launch pods")
117-
deployment, err = f.K8sResourceManagers.DeploymentManager().
118-
CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
119-
Expect(err).ToNot(HaveOccurred())
120-
121-
By("getting the list of pods using IP from primary and secondary ENI")
122-
interfaceTypeToPodList :=
123-
GetIPv6Pods(podLabelKey, podLabelVal)
124-
125-
By("generating the pod networking validation input to be passed to tester")
126-
podNetworkingValidationInput := GetIPv6PodNetworkingValidationInput(interfaceTypeToPodList)
127-
podNetworkingValidationInput.VethPrefix = NEW_VETH_PREFIX
128-
podNetworkingValidationInput.ValidateMTU = true
129-
podNetworkingValidationInput.MTU = NEW_MTU_VAL
130-
input, err := podNetworkingValidationInput.Serialize()
131-
Expect(err).NotTo(HaveOccurred())
132-
133-
By("validating host networking setup is setup correctly with MTU check as well")
134-
ValidateHostNetworking(NetworkingSetupSucceeds, input)
135-
136-
By("deleting the deployment to test teardown")
137-
err = f.K8sResourceManagers.DeploymentManager().
138-
DeleteAndWaitTillDeploymentIsDeleted(deployment)
139-
Expect(err).ToNot(HaveOccurred())
140-
141-
By("waiting to allow CNI to tear down networking for terminated pods")
142-
time.Sleep(time.Second * 60)
143-
144-
By("validating host networking is teared down correctly")
145-
ValidateHostNetworking(NetworkingTearDownSucceeds, input)
146115
})
147116
})
148117

@@ -277,3 +246,59 @@ func GetIPv6PodNetworkingValidationInput(podList v1.PodList) input.PodNetworking
277246
}
278247
return ip
279248
}
249+
250+
func mtuValidationTest(usePodMTU bool, mtuVal int) {
251+
deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
252+
Replicas(2).
253+
PodLabel(podLabelKey, podLabelVal).
254+
NodeName(primaryNode.Name).
255+
Build()
256+
257+
if usePodMTU {
258+
By("Configuring Veth Prefix and Pod MTU value on aws-node daemonset")
259+
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
260+
AWS_VPC_ENI_MTU: strconv.Itoa(NEW_MTU_VAL),
261+
POD_MTU: strconv.Itoa(NEW_POD_MTU),
262+
AWS_VPC_K8S_CNI_VETHPREFIX: NEW_VETH_PREFIX,
263+
})
264+
} else {
265+
By("Configuring Veth Prefix and ENI MTU value on aws-node daemonset")
266+
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
267+
AWS_VPC_ENI_MTU: strconv.Itoa(NEW_MTU_VAL),
268+
AWS_VPC_K8S_CNI_VETHPREFIX: NEW_VETH_PREFIX,
269+
})
270+
}
271+
// After updating daemonset pod, we must wait until conflist is updated so that container-runtime calls CNI ADD with the new VETH prefix and MTU.
272+
time.Sleep(utils.PollIntervalMedium)
273+
274+
By("creating a deployment to launch pods")
275+
deployment, err = f.K8sResourceManagers.DeploymentManager().
276+
CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
277+
Expect(err).ToNot(HaveOccurred())
278+
279+
By("getting the list of pods using IP from primary and secondary ENI")
280+
interfaceTypeToPodList :=
281+
GetIPv6Pods(podLabelKey, podLabelVal)
282+
283+
By("generating the pod networking validation input to be passed to tester")
284+
podNetworkingValidationInput := GetIPv6PodNetworkingValidationInput(interfaceTypeToPodList)
285+
podNetworkingValidationInput.VethPrefix = NEW_VETH_PREFIX
286+
podNetworkingValidationInput.ValidateMTU = true
287+
podNetworkingValidationInput.MTU = mtuVal
288+
input, err := podNetworkingValidationInput.Serialize()
289+
Expect(err).NotTo(HaveOccurred())
290+
291+
By("validating host networking setup is setup correctly with MTU check as well")
292+
ValidateHostNetworking(NetworkingSetupSucceeds, input)
293+
294+
By("deleting the deployment to test teardown")
295+
err = f.K8sResourceManagers.DeploymentManager().
296+
DeleteAndWaitTillDeploymentIsDeleted(deployment)
297+
Expect(err).ToNot(HaveOccurred())
298+
299+
By("waiting to allow CNI to tear down networking for terminated pods")
300+
time.Sleep(time.Second * 60)
301+
302+
By("validating host networking is teared down correctly")
303+
ValidateHostNetworking(NetworkingTearDownSucceeds, input)
304+
}

0 commit comments

Comments
 (0)