Skip to content

Commit cb63c72

Browse files
antoninbasantoninbas
authored
[SecondaryNetwork] Require resourceName annotations for SRIOV (#6999)
The "k8s.v1.cni.cncf.io/resourceName" annotation should be provided for all SRIOV NetworkAttachmentDefinitions. Otherwise, it is impossible to guarantee that the right device type is allocated for a given interface. Signed-off-by: Antonin Bas <[email protected]>
1 parent 26b44f1 commit cb63c72

File tree

3 files changed

+169
-58
lines changed

3 files changed

+169
-58
lines changed

‎pkg/agent/secondarynetwork/podwatch/controller.go

+26-7
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ const (
5757

5858
const (
5959
networkAttachDefAnnotationKey = "k8s.v1.cni.cncf.io/networks"
60+
resourceNameAnnotationKey = "k8s.v1.cni.cncf.io/resourceName"
6061
cniPath = "/opt/cni/bin/"
6162
startIfaceIndex = 1
6263
endIfaceIndex = 101
@@ -324,8 +325,10 @@ func (pc *PodController) processNextWorkItem() bool {
324325
func (pc *PodController) configureSecondaryInterface(
325326
pod *corev1.Pod,
326327
network *netdefv1.NetworkSelectionElement,
328+
resourceName string,
327329
podCNIInfo *podCNIInfo,
328-
networkConfig *SecondaryNetworkConfig) error {
330+
networkConfig *SecondaryNetworkConfig,
331+
) error {
329332
var ipamResult *ipam.IPAMResult
330333
var ifConfigErr error
331334
if networkConfig.IPAM != nil {
@@ -357,7 +360,7 @@ func (pc *PodController) configureSecondaryInterface(
357360

358361
switch networkConfig.NetworkType {
359362
case sriovNetworkType:
360-
ifConfigErr = pc.configureSriovAsSecondaryInterface(pod, network, podCNIInfo, int(networkConfig.MTU), &ipamResult.Result)
363+
ifConfigErr = pc.configureSriovAsSecondaryInterface(pod, network, resourceName, podCNIInfo, int(networkConfig.MTU), &ipamResult.Result)
361364
case vlanNetworkType:
362365
if networkConfig.VLAN > 0 {
363366
// Let VLAN ID in the CNI network configuration override the IPPool subnet
@@ -384,15 +387,15 @@ func (pc *PodController) configurePodSecondaryNetwork(pod *corev1.Pod, networkLi
384387
interfacesConfigured := 0
385388
for _, network := range networkList {
386389
klog.V(2).InfoS("Secondary Network attached to Pod", "network", network, "Pod", klog.KObj(pod))
387-
netDefCRD, err := pc.netAttachDefClient.NetworkAttachmentDefinitions(network.Namespace).Get(context.TODO(), network.Name, metav1.GetOptions{})
390+
netAttachDef, err := pc.netAttachDefClient.NetworkAttachmentDefinitions(network.Namespace).Get(context.TODO(), network.Name, metav1.GetOptions{})
388391
if err != nil {
389392
klog.ErrorS(err, "Failed to get NetworkAttachmentDefinition",
390393
"network", network, "Pod", klog.KRef(pod.Namespace, pod.Name))
391394
savedErr = err
392395
continue
393396
}
394397

395-
cniConfig, err := netdefutils.GetCNIConfig(netDefCRD, "")
398+
cniConfig, err := netdefutils.GetCNIConfig(netAttachDef, "")
396399
if err != nil {
397400
klog.ErrorS(err, "Failed to parse NetworkAttachmentDefinition",
398401
"network", network, "Pod", klog.KRef(pod.Namespace, pod.Name))
@@ -405,14 +408,30 @@ func (pc *PodController) configurePodSecondaryNetwork(pod *corev1.Pod, networkLi
405408
if networkConfig != nil && networkConfig.Type != cniserver.AntreaCNIType {
406409
// Ignore non-Antrea CNI type.
407410
klog.InfoS("Not Antrea CNI type in NetworkAttachmentDefinition, ignoring",
408-
"NetworkAttachmentDefinition", klog.KObj(netDefCRD), "Pod", klog.KRef(pod.Namespace, pod.Name))
411+
"NetworkAttachmentDefinition", klog.KObj(netAttachDef), "Pod", klog.KRef(pod.Namespace, pod.Name))
409412
} else {
410413
klog.ErrorS(err, "NetworkConfig validation failed",
411-
"NetworkAttachmentDefinition", klog.KObj(netDefCRD), "Pod", klog.KRef(pod.Namespace, pod.Name))
414+
"NetworkAttachmentDefinition", klog.KObj(netAttachDef), "Pod", klog.KRef(pod.Namespace, pod.Name))
412415
}
413416
continue
414417
}
415418

419+
var resourceName string
420+
if networkConfig.NetworkType == sriovNetworkType {
421+
v, ok := netAttachDef.Annotations[resourceNameAnnotationKey]
422+
if !ok {
423+
// This annotation is required for SRIOV devices, otherwise there is
424+
// no way to make sure that we allocate the "right" type of device.
425+
err := fmt.Errorf("missing annotation: %s", resourceNameAnnotationKey)
426+
klog.ErrorS(err, "Invalid NetworkAttachmentDefinition", "NetworkAttachmentDefinition", klog.KObj(netAttachDef))
427+
// It is probably worth retrying as the NetworkAttachmentDefinition
428+
// may eventually be updated with the missing annotation.
429+
savedErr = err
430+
continue
431+
}
432+
resourceName = v
433+
}
434+
416435
// Generate a new interface name, if the secondary interface name was not provided in the
417436
// Pod annotation.
418437
if network.InterfaceRequest == "" {
@@ -425,7 +444,7 @@ func (pc *PodController) configurePodSecondaryNetwork(pod *corev1.Pod, networkLi
425444
}
426445

427446
// Secondary network information retrieved from API server. Proceed to configure secondary interface now.
428-
if err = pc.configureSecondaryInterface(pod, network, podCNIInfo, networkConfig); err != nil {
447+
if err = pc.configureSecondaryInterface(pod, network, resourceName, podCNIInfo, networkConfig); err != nil {
429448
klog.ErrorS(err, "Secondary interface configuration failed",
430449
"Pod", klog.KRef(pod.Namespace, pod.Name), "interface", network.InterfaceRequest,
431450
"networkType", networkConfig.NetworkType)

‎pkg/agent/secondarynetwork/podwatch/controller_test.go

+107-23
Original file line numberDiff line numberDiff line change
@@ -78,27 +78,34 @@ const (
7878
"vlan": {{.VLAN}}
7979
}`
8080

81-
defaultCNIVersion = "0.3.0"
82-
defaultMTU = 1500
83-
sriovDeviceID = "sriov-device-id"
84-
podName = "pod1"
85-
containerID = "container1"
86-
podIP = "1.2.3.4"
87-
networkName = "net"
88-
interfaceName = "eth2"
81+
defaultCNIVersion = "0.3.0"
82+
defaultMTU = 1500
83+
sriovResourceName1 = "intel.com/intel_sriov_netdevice"
84+
sriovResourceName2 = "mellanox.com/mlnx_connectx5"
85+
sriovDeviceID11 = "sriov-device-id-11"
86+
sriovDeviceID12 = "sriov-device-id-12"
87+
sriovDeviceID21 = "sriov-device-id-21"
88+
podName = "pod1"
89+
containerID = "container1"
90+
podIP = "1.2.3.4"
91+
networkName = "net"
92+
interfaceName = "eth2"
8993
)
9094

9195
func testNetwork(name string, networkType networkType) *netdefv1.NetworkAttachmentDefinition {
92-
return testNetworkExt(name, "", "", string(networkType), "", 0, 0, false)
96+
return testNetworkExt(name, "", "", networkType, "", "", 0, 0, false)
9397
}
9498

95-
func testNetworkExt(name, cniVersion, cniType, networkType, ipamType string, mtu, vlan int, noIPAM bool) *netdefv1.NetworkAttachmentDefinition {
99+
func testNetworkExt(name, cniVersion, cniType string, networkType networkType, resourceName, ipamType string, mtu, vlan int, noIPAM bool) *netdefv1.NetworkAttachmentDefinition {
96100
if cniVersion == "" {
97101
cniVersion = defaultCNIVersion
98102
}
99103
if cniType == "" {
100104
cniType = "antrea"
101105
}
106+
if networkType == sriovNetworkType && resourceName == "" {
107+
resourceName = sriovResourceName1
108+
}
102109
if ipamType == "" {
103110
ipamType = ipam.AntreaIPAMType
104111
}
@@ -109,7 +116,7 @@ func testNetworkExt(name, cniVersion, cniType, networkType, ipamType string, mtu
109116
IPAMType string
110117
MTU int
111118
VLAN int
112-
}{cniVersion, cniType, networkType, ipamType, mtu, vlan}
119+
}{cniVersion, cniType, string(networkType), ipamType, mtu, vlan}
113120

114121
var tmpl *template.Template
115122
if !noIPAM {
@@ -119,9 +126,14 @@ func testNetworkExt(name, cniVersion, cniType, networkType, ipamType string, mtu
119126
}
120127
var b bytes.Buffer
121128
tmpl.Execute(&b, &data)
129+
annotations := make(map[string]string)
130+
if resourceName != "" {
131+
annotations[resourceNameAnnotationKey] = resourceName
132+
}
122133
return &netdefv1.NetworkAttachmentDefinition{
123134
ObjectMeta: metav1.ObjectMeta{
124-
Name: name,
135+
Name: name,
136+
Annotations: annotations,
125137
},
126138
Spec: netdefv1.NetworkAttachmentDefinitionSpec{
127139
Config: b.String(),
@@ -189,8 +201,11 @@ func testIPAMResult(cidr string, vlan int) *ipam.IPAMResult {
189201
}
190202

191203
func init() {
192-
getPodContainerDeviceIDsFn = func(name string, namespace string) ([]string, error) {
193-
return []string{sriovDeviceID}, nil
204+
getPodContainerDeviceIDsFn = func(name string, namespace string) (map[string][]string, error) {
205+
return map[string][]string{
206+
sriovResourceName1: {sriovDeviceID11, sriovDeviceID12},
207+
sriovResourceName2: {sriovDeviceID21},
208+
}, nil
194209
}
195210
}
196211

@@ -258,7 +273,7 @@ func TestPodControllerRun(t *testing.T) {
258273
containerNetNs(containerID),
259274
interfaceName,
260275
defaultMTU,
261-
sriovDeviceID,
276+
sriovDeviceID11,
262277
&ipamResult.Result,
263278
).Do(func(string, string, string, string, string, int, string, *current.Result) {
264279
atomic.AddInt32(&interfaceConfigured, 1)
@@ -277,7 +292,7 @@ func TestPodControllerRun(t *testing.T) {
277292
_, err = client.CoreV1().Pods(testNamespace).Create(context.Background(), pod, metav1.CreateOptions{})
278293
require.NoError(t, err, "error when creating test Pod")
279294

280-
// Wait for ConfigureSriovSecondaryInterface is called.
295+
// Wait for ConfigureSriovSecondaryInterface to be called.
281296
assert.Eventually(t, func() bool {
282297
return atomic.LoadInt32(&interfaceConfigured) == 1
283298
}, 1*time.Second, 10*time.Millisecond)
@@ -292,7 +307,8 @@ func TestPodControllerRun(t *testing.T) {
292307
containerNetNs(containerID),
293308
interfaceName,
294309
defaultMTU,
295-
"",
310+
// We haven't updated the vfDeviceIDUsageMap, so a different device will be allocated.
311+
sriovDeviceID12,
296312
&ipamResult.Result,
297313
).Do(func(string, string, string, string, string, int, string, *current.Result) {
298314
atomic.AddInt32(&interfaceConfigured, 1)
@@ -329,7 +345,7 @@ func TestPodControllerRun(t *testing.T) {
329345
mockIPAM.EXPECT().SecondaryNetworkRelease(podOwner)
330346
// CNI Del event.
331347
event.IsAdd = false
332-
// Interfac is not deleted from the interface store, so CNI Del should trigger interface
348+
// Interface is not deleted from the interface store, so CNI Del should trigger interface
333349
// deletion again.
334350
podController.processCNIUpdate(event)
335351
_, exists = cniCache.Load(podKey)
@@ -455,7 +471,7 @@ func TestConfigurePodSecondaryNetwork(t *testing.T) {
455471
containerNetNs(containerID),
456472
interfaceName,
457473
1500,
458-
sriovDeviceID,
474+
sriovDeviceID11,
459475
&testIPAMResult("148.14.24.100/24", 0).Result,
460476
)
461477
},
@@ -561,7 +577,7 @@ func TestConfigurePodSecondaryNetwork(t *testing.T) {
561577

562578
if !tc.doNotCreateNetwork {
563579
network1 := testNetworkExt(networkName, tc.cniVersion, tc.cniType,
564-
string(tc.networkType), tc.ipamType, tc.mtu, tc.vlan, tc.noIPAM)
580+
tc.networkType, "", tc.ipamType, tc.mtu, tc.vlan, tc.noIPAM)
565581
pc.netAttachDefClient.NetworkAttachmentDefinitions(testNamespace).Create(context.Background(),
566582
network1, metav1.CreateOptions{})
567583
}
@@ -579,6 +595,74 @@ func TestConfigurePodSecondaryNetwork(t *testing.T) {
579595

580596
}
581597

598+
func TestConfigurePodSecondaryNetworkMultipleSriovDevices(t *testing.T) {
599+
ctx := context.Background()
600+
element1 := netdefv1.NetworkSelectionElement{
601+
Namespace: testNamespace,
602+
Name: "net1",
603+
InterfaceRequest: "eth10",
604+
}
605+
element2 := netdefv1.NetworkSelectionElement{
606+
Namespace: testNamespace,
607+
Name: "net2",
608+
InterfaceRequest: "eth11",
609+
}
610+
pod, cniInfo := testPod(podName, containerID, podIP, element1, element2)
611+
ctrl := gomock.NewController(t)
612+
pc, _, interfaceConfigurator := testPodControllerStart(ctrl)
613+
614+
network1 := testNetworkExt("net1", "", "", sriovNetworkType, sriovResourceName1, "", 1500, 0, true /* noIPAM */)
615+
pc.netAttachDefClient.NetworkAttachmentDefinitions(testNamespace).Create(ctx, network1, metav1.CreateOptions{})
616+
network2 := testNetworkExt("net2", "", "", sriovNetworkType, sriovResourceName2, "", 1500, 0, true /* noIPAM */)
617+
pc.netAttachDefClient.NetworkAttachmentDefinitions(testNamespace).Create(ctx, network2, metav1.CreateOptions{})
618+
619+
gomock.InOrder(
620+
interfaceConfigurator.EXPECT().ConfigureSriovSecondaryInterface(
621+
podName,
622+
testNamespace,
623+
containerID,
624+
containerNetNs(containerID),
625+
element2.InterfaceRequest,
626+
1500,
627+
sriovDeviceID21,
628+
gomock.Any(),
629+
),
630+
interfaceConfigurator.EXPECT().ConfigureSriovSecondaryInterface(
631+
podName,
632+
testNamespace,
633+
containerID,
634+
containerNetNs(containerID),
635+
element1.InterfaceRequest,
636+
1500,
637+
sriovDeviceID11,
638+
gomock.Any(),
639+
),
640+
)
641+
assert.NoError(t, pc.configurePodSecondaryNetwork(pod, []*netdefv1.NetworkSelectionElement{&element2, &element1}, cniInfo))
642+
643+
podKey := podKeyGet(pod.Name, pod.Namespace)
644+
deviceCache, ok := pc.vfDeviceIDUsageMap.Load(podKey)
645+
require.True(t, ok)
646+
expectedDeviceCache := []podSriovVFDeviceIDInfo{
647+
{
648+
resourceName: sriovResourceName1,
649+
vfDeviceID: sriovDeviceID11,
650+
ifName: element1.InterfaceRequest,
651+
},
652+
{
653+
resourceName: sriovResourceName1,
654+
vfDeviceID: sriovDeviceID12,
655+
ifName: "",
656+
},
657+
{
658+
resourceName: sriovResourceName2,
659+
vfDeviceID: sriovDeviceID21,
660+
ifName: element2.InterfaceRequest,
661+
},
662+
}
663+
assert.ElementsMatch(t, expectedDeviceCache, deviceCache.([]podSriovVFDeviceIDInfo))
664+
}
665+
582666
func TestPodControllerAddPod(t *testing.T) {
583667
pod, _ := testPod(podName, containerID, podIP, netdefv1.NetworkSelectionElement{
584668
Name: networkName,
@@ -623,7 +707,7 @@ func TestPodControllerAddPod(t *testing.T) {
623707
)
624708
network1 := testNetwork("net1", sriovNetworkType)
625709
testVLAN := 100
626-
network2 := testNetworkExt("net2", "", "", string(vlanNetworkType), "", defaultMTU, testVLAN, false)
710+
network2 := testNetworkExt("net2", "", "", vlanNetworkType, "", "", defaultMTU, testVLAN, false)
627711

628712
podOwner1 := &crdv1beta1.PodOwner{Name: podName, Namespace: testNamespace,
629713
ContainerID: containerID, IFName: "eth10"}
@@ -771,7 +855,7 @@ func TestPodControllerAddPod(t *testing.T) {
771855
containerNetNs(containerID),
772856
interfaceName,
773857
defaultMTU,
774-
sriovDeviceID,
858+
sriovDeviceID11,
775859
gomock.Any(),
776860
)
777861
mockIPAM.EXPECT().SecondaryNetworkAllocate(gomock.Any(), gomock.Any()).Return(testIPAMResult("148.14.24.100/24", 0), nil)
@@ -885,7 +969,7 @@ func TestPodControllerAddPod(t *testing.T) {
885969
t.Run("updating deviceID cache per Pod", func(t *testing.T) {
886970
ctrl := gomock.NewController(t)
887971
podController, _, _ := testPodController(ctrl)
888-
_, err := podController.assignUnusedSriovVFDeviceID(podName, testNamespace, interfaceName)
972+
_, err := podController.assignUnusedSriovVFDeviceID(podName, testNamespace, sriovResourceName1, interfaceName)
889973
_, exists := podController.vfDeviceIDUsageMap.Load(podKey)
890974
assert.True(t, exists)
891975
require.NoError(t, err, "error while assigning unused VfDevice ID")

0 commit comments

Comments
 (0)