Skip to content

Commit 53c16cc

Browse files
yuva29yuva29
committed
Support dual IP stacks in kubernetes CNI plugin
Signed-off-by: Yuva Shankar <[email protected]>
1 parent aaecab2 commit 53c16cc

File tree

9 files changed

+167
-36
lines changed

9 files changed

+167
-36
lines changed

‎drivers/endpointstate.go

+2
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ type OperEndpointState struct {
3131
ServiceName string `json:"serviceName"`
3232
ContUUID string `json:"contUUID"`
3333
IPAddress string `json:"ipAddress"`
34+
IPv6Address string `json:"ipv6Address"`
3435
MacAddress string `json:"macAddress"`
3536
HomingHost string `json:"homingHost"`
3637
IntfName string `json:"intfName"`
@@ -43,6 +44,7 @@ func (s *OperEndpointState) Matches(c *mastercfg.CfgEndpointState) bool {
4344
return s.NetID == c.NetID &&
4445
s.EndpointID == c.EndpointID &&
4546
s.IPAddress == c.IPAddress &&
47+
s.IPv6Address == c.IPv6Address &&
4648
s.MacAddress == c.MacAddress &&
4749
s.HomingHost == c.HomingHost &&
4850
s.IntfName == c.IntfName &&

‎drivers/ovsd/ovsdriver.go

+1
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,7 @@ func (d *OvsDriver) CreateEndpoint(id string) error {
451451
EndpointID: cfgEp.EndpointID,
452452
ServiceName: cfgEp.ServiceName,
453453
IPAddress: cfgEp.IPAddress,
454+
IPv6Address: cfgEp.IPv6Address,
454455
MacAddress: cfgEp.MacAddress,
455456
IntfName: cfgEp.IntfName,
456457
PortName: intfName,

‎mgmtfn/k8splugin/cniapi/api.go

+6-5
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,10 @@ type CNIPodAttr struct {
4040

4141
// RspAddPod contains the response to the AddPod
4242
type RspAddPod struct {
43-
Result uint `json:"result,omitempty"`
44-
EndpointID string `json:"endpointid,omitempty"`
45-
IPAddress string `json:"ipaddress,omitempty"`
46-
ErrMsg string `json:"errmsg,omitempty"`
47-
ErrInfo string `json:"errinfo,omitempty"`
43+
Result uint `json:"result,omitempty"`
44+
EndpointID string `json:"endpointid,omitempty"`
45+
IPAddress string `json:"ipaddress,omitempty"`
46+
IPv6Address string `json:"ipv6address,omitempty"`
47+
ErrMsg string `json:"errmsg,omitempty"`
48+
ErrInfo string `json:"errinfo,omitempty"`
4849
}

‎mgmtfn/k8splugin/contivk8s/k8s_cni.go

+46-6
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"encoding/json"
2020
"flag"
2121
"fmt"
22+
"net"
2223
"os"
2324
"strings"
2425

@@ -27,14 +28,20 @@ import (
2728
"github.com/contiv/netplugin/version"
2829

2930
logger "github.com/Sirupsen/logrus"
31+
ip "github.com/appc/cni/pkg/ip"
32+
cni "github.com/appc/cni/pkg/plugin"
3033
)
3134

35+
// CNIResponse response format expected from CNI plugins(version 0.1.0)
36+
type CNIResponse struct {
37+
CNIVersion string `json:"cniVersion"`
38+
cni.Result
39+
}
40+
3241
//CNIError : return format from CNI plugin
3342
type CNIError struct {
3443
CNIVersion string `json:"cniVersion"`
35-
Code uint `json:"code"`
36-
Msg string `json:"msg"`
37-
Details string `json:"details,omitempty"`
44+
cni.Error
3845
}
3946

4047
var log *logger.Entry
@@ -91,9 +98,42 @@ func addPodToContiv(nc *clients.NWClient, pInfo *cniapi.CNIPodAttr) {
9198

9299
log.Infof("EP created IP: %s\n", result.IPAddress)
93100
// Write the ip address of the created endpoint to stdout
94-
fmt.Printf("{\n\"cniVersion\": \"0.1.0\",\n")
95-
fmt.Printf("\"ip4\": {\n")
96-
fmt.Printf("\"ip\": \"%s\"\n}\n}\n", result.IPAddress)
101+
102+
// ParseCIDR returns a reference to IPNet
103+
ip4Net, err := ip.ParseCIDR(result.IPAddress)
104+
if err != nil {
105+
log.Errorf("Failed to parse IPv4 CIDR: %v", err)
106+
return
107+
}
108+
109+
out := CNIResponse{
110+
CNIVersion: "0.1.0",
111+
}
112+
113+
out.IP4 = &cni.IPConfig{
114+
IP: net.IPNet{IP: ip4Net.IP, Mask: ip4Net.Mask},
115+
}
116+
117+
if result.IPv6Address != "" {
118+
ip6Net, err := ip.ParseCIDR(result.IPv6Address)
119+
if err != nil {
120+
log.Errorf("Failed to parse IPv6 CIDR: %v", err)
121+
return
122+
}
123+
124+
out.IP6 = &cni.IPConfig{
125+
IP: net.IPNet{IP: ip6Net.IP, Mask: ip6Net.Mask},
126+
}
127+
}
128+
129+
data, err := json.MarshalIndent(out, "", " ")
130+
if err != nil {
131+
log.Errorf("Failed to marshal json: %v", err)
132+
return
133+
}
134+
135+
log.Infof("Response from CNI executable: \n%s", fmt.Sprintf("%s", data))
136+
fmt.Printf(fmt.Sprintf("%s", data))
97137
}
98138

99139
func deletePodFromContiv(nc *clients.NWClient, pInfo *cniapi.CNIPodAttr) {

‎mgmtfn/k8splugin/driver.go

+45-11
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,11 @@ type epSpec struct {
4848

4949
// epAttr contains the assigned attributes of the created ep
5050
type epAttr struct {
51-
IPAddress string
52-
PortName string
53-
Gateway string
51+
IPAddress string
52+
PortName string
53+
Gateway string
54+
IPv6Address string
55+
IPv6Gateway string
5456
}
5557

5658
// netdGetEndpoint is a utility that reads the EP oper state
@@ -152,6 +154,7 @@ func createEP(req *epSpec) (*epAttr, error) {
152154
return nil, err
153155
}
154156

157+
// this response should contain IPv6 if the underlying network is configured with IPv6
155158
log.Infof("Got endpoint create resp from master: %+v", mresp)
156159

157160
// Ask netplugin to create the endpoint
@@ -181,6 +184,11 @@ func createEP(req *epSpec) (*epAttr, error) {
181184
epResponse.IPAddress = ep.IPAddress + "/" + strconv.Itoa(int(nw.SubnetLen))
182185
epResponse.Gateway = nw.Gateway
183186

187+
if ep.IPv6Address != "" {
188+
epResponse.IPv6Address = ep.IPv6Address + "/" + strconv.Itoa(int(nw.IPv6SubnetLen))
189+
epResponse.IPv6Gateway = nw.IPv6Gateway
190+
}
191+
184192
return &epResponse, nil
185193
}
186194

@@ -237,8 +245,7 @@ func moveToNS(pid int, ifname string) error {
237245
}
238246

239247
// setIfAttrs sets the required attributes for the container interface
240-
func setIfAttrs(pid int, ifname, cidr, newname string) error {
241-
248+
func setIfAttrs(pid int, ifname, cidr, cidr6, newname string) error {
242249
nsenterPath, err := osexec.LookPath("nsenter")
243250
if err != nil {
244251
return err
@@ -285,6 +292,17 @@ func setIfAttrs(pid int, ifname, cidr, newname string) error {
285292
}
286293
log.Infof("Output from ip assign: %v", assignIP)
287294

295+
if cidr6 != "" {
296+
out, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", ipPath,
297+
"-6", "address", "add", cidr6, "dev", newname).CombinedOutput()
298+
if err != nil {
299+
log.Errorf("unable to assign IPv6 %s to %s. Error: %s",
300+
cidr6, newname, err)
301+
return nil
302+
}
303+
log.Infof("Output of IPv6 assign: %v", out)
304+
}
305+
288306
// Finally, mark the link up
289307
bringUp, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", ipPath,
290308
"link", "set", "dev", newname, "up").CombinedOutput()
@@ -324,7 +342,7 @@ func addStaticRoute(pid int, subnet, intfName string) error {
324342
}
325343

326344
// setDefGw sets the default gateway for the container namespace
327-
func setDefGw(pid int, gw, intfName string) error {
345+
func setDefGw(pid int, gw, gw6, intfName string) error {
328346
nsenterPath, err := osexec.LookPath("nsenter")
329347
if err != nil {
330348
return err
@@ -338,10 +356,19 @@ func setDefGw(pid int, gw, intfName string) error {
338356
out, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", routePath, "add",
339357
"default", "gw", gw, intfName).CombinedOutput()
340358
if err != nil {
341-
log.Errorf("unable to set default gw %s. Error: %s - %s",
342-
gw, err, out)
359+
log.Errorf("unable to set default gw %s. Error: %s - %s", gw, err, out)
343360
return nil
344361
}
362+
363+
if gw6 != "" {
364+
out, err := osexec.Command(nsenterPath, "-t", nsPid, "-n", "-F", "--", routePath,
365+
"-6", "add", "default", "gw", gw6, intfName).CombinedOutput()
366+
if err != nil {
367+
log.Errorf("unable to set default IPv6 gateway %s. Error: %s - %s", gw6, err, out)
368+
return nil
369+
}
370+
}
371+
345372
return nil
346373
}
347374

@@ -430,13 +457,14 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int
430457
}
431458

432459
// Set interface attributes for the new port
433-
epErr = setIfAttrs(pid, ep.PortName, ep.IPAddress, pInfo.IntfName)
460+
epErr = setIfAttrs(pid, ep.PortName, ep.IPAddress, ep.IPv6Address, pInfo.IntfName)
434461
if epErr != nil {
435462
log.Errorf("Error setting interface attributes. Err: %v", epErr)
436463
setErrorResp(&resp, "Error setting interface attributes", epErr)
437464
return resp, epErr
438465
}
439466

467+
//TODO: Host access needs to be enabled for IPv6
440468
// if Gateway is not specified on the nw, use the host gateway
441469
gwIntf := pInfo.IntfName
442470
gw := ep.Gateway
@@ -446,7 +474,7 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int
446474
if err != nil {
447475
log.Errorf("Error setting host access. Err: %v", err)
448476
} else {
449-
err = setIfAttrs(pid, hostIf, hostIP, "host1")
477+
err = setIfAttrs(pid, hostIf, hostIP, "", "host1")
450478
if err != nil {
451479
log.Errorf("Move to pid %d failed", pid)
452480
} else {
@@ -465,7 +493,7 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int
465493
}
466494

467495
// Set default gateway
468-
epErr = setDefGw(pid, gw, gwIntf)
496+
epErr = setDefGw(pid, gw, ep.IPv6Gateway, gwIntf)
469497
if epErr != nil {
470498
log.Errorf("Error setting default gateway. Err: %v", epErr)
471499
setErrorResp(&resp, "Error setting default gateway", epErr)
@@ -474,7 +502,13 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int
474502

475503
resp.Result = 0
476504
resp.IPAddress = ep.IPAddress
505+
506+
if ep.IPv6Address != "" {
507+
resp.IPv6Address = ep.IPv6Address
508+
}
509+
477510
resp.EndpointID = pInfo.InfraContainerID
511+
478512
return resp, nil
479513
}
480514

‎mgmtfn/k8splugin/driver_test.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -116,12 +116,14 @@ func (s *NetSetup) TestNetSetup(c *C) {
116116
address := "192.168.68.68/24"
117117
defGW := "192.168.68.1"
118118
staticRoute := "192.168.32.0/24"
119+
ipv6Address := "2001::100/100"
120+
ipv6Gateway := "2001::1/100"
119121

120-
if err := setIfAttrs(s.pid, s.ifName, address, newName); err != nil {
122+
if err := setIfAttrs(s.pid, s.ifName, address, ipv6Address, newName); err != nil {
121123
c.Fatalf("setIfAttrs failed: %v", err)
122124
}
123125

124-
if err := setDefGw(s.pid, defGW, newName); err != nil {
126+
if err := setDefGw(s.pid, defGW, ipv6Gateway, newName); err != nil {
125127
c.Fatalf("setDefGw failed: %v", err)
126128
}
127129

‎test/systemtests/k8setup_test.go

+14-3
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ func (k *kubernetes) newContainer(node *node, containerID, name string, spec con
4747
}
4848
cont.eth0.ip = out
4949

50-
out, err = cont.node.exec.getIPv6Addr(cont, "eth0")
50+
out, err = k8master.exec.getIPv6Addr(cont, "eth0")
5151
if err == nil {
5252
cont.eth0.ipv6 = out
5353
}
@@ -206,8 +206,19 @@ func (k *kubernetes) getIPAddr(c *container, dev string) (string, error) {
206206
}
207207

208208
func (k *kubernetes) getIPv6Addr(c *container, dev string) (string, error) {
209-
/*FIXME: fix for k8 v6 */
210-
return "", nil
209+
out, err := k8master.tbnode.RunCommandWithOutput(fmt.Sprintf("kubectl exec %s ip addr show dev %s | grep 'inet6.*scope.*global' | head -1", c.containerID, dev))
210+
if err != nil {
211+
logrus.Errorf("Failed to get IPv6 for container %q", c.containerID)
212+
logrus.Println(out)
213+
}
214+
215+
parts := regexp.MustCompile(`\s+`).Split(strings.TrimSpace(out), -1)
216+
if len(parts) < 2 {
217+
return "", fmt.Errorf("Invalid output from container %q: %s", c.containerID, out)
218+
}
219+
220+
parts = strings.Split(parts[1], "/")
221+
return strings.TrimSpace(parts[0]), err
211222
}
212223

213224
func (k *kubernetes) getMACAddr(c *container, dev string) (string, error) {

‎test/systemtests/kubeadm_test.go

+16-3
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ func (k *kubePod) newContainer(node *node, containerID, name string, spec contai
5656
}
5757
cont.eth0.ip = out
5858

59-
out, err = cont.node.exec.getIPv6Addr(cont, "eth0")
59+
out, err = k8sMaster.exec.getIPv6Addr(cont, "eth0")
6060
if err == nil {
6161
cont.eth0.ipv6 = out
6262
}
@@ -213,8 +213,21 @@ func (k *kubePod) getIPAddr(c *container, dev string) (string, error) {
213213
}
214214

215215
func (k *kubePod) getIPv6Addr(c *container, dev string) (string, error) {
216-
/*FIXME: fix for k8 v6 */
217-
return "", nil
216+
out, err := k8sMaster.tbnode.RunCommandWithOutput(
217+
fmt.Sprintf("kubectl exec %s ip addr show dev %s | grep 'inet6.*scope.*global' | head -1",
218+
c.containerID, dev))
219+
if err != nil {
220+
logrus.Errorf("Failed to get IPv6 for container %q", c.containerID)
221+
logrus.Println(out)
222+
}
223+
224+
parts := regexp.MustCompile(`\s+`).Split(strings.TrimSpace(out), -1)
225+
if len(parts) < 2 {
226+
return "", fmt.Errorf("Invalid output from container %q: %s", c.containerID, out)
227+
}
228+
229+
parts = strings.Split(parts[1], "/")
230+
return strings.TrimSpace(parts[0]), err
218231
}
219232

220233
func (k *kubePod) getMACAddr(c *container, dev string) (string, error) {

0 commit comments

Comments
 (0)