Skip to content

Commit 6ae36c1

Browse files
author
Vipin Jain
committed
kubernetes style infra-container support
1 parent a65ae90 commit 6ae36c1

15 files changed

+383
-50
lines changed

‎crtclient/crtclient.go

+8-6
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,14 @@ type Config struct {
2323
}
2424

2525
type ContainerEpContext struct {
26-
NewContName string
27-
CurrContName string
28-
InterfaceId string
29-
IpAddress string
30-
SubnetLen uint
31-
DefaultGw string
26+
NewContName string
27+
NewAttachUUID string
28+
CurrContName string
29+
CurrAttachUUID string
30+
InterfaceId string
31+
IpAddress string
32+
SubnetLen uint
33+
DefaultGw string
3234
}
3335

3436
type ContainerIf interface {

‎crtclient/docker/docker.go

+23-12
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,17 @@ func (d *Docker) Init(config *crtclient.Config) error {
6464
func (d *Docker) Deinit() {
6565
}
6666

67-
func (d *Docker) getContPid(contName string) (string, error) {
68-
contInfo, err := d.Client.InspectContainer(contName)
67+
func (d *Docker) getContPid(ctx *crtclient.ContainerEpContext) (string, error) {
68+
69+
contNameOrId := ctx.NewContName
70+
if ctx.NewAttachUUID != "" {
71+
contNameOrId = ctx.NewAttachUUID
72+
}
73+
74+
contInfo, err := d.Client.InspectContainer(contNameOrId)
6975
if err != nil {
76+
log.Printf("unable to get container info for '%s' \n",
77+
contNameOrId)
7078
return "", errors.New("couldn't obtain container info")
7179
}
7280

@@ -97,12 +105,14 @@ func setIfNs(ifname string, pid int) error {
97105
// Note: most of the work in this function is a temporary workaround for
98106
// what docker daemon would eventually do; in the meanwhile
99107
// the essense of the logic is borrowed from pipework
100-
func (d *Docker) moveIfToContainer(ifId string, contName string) error {
108+
func (d *Docker) moveIfToContainer(ctx *crtclient.ContainerEpContext) error {
101109

102110
// log.Printf("Moving interface '%s' into container '%s' \n", ifId, contName)
103111

104-
contPid, err := d.getContPid(contName)
112+
contPid, err := d.getContPid(ctx)
105113
if err != nil {
114+
log.Printf("error '%s' querying container name %s, uuid %s\n",
115+
err, ctx.NewContName, ctx.NewAttachUUID)
106116
return err
107117
}
108118

@@ -131,18 +141,19 @@ func (d *Docker) moveIfToContainer(ifId string, contName string) error {
131141
}
132142

133143
intPid, _ := strconv.Atoi(contPid)
134-
err = setIfNs(ifId, intPid)
144+
err = setIfNs(ctx.InterfaceId, intPid)
135145
if err != nil {
136146
log.Printf("err '%s' moving if '%s' into container '%s' namespace\n",
137-
err, ifId, contName)
147+
err, ctx.InterfaceId, ctx.NewContName)
138148
return err
139149
}
140150

141151
return err
142152
}
143153

144-
func (d *Docker) cleanupNetns(contName string) error {
145-
contPid, err := d.getContPid(contName)
154+
func (d *Docker) cleanupNetns(ctx *crtclient.ContainerEpContext) error {
155+
156+
contPid, err := d.getContPid(ctx)
146157
if err != nil {
147158
return err
148159
}
@@ -169,7 +180,7 @@ func (d *Docker) configureIfAddress(ctx *crtclient.ContainerEpContext) error {
169180
errors.New("Subnet mask unspecified \n")
170181
}
171182
172-
contPid, err := d.getContPid(ctx.NewContName)
183+
contPid, err := d.getContPid(ctx)
173184
if err != nil {
174185
return err
175186
}
@@ -244,7 +255,7 @@ func (d *Docker) configureIfAddress(ctx *crtclient.ContainerEpContext) error {
244255
errors.New("Subnet mask unspecified \n")
245256
}
246257

247-
contPid, err := d.getContPid(ctx.NewContName)
258+
contPid, err := d.getContPid(ctx)
248259
if err != nil {
249260
return err
250261
}
@@ -274,7 +285,7 @@ func (d *Docker) configureIfAddress(ctx *crtclient.ContainerEpContext) error {
274285
// before the container becomes active
275286
func (d *Docker) AttachEndpoint(ctx *crtclient.ContainerEpContext) error {
276287

277-
err := d.moveIfToContainer(ctx.InterfaceId, ctx.NewContName)
288+
err := d.moveIfToContainer(ctx)
278289
if err != nil {
279290
return err
280291
}
@@ -287,7 +298,7 @@ func (d *Docker) AttachEndpoint(ctx *crtclient.ContainerEpContext) error {
287298
// configure policies: acl/qos for the container on the host
288299

289300
// cleanup intermediate things (overdoing it?)
290-
d.cleanupNetns(ctx.NewContName)
301+
d.cleanupNetns(ctx)
291302

292303
return err
293304
}

‎drivers/ovsdriver.go

+1
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,7 @@ func (d *OvsDriver) CreateEndpoint(id string) error {
499499
Id: id,
500500
PortName: portName,
501501
NetId: epCfg.NetId,
502+
AttachUUID: epCfg.AttachUUID,
502503
ContName: epCfg.ContName,
503504
IpAddress: epCfg.IpAddress,
504505
IntfName: intfName,

‎drivers/ovsendpointstate.go

+2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ type OvsCfgEndpointState struct {
3030
Id string `json:"id"`
3131
NetId string `json:"netId"`
3232
ContName string `json:"contName"`
33+
AttachUUID string `json:"attachUUID"`
3334
IpAddress string `json:"ipAddress"`
3435
HomingHost string `json:"homingHost"`
3536
IntfName string `json:"intfName"`
@@ -73,6 +74,7 @@ type OvsOperEndpointState struct {
7374
Id string `json:"id"`
7475
NetId string `json:"netId"`
7576
ContName string `json:"contName"`
77+
AttachUUID string `json:"attachUUID"`
7678
IpAddress string `json:"ipAddress"`
7779
PortName string `json:"portName"`
7880
HomingHost string `json:"homingHost"`

‎examples/container_bindings/multiple_vxlan_nets.json

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
2+
{
3+
"Hosts" : [{
4+
"Name" : "host1",
5+
"VtepIp" : "192.168.2.10"
6+
},
7+
{
8+
"Name" : "host2",
9+
"VtepIp" : "192.168.2.11"
10+
}],
11+
"Tenants" : [ {
12+
"Name" : "tenant-one",
13+
"DefaultNetType" : "vxlan",
14+
"SubnetPool" : "11.1.0.0/16",
15+
"AllocSubnetLen" : 24,
16+
"Vxlans" : "10001-20000",
17+
"Networks" : [ {
18+
"Name" : "orange",
19+
"Endpoints" : [ {
20+
"Container" : "myPod1"
21+
},
22+
{
23+
"Container" : "myContainer3"
24+
} ]
25+
},
26+
{
27+
"Name" : "purple",
28+
"Endpoints" : [ {
29+
"Container" : "myPod2"
30+
},
31+
{
32+
"Container" : "myContainer4"
33+
} ]
34+
} ]
35+
} ]
36+
}

‎examples/container_bindings/multiple_vxlan_nets_host_bindings.json

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
2+
[{
3+
"Container" : "myPod1",
4+
"Host" : "host1"
5+
},
6+
{
7+
"Container" : "myPod2",
8+
"Host" : "host1"
9+
},
10+
{
11+
"Container" : "myContainer3",
12+
"Host" : "host2"
13+
},
14+
{
15+
"Container" : "myContainer4",
16+
"Host" : "host2"
17+
}]

‎examples/container_bindings/multiple_vxlan_nets_infra_container_bindings.json

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
myPod1:myContainer1
2+
myPod2:myContainer2

‎netd.go

+71-24
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ const (
4242

4343
type cliOpts struct {
4444
hostLabel string
45+
nativeInteg bool
4546
publishVtep bool
4647
}
4748

@@ -213,6 +214,7 @@ func getEndpointContainerContext(state core.StateDriver, epId string) (
213214
return &epCtx, nil
214215
}
215216
epCtx.NewContName = epCfg.ContName
217+
epCtx.NewAttachUUID = epCfg.AttachUUID
216218

217219
cfgNet := &drivers.OvsCfgNetworkState{StateDriver: state}
218220
err = cfgNet.Read(epCfg.NetId)
@@ -230,6 +232,7 @@ func getEndpointContainerContext(state core.StateDriver, epId string) (
230232
epCtx.CurrContName = operEp.ContName
231233
epCtx.InterfaceId = operEp.PortName
232234
epCtx.IpAddress = operEp.IpAddress
235+
epCtx.CurrAttachUUID = operEp.AttachUUID
233236

234237
return &epCtx, err
235238
}
@@ -264,21 +267,47 @@ func getContainerEpContextByContName(state core.StateDriver, contName string) (
264267
return epCtxs[:idx], nil
265268
}
266269

270+
func contAttachPointAdded(epCtx *crtclient.ContainerEpContext) bool {
271+
if epCtx.CurrAttachUUID == "" && epCtx.NewAttachUUID != "" {
272+
return true
273+
}
274+
if epCtx.CurrContName == "" && epCtx.NewContName != "" {
275+
return true
276+
}
277+
return false
278+
}
279+
280+
func contAttachPointDeleted(epCtx *crtclient.ContainerEpContext) bool {
281+
if epCtx.CurrAttachUUID != "" && epCtx.NewAttachUUID == "" {
282+
return true
283+
}
284+
if epCtx.CurrContName != "" && epCtx.NewContName == "" {
285+
return true
286+
}
287+
return false
288+
}
289+
267290
func processEpEvent(netPlugin *plugin.NetPlugin, crt *crt.Crt,
268291
epId string, preValue string, opts cliOpts) (err error) {
269292

293+
deleteOp := false
270294
homingHost := ""
271295
vtepIp := ""
296+
297+
epCfg := &drivers.OvsCfgEndpointState{StateDriver: netPlugin.StateDriver}
298+
err = epCfg.Read(epId)
272299
if preValue == "" {
273-
epCfg := &drivers.OvsCfgEndpointState{StateDriver: netPlugin.StateDriver}
274-
err = epCfg.Read(epId)
275300
if err != nil {
276301
log.Printf("Failed to read config for ep '%s' \n", epId)
277302
return
278303
}
304+
279305
homingHost = epCfg.HomingHost
280306
vtepIp = epCfg.VtepIp
281307
} else {
308+
if err != nil {
309+
deleteOp = true
310+
}
282311
epOper := &drivers.OvsOperEndpointState{StateDriver: netPlugin.StateDriver}
283312
err = epOper.Read(epId)
284313
if err != nil {
@@ -305,7 +334,7 @@ func processEpEvent(netPlugin *plugin.NetPlugin, crt *crt.Crt,
305334
// log.Printf("read endpoint context: %s \n", contEpContext)
306335

307336
operStr := ""
308-
if preValue != "" {
337+
if deleteOp {
309338
err = netPlugin.DeleteEndpoint(epId)
310339
operStr = "delete"
311340
} else {
@@ -320,8 +349,7 @@ func processEpEvent(netPlugin *plugin.NetPlugin, crt *crt.Crt,
320349
log.Printf("Endpoint operation %s succeeded", operStr)
321350

322351
// attach or detach an endpoint to a container
323-
if preValue != "" ||
324-
(contEpContext.NewContName == "" && contEpContext.CurrContName != "") {
352+
if deleteOp || contAttachPointDeleted(contEpContext) {
325353
err = crt.ContainerIf.DetachEndpoint(contEpContext)
326354
if err != nil {
327355
log.Printf("Endpoint detach container '%s' from ep '%s' failed . "+
@@ -331,7 +359,7 @@ func processEpEvent(netPlugin *plugin.NetPlugin, crt *crt.Crt,
331359
contEpContext.CurrContName, epId)
332360
}
333361
}
334-
if preValue == "" && contEpContext.NewContName != "" {
362+
if !deleteOp && contAttachPointAdded(contEpContext) {
335363
// re-read post ep updated state
336364
newContEpContext, err1 := getEndpointContainerContext(
337365
netPlugin.StateDriver, epId)
@@ -392,27 +420,22 @@ func handleEtcdEvents(netPlugin *plugin.NetPlugin, crt *crt.Crt,
392420
retErr <- nil
393421
}
394422

395-
func handleContainerStart(netPlugin *plugin.NetPlugin, crt *crt.Crt,
396-
contId string) error {
397-
var err error
398-
var epContexts []crtclient.ContainerEpContext
423+
func attachContainer(stateDriver core.StateDriver, crt *crt.Crt, contName string) error {
399424

400-
contName, err := crt.GetContainerName(contId)
401-
if err != nil {
402-
log.Printf("Could not find container name from container id %s \n", contId)
403-
return err
404-
}
405-
406-
epContexts, err = getContainerEpContextByContName(netPlugin.StateDriver,
407-
contName)
425+
epContexts, err := getContainerEpContextByContName(stateDriver, contName)
408426
if err != nil {
409427
log.Printf("Error '%s' getting Ep context for container %s \n",
410428
err, contName)
411429
return err
412430
}
413431

414432
for _, epCtx := range epContexts {
415-
log.Printf("## trying attach on epctx %v \n", epCtx)
433+
if epCtx.NewAttachUUID != "" || epCtx.InterfaceId == "" {
434+
log.Printf("## skipping attach on epctx %v \n", epCtx)
435+
continue
436+
} else {
437+
log.Printf("## trying attach on epctx %v \n", epCtx)
438+
}
416439
err = crt.AttachEndpoint(&epCtx)
417440
if err != nil {
418441
log.Printf("Error '%s' attaching container to the network \n", err)
@@ -423,6 +446,24 @@ func handleContainerStart(netPlugin *plugin.NetPlugin, crt *crt.Crt,
423446
return nil
424447
}
425448

449+
func handleContainerStart(netPlugin *plugin.NetPlugin, crt *crt.Crt,
450+
contId string) error {
451+
// var epContexts []crtclient.ContainerEpContext
452+
453+
contName, err := crt.GetContainerName(contId)
454+
if err != nil {
455+
log.Printf("Could not find container name from container id %s \n", contId)
456+
return err
457+
}
458+
459+
err = attachContainer(netPlugin.StateDriver, crt, contName)
460+
if err != nil {
461+
log.Printf("error attaching container err \n", err)
462+
}
463+
464+
return err
465+
}
466+
426467
func handleDockerEvents(event *dockerclient.Event, retErr chan error,
427468
args ...interface{}) {
428469
var err error
@@ -474,11 +515,13 @@ func handleEvents(netPlugin *plugin.NetPlugin, crt *crt.Crt,
474515

475516
go handleEtcdEvents(netPlugin, crt, rsps, stop, recvErr, opts)
476517

477-
// start docker client and handle docker events
478-
// wait on error chan for problems handling the docker events
479-
dockerCrt := crt.ContainerIf.(*docker.Docker)
480-
dockerCrt.Client.StartMonitorEvents(handleDockerEvents, recvErr,
481-
netPlugin, crt)
518+
if !opts.nativeInteg {
519+
// start docker client and handle docker events
520+
// wait on error chan for problems handling the docker events
521+
dockerCrt := crt.ContainerIf.(*docker.Docker)
522+
dockerCrt.Client.StartMonitorEvents(handleDockerEvents, recvErr,
523+
netPlugin, crt)
524+
}
482525

483526
// XXX: todo, restore any config that might have been created till this
484527
// point
@@ -512,6 +555,10 @@ func main() {
512555
"host-label",
513556
defHostLabel,
514557
"label used to identify endpoints homed for this host, default is host name")
558+
flagSet.BoolVar(&opts.nativeInteg,
559+
"native-integration",
560+
false,
561+
"do not listen to container runtime events, because the events are natively integrated into their call sequence and external integration is not required")
515562
flagSet.BoolVar(&opts.publishVtep,
516563
"publish-vtep",
517564
false,

0 commit comments

Comments
 (0)