Skip to content

Commit 1b0c58f

Browse files
authored
Merge pull request #305 from hhyasdf/improve/make-cni-conf-configurable
make cni conf configurable
2 parents 433e71b + 6d073ee commit 1b0c58f

File tree

8 files changed

+53
-217
lines changed

8 files changed

+53
-217
lines changed
+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: hybridnet-cni-conf
5+
namespace: kube-system
6+
data:
7+
cni-config: |-
8+
{
9+
"name":"hybridnet",
10+
"cniVersion":"0.3.1",
11+
"plugins":[
12+
{
13+
"type":"hybridnet",
14+
"server_socket":"/run/cni/hybridnet.sock"
15+
},{
16+
"type": "bandwidth",
17+
"capabilities": {"bandwidth": true}
18+
}
19+
]
20+
}

charts/hybridnet/templates/daemonsets.yaml

+13-24
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ spec:
3333
name: cni-conf
3434
- mountPath: /opt/cni/bin
3535
name: cni-bin
36+
- mountPath: /hybridnet/00-hybridnet.conflist
37+
name: cni-conf-source
38+
subPath: cni-config
39+
env:
40+
- name: NEEDED_COMMUNITY_CNI_PLUGINS
41+
value: {{ .Values.daemon.neededCommunityCNIPlugins }}
3642
containers:
3743
- name: cni-daemon
3844
image: "{{ .Values.images.registryURL }}/{{ .Values.images.hybridnet.image }}:{{ .Values.images.hybridnet.tag }}"
@@ -86,18 +92,6 @@ spec:
8692
name: host-modules
8793
- mountPath: /run/xtables.lock
8894
name: xtables-lock
89-
- mountPath: /var/run/netns
90-
mountPropagation: HostToContainer
91-
name: host-var-run-netns
92-
- mountPath: /var/run/docker/netns
93-
mountPropagation: HostToContainer
94-
name: host-var-docker-netns
95-
- mountPath: /run/netns
96-
mountPropagation: HostToContainer
97-
name: host-run-netns
98-
- mountPath: /run/docker/netns
99-
mountPropagation: HostToContainer
100-
name: host-docker-netns
10195
# TODO: add liveness probe
10296
{{ if .Values.daemon.enableNetworkPolicy }}
10397
- name: policy
@@ -137,15 +131,10 @@ spec:
137131
hostPath:
138132
path: /run/xtables.lock
139133
type: FileOrCreate
140-
- name: host-run-netns
141-
hostPath:
142-
path: /run/netns
143-
- name: host-docker-netns
144-
hostPath:
145-
path: /run/docker/netns
146-
- name: host-var-run-netns
147-
hostPath:
148-
path: /var/run/netns
149-
- name: host-var-docker-netns
150-
hostPath:
151-
path: /var/run/docker/netns
134+
- name: cni-conf-source
135+
configMap:
136+
name: hybridnet-cni-conf
137+
items:
138+
- key: cni-config
139+
path: cni-config
140+

charts/hybridnet/values.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,12 @@ daemon:
6161
## randomly. If is is not empty, the first result matches any of the CIDRs will be chose as VTEP address.
6262
vtepAddressCIDRs: "0.0.0.0/0,::/0"
6363

64+
# -- The community CNI plugins needed to be copied by hybridnet from inside container to the /opt/cni/bin/ directory of host
65+
neededCommunityCNIPlugins: "loopback,bandwidth"
66+
67+
# -- The name of hybridnet CNI conf file generated in the /etc/cni/net.d/ directory of host
68+
cniConfName: "06-hybridnet.conflist"
69+
6470
# -- Whether pod IP of stateful workloads will be retained by default. true or false
6571
## Ref: https://github.com/alibaba/hybridnet/wiki/Static-pod-ip-addresses-for-StatefulSet
6672
defaultIPRetain: true

dist/images/00-hybridnet.conflist

+1-5
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,6 @@
55
{
66
"type":"hybridnet",
77
"server_socket":"/run/cni/hybridnet.sock"
8-
},
9-
{
10-
"type": "bandwidth",
11-
"capabilities": {"bandwidth": true}
128
}
139
]
14-
}
10+
}

dist/images/install-cni.sh

+13-10
Original file line numberDiff line numberDiff line change
@@ -33,16 +33,19 @@ fi
3333
CNI_BIN_SRC=/hybridnet/hybridnet
3434
CNI_BIN_DST=/opt/cni/bin/hybridnet
3535

36-
CNI_CONF_SRC=/hybridnet/00-hybridnet.conflist
37-
CNI_CONF_DST=/etc/cni/net.d/00-hybridnet.conflist
36+
COMMUNITY_CNI_PLUGINS_SRC_DIR=/cni-plugins
37+
COMMUNITY_CNI_PLUGINS_DST_DIR=/opt/cni/bin
3838

39-
LOOPBACK_BIN_SRC=/cni-plugins/loopback
40-
LOOPBACK_BIN_DST=/opt/cni/bin/loopback
39+
PLUGINS=${NEEDED_COMMUNITY_CNI_PLUGINS-"loopback"}
40+
for plugin in ${PLUGINS//,/ }
41+
do
42+
cp -f $COMMUNITY_CNI_PLUGINS_SRC_DIR/"$plugin" $COMMUNITY_CNI_PLUGINS_DST_DIR/"$plugin"
43+
done
4144

42-
BANDWIDTH_BIN_SRC=/cni-plugins/bandwidth
43-
BANDWIDTH_BIN_DST=/opt/cni/bin/bandwidth
44-
45-
cp -f $LOOPBACK_BIN_SRC $LOOPBACK_BIN_DST
46-
cp -f $BANDWIDTH_BIN_SRC $BANDWIDTH_BIN_DST
4745
cp -f $CNI_BIN_SRC $CNI_BIN_DST
48-
cp -f $CNI_CONF_SRC $CNI_CONF_DST
46+
47+
# clean the out-of-date configuration
48+
rm -rf /etc/cni/net.d/*-hybridnet.conflist
49+
50+
CNI_CONF_DST=/etc/cni/net.d/${CNI_CONF_NAME-"06-hybridnet.conflist"}
51+
cp -f "${CNI_CONF_SRC-"/hybridnet/00-hybridnet.conflist"}" "$CNI_CONF_DST"

pkg/constants/netns.go

-22
This file was deleted.

pkg/daemon/controller/controller.go

-2
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,6 @@ type CtrlHub struct {
9292

9393
nodeIPCache *NodeIPCache
9494

95-
upgradeWorkDone bool
96-
9795
logger logr.Logger
9896
}
9997

pkg/daemon/controller/ipinstance.go

-154
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,7 @@ package controller
1919
import (
2020
"context"
2121
"fmt"
22-
"io/ioutil"
2322
"net"
24-
"os"
25-
"strings"
2623
"time"
2724

2825
ctrl "sigs.k8s.io/controller-runtime"
@@ -32,21 +29,14 @@ import (
3229
"sigs.k8s.io/controller-runtime/pkg/predicate"
3330
"sigs.k8s.io/controller-runtime/pkg/source"
3431

35-
"github.com/go-logr/logr"
36-
3732
"k8s.io/apimachinery/pkg/types"
3833

3934
"sigs.k8s.io/controller-runtime/pkg/client"
4035
"sigs.k8s.io/controller-runtime/pkg/log"
4136
"sigs.k8s.io/controller-runtime/pkg/reconcile"
4237

43-
"github.com/containernetworking/plugins/pkg/ip"
44-
"github.com/containernetworking/plugins/pkg/ns"
45-
"github.com/vishvananda/netlink"
46-
4738
networkingv1 "github.com/alibaba/hybridnet/pkg/apis/networking/v1"
4839
"github.com/alibaba/hybridnet/pkg/constants"
49-
"github.com/alibaba/hybridnet/pkg/daemon/containernetwork"
5040
daemonutils "github.com/alibaba/hybridnet/pkg/daemon/utils"
5141
)
5242

@@ -64,13 +54,6 @@ func (r *ipInstanceReconciler) Reconcile(ctx context.Context, request reconcile.
6454
logger.V(2).Info("IPInstance information reconciled", "time", endTime)
6555
}()
6656

67-
if !r.ctrlHubRef.upgradeWorkDone {
68-
if err := ensureExistPodConfigs(r.ctrlHubRef.config.LocalDirectTableNum, logger); err != nil {
69-
return reconcile.Result{Requeue: true}, fmt.Errorf("failed to ensure exist pod config: %v", err)
70-
}
71-
r.ctrlHubRef.upgradeWorkDone = true
72-
}
73-
7457
ipInstanceList := &networkingv1.IPInstanceList{}
7558
if err := r.List(ctx, ipInstanceList,
7659
client.MatchingLabels{constants.LabelNode: r.ctrlHubRef.config.NodeName}); err != nil {
@@ -219,140 +202,3 @@ func (r *ipInstanceReconciler) SetupWithManager(mgr ctrl.Manager) error {
219202

220203
return nil
221204
}
222-
223-
// TODO: update logic, need to be removed further
224-
func ensureExistPodConfigs(localDirectTableNum int, logger logr.Logger) error {
225-
var netnsPaths []string
226-
var netnsDir string
227-
228-
if daemonutils.ValidDockerNetnsDir(constants.DockerNetnsDir) {
229-
netnsDir = constants.DockerNetnsDir
230-
} else {
231-
logger.Info("docker netns path not exist, try containerd netns path",
232-
"docker-netns-path", constants.DockerNetnsDir,
233-
"containerd-netns-path", constants.ContainerdNetnsDir)
234-
netnsDir = constants.ContainerdNetnsDir
235-
}
236-
237-
files, err := ioutil.ReadDir(netnsDir)
238-
if err != nil && !os.IsNotExist(err) {
239-
return err
240-
}
241-
242-
for _, f := range files {
243-
if f.Name() == "default" {
244-
continue
245-
}
246-
fpath := netnsDir + "/" + f.Name()
247-
if daemonutils.IsProcFS(fpath) || daemonutils.IsNsFS(fpath) {
248-
netnsPaths = append(netnsPaths, fpath)
249-
}
250-
}
251-
252-
logger.Info("load exist netns", "netns-path", netnsPaths)
253-
254-
var hostLinkIndex int
255-
allocatedIPs := map[networkingv1.IPVersion]*daemonutils.IPInfo{}
256-
257-
for _, netns := range netnsPaths {
258-
nsHandler, err := ns.GetNS(netns)
259-
if err != nil {
260-
return fmt.Errorf("get ns error: %v", err)
261-
}
262-
263-
err = nsHandler.Do(func(netNS ns.NetNS) error {
264-
link, err := netlink.LinkByName(constants.ContainerNicName)
265-
if err != nil {
266-
return fmt.Errorf("get container interface error: %v", err)
267-
}
268-
269-
v4Addrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
270-
if err != nil {
271-
return fmt.Errorf("failed to get v4 container interface addr: %v", err)
272-
}
273-
274-
var v4GatewayIP net.IP
275-
if len(v4Addrs) == 0 {
276-
allocatedIPs[networkingv1.IPv4] = nil
277-
} else {
278-
defaultRoute, err := daemonutils.GetDefaultRoute(netlink.FAMILY_V4)
279-
if err != nil {
280-
return fmt.Errorf("failed to get ipv4 default route: %v", err)
281-
}
282-
v4GatewayIP = defaultRoute.Gw
283-
}
284-
285-
for _, addr := range v4Addrs {
286-
allocatedIPs[networkingv1.IPv4] = &daemonutils.IPInfo{
287-
Addr: addr.IP,
288-
Gw: v4GatewayIP,
289-
}
290-
}
291-
292-
v6Addrs, err := netlink.AddrList(link, netlink.FAMILY_V6)
293-
if err != nil {
294-
return fmt.Errorf("failed to get v6 container interface addr: %v", err)
295-
}
296-
297-
var v6GatewayIP net.IP
298-
if len(v6Addrs) == 0 {
299-
allocatedIPs[networkingv1.IPv6] = nil
300-
} else {
301-
defaultRoute, err := daemonutils.GetDefaultRoute(netlink.FAMILY_V6)
302-
if err != nil {
303-
return fmt.Errorf("failed to get ipv6 default route: %v", err)
304-
}
305-
v6GatewayIP = defaultRoute.Gw
306-
}
307-
308-
for _, addr := range v6Addrs {
309-
allocatedIPs[networkingv1.IPv6] = &daemonutils.IPInfo{
310-
Addr: addr.IP,
311-
Gw: v6GatewayIP,
312-
}
313-
}
314-
315-
_, hostLinkIndex, err = ip.GetVethPeerIfindex(constants.ContainerNicName)
316-
if err != nil {
317-
return fmt.Errorf("get host link index error: %v", err)
318-
}
319-
320-
return nil
321-
})
322-
323-
if err != nil {
324-
logger.Error(err, "get pod addresses and host link index error")
325-
}
326-
327-
if hostLinkIndex == 0 {
328-
continue
329-
}
330-
331-
hostLink, err := netlink.LinkByIndex(hostLinkIndex)
332-
if err != nil {
333-
return fmt.Errorf("failed to get host link by index %v: %v", hostLinkIndex, err)
334-
}
335-
336-
// this container doesn't belong to k8s
337-
if !strings.HasSuffix(hostLink.Attrs().Name, "_h") {
338-
continue
339-
}
340-
341-
if hostLink.Attrs().MasterIndex != 0 {
342-
bridge, err := netlink.LinkByIndex(hostLink.Attrs().MasterIndex)
343-
if err != nil {
344-
return fmt.Errorf("failed to get bridge by index %v: %v", hostLink.Attrs().MasterIndex, err)
345-
}
346-
347-
if err := netlink.LinkDel(bridge); err != nil {
348-
return fmt.Errorf("failed to delete bridge %v: %v", bridge.Attrs().Name, err)
349-
}
350-
}
351-
352-
if err := containernetwork.ConfigureHostNic(hostLink.Attrs().Name, allocatedIPs, localDirectTableNum); err != nil {
353-
return fmt.Errorf("failed to reconfigure host nic %v: %v", hostLink.Attrs().Name, err)
354-
}
355-
}
356-
357-
return nil
358-
}

0 commit comments

Comments
 (0)