Skip to content

Commit d1385c6

Browse files
changluyichangluyi
committed
support metallb underlay
Signed-off-by: clyi <[email protected]>
1 parent 3f68a03 commit d1385c6

26 files changed

+1394
-26
lines changed

‎.github/workflows/build-x86-image.yaml

+138
Original file line numberDiff line numberDiff line change
@@ -3144,6 +3144,143 @@ jobs:
31443144
name: kube-ovn-connectivity-e2e-${{ matrix.mode }}-ko-log
31453145
path: kube-ovn-connectivity-e2e-${{ matrix.mode }}-ko-log.tar.gz
31463146

3147+
3148+
kube-ovn-metallb-e2e:
3149+
name: OVN METALLB E2E
3150+
needs:
3151+
- build-kube-ovn
3152+
- build-e2e-binaries
3153+
runs-on: ubuntu-24.04
3154+
timeout-minutes: 15
3155+
steps:
3156+
- uses: jlumbroso/[email protected]
3157+
with:
3158+
android: true
3159+
dotnet: true
3160+
haskell: true
3161+
docker-images: false
3162+
large-packages: false
3163+
tool-cache: false
3164+
swap-storage: false
3165+
3166+
- uses: actions/checkout@v4
3167+
3168+
- name: Create the default branch directory
3169+
if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
3170+
run: mkdir -p test/e2e/source
3171+
3172+
- name: Check out the default branch
3173+
if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
3174+
uses: actions/checkout@v4
3175+
with:
3176+
ref: ${{ github.event.repository.default_branch }}
3177+
fetch-depth: 1
3178+
path: test/e2e/source
3179+
3180+
- name: Export E2E directory
3181+
run: |
3182+
if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
3183+
echo "E2E_DIR=." >> "$GITHUB_ENV"
3184+
else
3185+
echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
3186+
fi
3187+
3188+
- uses: actions/setup-go@v5
3189+
id: setup-go
3190+
with:
3191+
go-version-file: ${{ env.E2E_DIR }}/go.mod
3192+
check-latest: true
3193+
cache: false
3194+
3195+
- name: Export Go full version
3196+
run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"
3197+
3198+
- name: Go cache
3199+
uses: actions/cache/restore@v4
3200+
with:
3201+
path: |
3202+
~/.cache/go-build
3203+
~/go/pkg/mod
3204+
key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
3205+
restore-keys: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-
3206+
3207+
- name: Install kind
3208+
uses: helm/[email protected]
3209+
with:
3210+
version: ${{ env.KIND_VERSION }}
3211+
install_only: true
3212+
3213+
- name: Install ginkgo
3214+
working-directory: ${{ env.E2E_DIR }}
3215+
run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo
3216+
3217+
- name: Download kube-ovn image
3218+
uses: actions/download-artifact@v4
3219+
with:
3220+
name: kube-ovn
3221+
3222+
- name: Load images
3223+
run: docker load -i kube-ovn.tar
3224+
3225+
- name: Create kind cluster
3226+
run: |
3227+
pipx install jinjanator
3228+
make kind-init
3229+
3230+
- name: Install Kube-OVN
3231+
id: install
3232+
run: make kind-install-metallb-pool-from-underlay
3233+
3234+
- name: Run Ovn Metallb and Kube-OVN Combine E2E
3235+
id: kube-ovn-metallb-e2e
3236+
working-directory: ${{ env.E2E_DIR }}
3237+
env:
3238+
E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
3239+
run: make kube-ovn-metallb-e2e
3240+
3241+
- name: Collect k8s events
3242+
if: failure() && ( steps.ovn-metallb-e2e.conclusion == 'failure')
3243+
run: |
3244+
kubectl get events -A -o yaml > kube-ovn-metallb-e2e-events.yaml
3245+
tar zcf kube-ovn-metallb-e2e-events.tar.gz kube-ovn-metallb-e2e-events.yaml
3246+
3247+
- name: Upload k8s events
3248+
uses: actions/upload-artifact@v4
3249+
if: failure() && (steps.kube-ovn-metallb-e2e.conclusion == 'failure')
3250+
with:
3251+
name: kube-ovn-metallb-e2e-events
3252+
path: kube-ovn-metallb-e2e-events.tar.gz
3253+
3254+
- name: Collect apiserver audit logs
3255+
if: failure() && (steps.kube-ovn-metallb-e2e.conclusion == 'failure')
3256+
run: |
3257+
docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
3258+
tar zcf kube-ovn-metallb-e2e-audit-log.tar.gz kube-apiserver-audit.log
3259+
3260+
- name: Upload apiserver audit logs
3261+
uses: actions/upload-artifact@v4
3262+
if: failure() && (steps.kube-ovn-metallb-e2e.conclusion == 'failure')
3263+
with:
3264+
name: kube-ovn-metallb-e2e-audit-log
3265+
path: kube-ovn-metallb-e2e-audit-log.tar.gz
3266+
3267+
- name: kubectl ko log
3268+
if: failure() && (steps.kube-ovn-metallb-e2e.conclusion == 'failure')
3269+
run: |
3270+
make kubectl-ko-log
3271+
mv kubectl-ko-log.tar.gz kube-ovn-metallb-e2e-ko-log.tar.gz
3272+
3273+
- name: upload kubectl ko log
3274+
uses: actions/upload-artifact@v4
3275+
if: failure() && (steps.kube-ovn-metallb-e2e.conclusion == 'failure')
3276+
with:
3277+
name: kube-ovn-metallb-e2e-ko-log
3278+
path: kube-ovn-metallb-e2e-ko-log.tar.gz
3279+
3280+
- name: Check kube ovn pod restarts
3281+
if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.kube-ovn-metallb-e2e.conclusion == 'failure')) }}
3282+
run: make check-kube-ovn-pod-restarts
3283+
31473284
push:
31483285
name: Push Images
31493286
needs:
@@ -3153,6 +3290,7 @@ jobs:
31533290
- kube-ovn-conformance-e2e
31543291
- kube-ovn-ic-conformance-e2e
31553292
- kube-ovn-ipsec-e2e
3293+
- kube-ovn-metallb-e2e
31563294
- multus-conformance-e2e
31573295
- vpc-egress-gateway-e2e
31583296
- ovn-vpc-nat-gw-conformance-e2e

‎Makefile

+8
Original file line numberDiff line numberDiff line change
@@ -801,6 +801,9 @@ kind-install-metallb:
801801
--set speaker.frr.image.tag=$(FRR_VERSION)
802802
$(call kubectl_wait_exist_and_ready,metallb-system,deployment,metallb-controller)
803803
$(call kubectl_wait_exist_and_ready,metallb-system,daemonset,metallb-speaker)
804+
805+
.PHONY: kind-configure-metallb
806+
kind-configure-metallb:
804807
@metallb_pool=$(shell echo $(KIND_IPV4_SUBNET) | sed 's/.[^.]\+$$/.201/')-$(shell echo $(KIND_IPV4_SUBNET) | sed 's/.[^.]\+$$/.250/') \
805808
jinjanate yamls/metallb-cr.yaml.j2 -o metallb-cr.yaml
806809
kubectl apply -f metallb-cr.yaml
@@ -977,6 +980,11 @@ kind-install-anp: kind-load-image
977980
kubectl apply -f "$(BANP_CR_YAML)"
978981
@$(MAKE) ENABLE_ANP=true kind-install
979982

983+
.PHONY: kind-install-metallb-pool-from-underlay
984+
kind-install-metallb-pool-from-underlay: kind-load-image
985+
@$(MAKE) ENABLE_OVN_LB_PREFER_LOCAL=true LS_CT_SKIP_DST_LPORT_IPS=false kind-install
986+
@$(MAKE) kind-install-metallb
987+
980988
.PHONY: kind-reload
981989
kind-reload: kind-reload-ovs
982990
kubectl delete pod -n kube-system -l app=kube-ovn-controller

‎charts/kube-ovn/templates/controller-deploy.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ spec:
140140
- --ovsdb-con-timeout={{- .Values.func.OVSDB_CON_TIMEOUT }}
141141
- --ovsdb-inactivity-timeout={{- .Values.func.OVSDB_INACTIVITY_TIMEOUT }}
142142
- --enable-live-migration-optimize={{- .Values.func.ENABLE_LIVE_MIGRATION_OPTIMIZE }}
143+
- --enable-ovn-lb-prefer-local={{- .Values.func.ENABLE_OVN_LB_PREFER_LOCAL }}
143144
- --image={{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
144145
securityContext:
145146
runAsUser: {{ include "kubeovn.runAsUser" . }}

‎charts/kube-ovn/templates/kube-ovn-crd.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -2564,6 +2564,8 @@ spec:
25642564
type: boolean
25652565
enableMulticastSnoop:
25662566
type: boolean
2567+
enableExternalLBAddress:
2568+
type: boolean
25672569
routeTable:
25682570
type: string
25692571
namespaceSelectors:

‎charts/kube-ovn/templates/ovn-CR.yaml

+3-11
Original file line numberDiff line numberDiff line change
@@ -295,21 +295,13 @@ rules:
295295
- list
296296
- patch
297297
- watch
298-
- apiGroups:
299-
- "kubeovn.io"
300-
resources:
301-
- ips
302-
verbs:
303-
- get
304-
- update
305298
- apiGroups:
306299
- ""
307300
resources:
308-
- events
301+
- services
309302
verbs:
310-
- create
311-
- patch
312-
- update
303+
- list
304+
- watch
313305
- apiGroups:
314306
- ""
315307
resources:

‎charts/kube-ovn/values.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ func:
7878
OVSDB_CON_TIMEOUT: 3
7979
OVSDB_INACTIVITY_TIMEOUT: 10
8080
ENABLE_LIVE_MIGRATION_OPTIMIZE: true
81+
ENABLE_OVN_LB_PREFER_LOCAL: false
8182

8283
ipv4:
8384
POD_CIDR: "10.16.0.0/16"

‎dist/images/install.sh

+11
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ SET_VXLAN_TX_OFF=${SET_VXLAN_TX_OFF:-false}
4646
OVSDB_CON_TIMEOUT=${OVSDB_CON_TIMEOUT:-3}
4747
OVSDB_INACTIVITY_TIMEOUT=${OVSDB_INACTIVITY_TIMEOUT:-10}
4848
ENABLE_LIVE_MIGRATION_OPTIMIZE=${ENABLE_LIVE_MIGRATION_OPTIMIZE:-true}
49+
ENABLE_OVN_LB_PREFER_LOCAL=${ENABLE_OVN_LB_PREFER_LOCAL:-false}
4950

5051
# debug
5152
DEBUG_WRAPPER=${DEBUG_WRAPPER:-}
@@ -2810,6 +2811,8 @@ spec:
28102811
type: boolean
28112812
enableMulticastSnoop:
28122813
type: boolean
2814+
enableExternalLBAddress:
2815+
type: boolean
28132816
routeTable:
28142817
type: string
28152818
namespaceSelectors:
@@ -3687,6 +3690,13 @@ rules:
36873690
- create
36883691
- patch
36893692
- update
3693+
- apiGroups:
3694+
- ""
3695+
resources:
3696+
- services
3697+
verbs:
3698+
- list
3699+
- watch
36903700
- apiGroups:
36913701
- ""
36923702
resources:
@@ -4729,6 +4739,7 @@ spec:
47294739
- --ovsdb-con-timeout=$OVSDB_CON_TIMEOUT
47304740
- --ovsdb-inactivity-timeout=$OVSDB_INACTIVITY_TIMEOUT
47314741
- --enable-live-migration-optimize=$ENABLE_LIVE_MIGRATION_OPTIMIZE
4742+
- --enable-ovn-lb-prefer-local=$ENABLE_OVN_LB_PREFER_LOCAL
47324743
- --image=$REGISTRY/kube-ovn:$VERSION
47334744
securityContext:
47344745
runAsUser: ${RUN_AS_USER}

‎e2e.mk

+9
Original file line numberDiff line numberDiff line change
@@ -251,3 +251,12 @@ kube-ovn-connectivity-e2e:
251251
E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
252252
ginkgo $(GINKGO_OUTPUT_OPT) --procs 2 --randomize-all -v \
253253
--focus=CNI:Kube-OVN ./test/e2e/connectivity -- $(TEST_BIN_ARGS)
254+
255+
.PHONY: kube-ovn-metallb-e2e
256+
kube-ovn-metallb-e2e:
257+
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/metallb
258+
E2E_BRANCH=$(E2E_BRANCH) \
259+
E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
260+
E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
261+
ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \
262+
--focus=CNI:Kube-OVN ./test/e2e/metallb/metallb.test -- $(TEST_BIN_ARGS)

‎go.mod

+2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ require (
1212
github.com/containerd/containerd v1.7.22
1313
github.com/containernetworking/cni v1.2.3
1414
github.com/containernetworking/plugins v1.6.0
15+
github.com/digitalocean/go-openvswitch v0.0.0-20240130171624-c0f7d42efe24
1516
github.com/docker/docker v27.5.0+incompatible
1617
github.com/emicklei/go-restful/v3 v3.12.1
1718
github.com/evanphx/json-patch/v5 v5.9.0
@@ -41,6 +42,7 @@ require (
4142
github.com/stretchr/testify v1.10.0
4243
github.com/vishvananda/netlink v1.3.1-0.20240905180732-b1ce50cfa9be
4344
go.uber.org/mock v0.5.0
45+
go.universe.tf/metallb v0.14.9
4446
golang.org/x/mod v0.22.0
4547
golang.org/x/sys v0.29.0
4648
golang.org/x/time v0.9.0

0 commit comments

Comments
 (0)