ovn lb select the local chassis's backend prefer (#4894)

* add lb:option prefer_local_backend

Signed-off-by: clyi <[email protected]>

* support metallb underlay

Signed-off-by: clyi <[email protected]>

Author: changluyi

.github/workflows/build-x86-image.yaml

@@ -3144,6 +3144,151 @@ jobs:
           name: kube-ovn-connectivity-e2e-${{ matrix.mode }}-ko-log
           path: kube-ovn-connectivity-e2e-${{ matrix.mode }}-ko-log.tar.gz
 
+
+  kube-ovn-underlay-metallb-e2e:
+    name: OVN METALLB E2E
+    needs:
+      - build-kube-ovn
+      - build-e2e-binaries
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - uses: jlumbroso/[email protected]
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          docker-images: false
+          large-packages: false
+          tool-cache: false
+          swap-storage: false
+
+      - uses: actions/checkout@v4
+
+      - name: Create the default branch directory
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        run: mkdir -p test/e2e/source
+
+      - name: Check out the default branch
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          fetch-depth: 1
+          path: test/e2e/source
+
+      - name: Export E2E directory
+        run: |
+          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
+            echo "E2E_DIR=." >> "$GITHUB_ENV"
+          else
+            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
+          fi
+
+      - uses: actions/setup-go@v5
+        id: setup-go
+        with:
+          go-version-file: ${{ env.E2E_DIR }}/go.mod
+          check-latest: true
+          cache: false
+
+      - name: Export Go full version
+        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"
+
+      - name: Go cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
+          restore-keys: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-
+
+      - name: Install kind
+        uses: helm/[email protected]
+        with:
+          version: ${{ env.KIND_VERSION }}
+          install_only: true
+
+      - name: Install ginkgo
+        working-directory: ${{ env.E2E_DIR }}
+        run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo
+
+      - name: Download kube-ovn image
+        uses: actions/download-artifact@v4
+        with:
+          name: kube-ovn
+
+      - name: Load images
+        run: docker load -i kube-ovn.tar
+
+      - name: Set environment variables
+        run: |
+          if [ $(($RANDOM%2)) -ne 0 ]; then
+            # run as root and use valgrind to debug memory leak
+            echo "VERSION=$(cat VERSION)-debug" >> "$GITHUB_ENV"
+            echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV"
+          fi
+
+      - name: Create kind cluster
+        run: |
+          pipx install jinjanator
+          make kind-init
+
+      - name: Install Kube-OVN
+        id: install
+        run: make kind-install-metallb-pool-from-underlay
+
+      - name: Run Ovn Metallb and Kube-OVN Combine E2E
+        id: kube-ovn-underlay-metallb-e2e
+        working-directory: ${{ env.E2E_DIR }}
+        env:
+          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
+        run: make kube-ovn-underlay-metallb-e2e
+
+      - name: Collect k8s events
+        if: failure() && ( steps.ovn-metallb-e2e.conclusion == 'failure')
+        run: |
+          kubectl get events -A -o yaml > kube-ovn-underlay-metallb-e2e-events.yaml
+          tar zcf kube-ovn-underlay-metallb-e2e-events.tar.gz kube-ovn-underlay-metallb-e2e-events.yaml
+
+      - name: Upload k8s events
+        uses: actions/upload-artifact@v4
+        if: failure() && (steps.kube-ovn-underlay-metallb-e2e.conclusion == 'failure')
+        with:
+          name: kube-ovn-underlay-metallb-e2e-events
+          path: kube-ovn-underlay-metallb-e2e-events.tar.gz
+
+      - name: Collect apiserver audit logs
+        if: failure() && (steps.kube-ovn-underlay-metallb-e2e.conclusion == 'failure')
+        run: |
+          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
+          tar zcf kube-ovn-underlay-metallb-e2e-audit-log.tar.gz kube-apiserver-audit.log
+
+      - name: Upload apiserver audit logs
+        uses: actions/upload-artifact@v4
+        if: failure() && (steps.kube-ovn-underlay-metallb-e2e.conclusion == 'failure')
+        with:
+          name: kube-ovn-underlay-metallb-e2e-audit-log
+          path: kube-ovn-underlay-metallb-e2e-audit-log.tar.gz
+
+      - name: kubectl ko log
+        if: failure() && (steps.kube-ovn-underlay-metallb-e2e.conclusion == 'failure')
+        run: |
+          make kubectl-ko-log
+          mv kubectl-ko-log.tar.gz kube-ovn-underlay-metallb-e2e-ko-log.tar.gz
+
+      - name: upload kubectl ko log
+        uses: actions/upload-artifact@v4
+        if: failure() && (steps.kube-ovn-underlay-metallb-e2e.conclusion == 'failure')
+        with:
+          name: kube-ovn-underlay-metallb-e2e-ko-log
+          path: kube-ovn-underlay-metallb-e2e-ko-log.tar.gz
+
+      - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.kube-ovn-underlay-metallb-e2e.conclusion == 'failure')) }}
+        run: make check-kube-ovn-pod-restarts
+
   push:
     name: Push Images
     needs:
@@ -3153,6 +3298,7 @@ jobs:
       - kube-ovn-conformance-e2e
       - kube-ovn-ic-conformance-e2e
       - kube-ovn-ipsec-e2e
+      - kube-ovn-underlay-metallb-e2e
       - multus-conformance-e2e
       - vpc-egress-gateway-e2e
       - ovn-vpc-nat-gw-conformance-e2e

Makefile

@@ -803,6 +803,9 @@ kind-install-metallb:
 		--set speaker.frr.image.tag=$(FRR_VERSION)
 	$(call kubectl_wait_exist_and_ready,metallb-system,deployment,metallb-controller)
 	$(call kubectl_wait_exist_and_ready,metallb-system,daemonset,metallb-speaker)
+
+.PHONY: kind-configure-metallb
+kind-configure-metallb:
 	@metallb_pool=$(shell echo $(KIND_IPV4_SUBNET) | sed 's/.[^.]\+$$/.201/')-$(shell echo $(KIND_IPV4_SUBNET) | sed 's/.[^.]\+$$/.250/') \
 		jinjanate yamls/metallb-cr.yaml.j2 -o metallb-cr.yaml
 	kubectl apply -f metallb-cr.yaml
@@ -979,6 +982,11 @@ kind-install-anp: kind-load-image
 	kubectl apply -f "$(BANP_CR_YAML)"
 	@$(MAKE) ENABLE_ANP=true kind-install
 
+.PHONY: kind-install-metallb-pool-from-underlay
+kind-install-metallb-pool-from-underlay: kind-load-image
+	@$(MAKE) ENABLE_OVN_LB_PREFER_LOCAL=true LS_CT_SKIP_DST_LPORT_IPS=false kind-install
+	@$(MAKE) kind-install-metallb
+
 .PHONY: kind-reload
 kind-reload: kind-reload-ovs
 	kubectl delete pod -n kube-system -l app=kube-ovn-controller

charts/kube-ovn/templates/controller-deploy.yaml

@@ -140,6 +140,7 @@ spec:
           - --ovsdb-con-timeout={{- .Values.func.OVSDB_CON_TIMEOUT }}
           - --ovsdb-inactivity-timeout={{- .Values.func.OVSDB_INACTIVITY_TIMEOUT }}
           - --enable-live-migration-optimize={{- .Values.func.ENABLE_LIVE_MIGRATION_OPTIMIZE }}
+          - --enable-ovn-lb-prefer-local={{- .Values.func.ENABLE_OVN_LB_PREFER_LOCAL }}
           - --image={{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
           securityContext:
             runAsUser: {{ include "kubeovn.runAsUser" . }}

charts/kube-ovn/templates/kube-ovn-crd.yaml

@@ -2564,6 +2564,8 @@ spec:
                   type: boolean
                 enableMulticastSnoop:
                   type: boolean
+                enableExternalLBAddress:
+                  type: boolean
                 routeTable:
                   type: string
                 namespaceSelectors:

charts/kube-ovn/templates/ovn-CR.yaml

@@ -290,6 +290,7 @@ rules:
       - nodes
       - nodes/status
       - pods
+      - services
     verbs:
       - get
       - list

charts/kube-ovn/values.yaml

@@ -78,6 +78,7 @@ func:
   OVSDB_CON_TIMEOUT: 3
   OVSDB_INACTIVITY_TIMEOUT: 10
   ENABLE_LIVE_MIGRATION_OPTIMIZE: true
+  ENABLE_OVN_LB_PREFER_LOCAL: false
 
 ipv4:
   POD_CIDR: "10.16.0.0/16"

dist/images/Dockerfile.base

@@ -65,7 +65,9 @@ RUN cd /usr/src/ && git clone -b branch-24.03 --depth=1 https://github.com/ovn-o
     # support dedicated BFD LRP
     curl -s https://github.com/kubeovn/ovn/commit/40345aa35d03c93cde877ccfa8111346291ebc7c.patch | git apply && \
     # skip node local dns ip conntrack when set acl
-    curl -s https://github.com/kubeovn/ovn/commit/e7d3ba53cdcbc524bb29c54ddb07b83cc4258ed7.patch | git apply
+    curl -s https://github.com/kubeovn/ovn/commit/e7d3ba53cdcbc524bb29c54ddb07b83cc4258ed7.patch | git apply && \
+    # select local backend first
+    curl -s https://github.com/kubeovn/ovn/commit/a9e009136a42cf6d985f97e2bf1ec41df6b5ca29.patch | git apply
 
 RUN apt install -y build-essential fakeroot \
     autoconf automake bzip2 debhelper-compat dh-exec dh-python dh-sequence-python3 dh-sequence-sphinxdoc \

dist/images/install.sh

@@ -46,6 +46,7 @@ SET_VXLAN_TX_OFF=${SET_VXLAN_TX_OFF:-false}
 OVSDB_CON_TIMEOUT=${OVSDB_CON_TIMEOUT:-3}
 OVSDB_INACTIVITY_TIMEOUT=${OVSDB_INACTIVITY_TIMEOUT:-10}
 ENABLE_LIVE_MIGRATION_OPTIMIZE=${ENABLE_LIVE_MIGRATION_OPTIMIZE:-true}
+ENABLE_OVN_LB_PREFER_LOCAL=${ENABLE_OVN_LB_PREFER_LOCAL:-false}
 
 PROBE_HTTP_SCHEME="HTTP"
 if [ "$SECURE_SERVING" = "true" ]; then
@@ -2815,6 +2816,8 @@ spec:
                   type: boolean
                 enableMulticastSnoop:
                   type: boolean
+                enableExternalLBAddress:
+                  type: boolean
                 routeTable:
                   type: string
                 namespaceSelectors:
@@ -3672,6 +3675,7 @@ rules:
       - nodes
       - nodes/status
       - pods
+      - services
     verbs:
       - get
       - list
@@ -4734,6 +4738,7 @@ spec:
           - --ovsdb-con-timeout=$OVSDB_CON_TIMEOUT
           - --ovsdb-inactivity-timeout=$OVSDB_INACTIVITY_TIMEOUT
           - --enable-live-migration-optimize=$ENABLE_LIVE_MIGRATION_OPTIMIZE
+          - --enable-ovn-lb-prefer-local=$ENABLE_OVN_LB_PREFER_LOCAL
           - --image=$REGISTRY/kube-ovn:$VERSION
           securityContext:
             runAsUser: ${RUN_AS_USER}

e2e.mk

@@ -84,6 +84,7 @@ e2e-build:
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/kubevirt
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/webhook
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/connectivity
+	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/metallb
 
 .PHONY: k8s-conformance-e2e
 k8s-conformance-e2e:
@@ -253,3 +254,12 @@ kube-ovn-connectivity-e2e:
 	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
 	ginkgo $(GINKGO_OUTPUT_OPT) --procs 2 --randomize-all -v \
 		--focus=CNI:Kube-OVN ./test/e2e/connectivity -- $(TEST_BIN_ARGS)
+
+.PHONY: kube-ovn-underlay-metallb-e2e
+kube-ovn-underlay-metallb-e2e:
+	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/metallb
+	E2E_BRANCH=$(E2E_BRANCH) \
+	E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
+	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
+	ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \
+		--focus=CNI:Kube-OVN ./test/e2e/metallb/metallb.test -- $(TEST_BIN_ARGS)

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/containerd/containerd v1.7.27
 	github.com/containernetworking/cni v1.2.3
 	github.com/containernetworking/plugins v1.6.2
+	github.com/digitalocean/go-openvswitch v0.0.0-20241021184246-19e734367535
 	github.com/docker/docker v28.0.4+incompatible
 	github.com/emicklei/go-restful/v3 v3.12.2
 	github.com/evanphx/json-patch/v5 v5.9.11
@@ -42,6 +43,7 @@ require (
 	github.com/stretchr/testify v1.10.0
 	github.com/vishvananda/netlink v1.3.1-0.20250303224720-0e7078ed04c8
 	go.uber.org/mock v0.5.0
+	go.universe.tf/metallb v0.14.9
 	golang.org/x/mod v0.24.0
 	golang.org/x/sys v0.31.0
 	golang.org/x/time v0.11.0