support metallb underlay

Signed-off-by: clyi <[email protected]>

Author: changluyi

.github/workflows/build-x86-image.yaml

@@ -3144,6 +3144,143 @@ jobs:
           name: kube-ovn-connectivity-e2e-${{ matrix.mode }}-ko-log
           path: kube-ovn-connectivity-e2e-${{ matrix.mode }}-ko-log.tar.gz
 
+
+  kube-ovn-metallb-e2e:
+    name: OVN METALLB E2E
+    needs:
+      - build-kube-ovn
+      - build-e2e-binaries
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - uses: jlumbroso/[email protected]
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          docker-images: false
+          large-packages: false
+          tool-cache: false
+          swap-storage: false
+
+      - uses: actions/checkout@v4
+
+      - name: Create the default branch directory
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        run: mkdir -p test/e2e/source
+
+      - name: Check out the default branch
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          fetch-depth: 1
+          path: test/e2e/source
+
+      - name: Export E2E directory
+        run: |
+          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
+            echo "E2E_DIR=." >> "$GITHUB_ENV"
+          else
+            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
+          fi
+
+      - uses: actions/setup-go@v5
+        id: setup-go
+        with:
+          go-version-file: ${{ env.E2E_DIR }}/go.mod
+          check-latest: true
+          cache: false
+
+      - name: Export Go full version
+        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"
+
+      - name: Go cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
+          restore-keys: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-
+
+      - name: Install kind
+        uses: helm/[email protected]
+        with:
+          version: ${{ env.KIND_VERSION }}
+          install_only: true
+
+      - name: Install ginkgo
+        working-directory: ${{ env.E2E_DIR }}
+        run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo
+
+      - name: Download kube-ovn image
+        uses: actions/download-artifact@v4
+        with:
+          name: kube-ovn
+
+      - name: Load images
+        run: docker load -i kube-ovn.tar
+
+      - name: Create kind cluster
+        run: |
+          pipx install jinjanator
+          make kind-init
+
+      - name: Install Kube-OVN
+        id: install
+        run: make kind-install-metallb-pool-from-underlay
+
+      - name: Run Ovn Metallb and Kube-OVN Combine E2E
+        id: kube-ovn-metallb-e2e
+        working-directory: ${{ env.E2E_DIR }}
+        env:
+          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
+        run: make kube-ovn-metallb-e2e
+
+      - name: Collect k8s events
+        if: failure() && ( steps.ovn-metallb-e2e.conclusion == 'failure')
+        run: |
+          kubectl get events -A -o yaml > kube-ovn-metallb-e2e-events.yaml
+          tar zcf kube-ovn-metallb-e2e-events.tar.gz kube-ovn-metallb-e2e-events.yaml
+
+      - name: Upload k8s events
+        uses: actions/upload-artifact@v4
+        if: failure() && (steps.kube-ovn-metallb-e2e.conclusion == 'failure')
+        with:
+          name: kube-ovn-metallb-e2e-events
+          path: kube-ovn-metallb-e2e-events.tar.gz
+
+      - name: Collect apiserver audit logs
+        if: failure() && (steps.kube-ovn-metallb-e2e.conclusion == 'failure')
+        run: |
+          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
+          tar zcf kube-ovn-metallb-e2e-audit-log.tar.gz kube-apiserver-audit.log
+
+      - name: Upload apiserver audit logs
+        uses: actions/upload-artifact@v4
+        if: failure() && (steps.kube-ovn-metallb-e2e.conclusion == 'failure')
+        with:
+          name: kube-ovn-metallb-e2e-audit-log
+          path: kube-ovn-metallb-e2e-audit-log.tar.gz
+
+      - name: kubectl ko log
+        if: failure() && (steps.kube-ovn-metallb-e2e.conclusion == 'failure')
+        run: |
+          make kubectl-ko-log
+          mv kubectl-ko-log.tar.gz kube-ovn-metallb-e2e-ko-log.tar.gz
+
+      - name: upload kubectl ko log
+        uses: actions/upload-artifact@v4
+        if: failure() && (steps.kube-ovn-metallb-e2e.conclusion == 'failure')
+        with:
+          name: kube-ovn-metallb-e2e-ko-log
+          path: kube-ovn-metallb-e2e-ko-log.tar.gz
+
+      - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.kube-ovn-metallb-e2e.conclusion == 'failure')) }}
+        run: make check-kube-ovn-pod-restarts
+
   push:
     name: Push Images
     needs:
@@ -3153,6 +3290,7 @@ jobs:
       - kube-ovn-conformance-e2e
       - kube-ovn-ic-conformance-e2e
       - kube-ovn-ipsec-e2e
+      - kube-ovn-metallb-e2e
       - multus-conformance-e2e
       - vpc-egress-gateway-e2e
       - ovn-vpc-nat-gw-conformance-e2e

Makefile

@@ -801,6 +801,9 @@ kind-install-metallb:
 		--set speaker.frr.image.tag=$(FRR_VERSION)
 	$(call kubectl_wait_exist_and_ready,metallb-system,deployment,metallb-controller)
 	$(call kubectl_wait_exist_and_ready,metallb-system,daemonset,metallb-speaker)
+
+.PHONY: kind-configure-metallb
+kind-configure-metallb:
 	@metallb_pool=$(shell echo $(KIND_IPV4_SUBNET) | sed 's/.[^.]\+$$/.201/')-$(shell echo $(KIND_IPV4_SUBNET) | sed 's/.[^.]\+$$/.250/') \
 		jinjanate yamls/metallb-cr.yaml.j2 -o metallb-cr.yaml
 	kubectl apply -f metallb-cr.yaml
@@ -977,6 +980,11 @@ kind-install-anp: kind-load-image
 	kubectl apply -f "$(BANP_CR_YAML)"
 	@$(MAKE) ENABLE_ANP=true kind-install
 
+.PHONY: kind-install-metallb-pool-from-underlay
+kind-install-metallb-pool-from-underlay: kind-load-image
+	@$(MAKE) ENABLE_OVN_LB_PREFER_LOCAL=true LS_CT_SKIP_DST_LPORT_IPS=false kind-install
+	@$(MAKE) kind-install-metallb
+
 .PHONY: kind-reload
 kind-reload: kind-reload-ovs
 	kubectl delete pod -n kube-system -l app=kube-ovn-controller

charts/kube-ovn/templates/controller-deploy.yaml

@@ -140,6 +140,7 @@ spec:
           - --ovsdb-con-timeout={{- .Values.func.OVSDB_CON_TIMEOUT }}
           - --ovsdb-inactivity-timeout={{- .Values.func.OVSDB_INACTIVITY_TIMEOUT }}
           - --enable-live-migration-optimize={{- .Values.func.ENABLE_LIVE_MIGRATION_OPTIMIZE }}
+          - --enable-ovn-lb-prefer-local={{- .Values.func.ENABLE_OVN_LB_PREFER_LOCAL }}
           - --image={{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
           securityContext:
             runAsUser: {{ include "kubeovn.runAsUser" . }}

charts/kube-ovn/templates/kube-ovn-crd.yaml

@@ -2564,6 +2564,8 @@ spec:
                   type: boolean
                 enableMulticastSnoop:
                   type: boolean
+                enableExternalLBAddress:
+                  type: boolean
                 routeTable:
                   type: string
                 namespaceSelectors:

charts/kube-ovn/templates/ovn-CR.yaml

@@ -295,21 +295,13 @@ rules:
       - list
       - patch
       - watch
-  - apiGroups:
-      - "kubeovn.io"
-    resources:
-      - ips
-    verbs:
-      - get
-      - update
   - apiGroups:
       - ""
     resources:
-      - events
+      - services
     verbs:
-      - create
-      - patch
-      - update
+      - list
+      - watch
   - apiGroups:
       - ""
     resources:

charts/kube-ovn/values.yaml

@@ -78,6 +78,7 @@ func:
   OVSDB_CON_TIMEOUT: 3
   OVSDB_INACTIVITY_TIMEOUT: 10
   ENABLE_LIVE_MIGRATION_OPTIMIZE: true
+  ENABLE_OVN_LB_PREFER_LOCAL: false
 
 ipv4:
   POD_CIDR: "10.16.0.0/16"

dist/images/install.sh

@@ -46,6 +46,7 @@ SET_VXLAN_TX_OFF=${SET_VXLAN_TX_OFF:-false}
 OVSDB_CON_TIMEOUT=${OVSDB_CON_TIMEOUT:-3}
 OVSDB_INACTIVITY_TIMEOUT=${OVSDB_INACTIVITY_TIMEOUT:-10}
 ENABLE_LIVE_MIGRATION_OPTIMIZE=${ENABLE_LIVE_MIGRATION_OPTIMIZE:-true}
+ENABLE_OVN_LB_PREFER_LOCAL=${ENABLE_OVN_LB_PREFER_LOCAL:-false}
 
 # debug
 DEBUG_WRAPPER=${DEBUG_WRAPPER:-}
@@ -2810,6 +2811,8 @@ spec:
                   type: boolean
                 enableMulticastSnoop:
                   type: boolean
+                enableExternalLBAddress:
+                  type: boolean
                 routeTable:
                   type: string
                 namespaceSelectors:
@@ -3687,6 +3690,13 @@ rules:
       - create
       - patch
       - update
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - ""
     resources:
@@ -4729,6 +4739,7 @@ spec:
           - --ovsdb-con-timeout=$OVSDB_CON_TIMEOUT
           - --ovsdb-inactivity-timeout=$OVSDB_INACTIVITY_TIMEOUT
           - --enable-live-migration-optimize=$ENABLE_LIVE_MIGRATION_OPTIMIZE
+          - --enable-ovn-lb-prefer-local=$ENABLE_OVN_LB_PREFER_LOCAL
           - --image=$REGISTRY/kube-ovn:$VERSION
           securityContext:
             runAsUser: ${RUN_AS_USER}

e2e.mk

@@ -251,3 +251,12 @@ kube-ovn-connectivity-e2e:
 	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
 	ginkgo $(GINKGO_OUTPUT_OPT) --procs 2 --randomize-all -v \
 		--focus=CNI:Kube-OVN ./test/e2e/connectivity -- $(TEST_BIN_ARGS)
+
+.PHONY: kube-ovn-metallb-e2e
+kube-ovn-metallb-e2e:
+	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/metallb
+	E2E_BRANCH=$(E2E_BRANCH) \
+	E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
+	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
+	ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \
+		--focus=CNI:Kube-OVN ./test/e2e/metallb/metallb.test -- $(TEST_BIN_ARGS)

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/containerd/containerd v1.7.22
 	github.com/containernetworking/cni v1.2.3
 	github.com/containernetworking/plugins v1.6.0
+	github.com/digitalocean/go-openvswitch v0.0.0-20240130171624-c0f7d42efe24
 	github.com/docker/docker v27.5.0+incompatible
 	github.com/emicklei/go-restful/v3 v3.12.1
 	github.com/evanphx/json-patch/v5 v5.9.0
@@ -41,6 +42,7 @@ require (
 	github.com/stretchr/testify v1.10.0
 	github.com/vishvananda/netlink v1.3.1-0.20240905180732-b1ce50cfa9be
 	go.uber.org/mock v0.5.0
+	go.universe.tf/metallb v0.14.9
 	golang.org/x/mod v0.22.0
 	golang.org/x/sys v0.29.0
 	golang.org/x/time v0.9.0