Merge pull request #786 from l1b0k/main

fix hubble usage

Author: BSWANG

cmd/terway-cli/policy.go

@@ -30,7 +30,7 @@ type PolicyConfig struct {
 }
 
 type CNIConfig struct {
-	HubbleEnabled       bool   `json:"cilium_enable_hubble,omitempty"`
+	HubbleEnabled       string `json:"cilium_enable_hubble,omitempty"`
 	HubbleMetrics       string `json:"cilium_hubble_metrics,omitempty"`
 	HubbleListenAddress string `json:"cilium_hubble_listen_address,omitempty"`
 	HubbleMetricServer  string `json:"cilium_hubble_metrics_server,omitempty"`
@@ -272,7 +272,7 @@ func policyConfig(container *gabs.Container) ([]string, error) {
 			return nil, fmt.Errorf("failed to unmarshal args: %w", err)
 		}
 
-		if h.HubbleEnabled {
+		if h.HubbleEnabled == "true" {
 			if h.HubbleMetrics == "" {
 				h.HubbleMetrics = "drop"
 			}

cmd/terway-cli/policy_test.go

@@ -150,6 +150,46 @@ func Test_policyConfig(t *testing.T) {
 				assert.Contains(t, strings, "--other=false")
 			},
 		},
+		{
+			name: "test hubble",
+			args: args{container: func() *gabs.Container {
+				cniJSON, _ := gabs.ParseJSON([]byte(`{
+  "cniVersion": "0.4.0",
+  "name": "terway-chainer",
+  "plugins": [
+    {
+      "bandwidth_mode": "edt",
+      "capabilities": {
+        "bandwidth": true
+      },
+      "cilium_args": "disable-per-package-lb=true",
+      "eniip_virtual_type": "datapathv2",
+      "network_policy_provider": "ebpf",
+      "cilium_enable_hubble": "true",
+      "cilium_hubble_listen_address": ":4244",
+      "cilium_hubble_metrics_server": ":9091",
+      "cilium_hubble_metrics": "drop,tcp,flow,port-distribution,icmp",
+      "type": "terway"
+    },
+    {
+      "data-path": "datapathv2",
+      "enable-debug": false,
+      "log-file": "/var/run/cilium/cilium-cni.log",
+      "type": "cilium-cni"
+    }
+  ]
+}`))
+				return cniJSON
+			}()},
+			readFunc: func(name string) ([]byte, error) {
+				return []byte("#define DIRECT_ROUTING_DEV_IFINDEX 0\n#define DISABLE_PER_PACKET_LB 1\n"), nil
+			},
+			checkFunc: func(t *testing.T, strings []string, err error) {
+				assert.NoError(t, err)
+				assert.Contains(t, strings, "--disable-per-package-lb=true")
+				assert.Contains(t, strings, "--enable-hubble=true")
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

deploy/images/terway-controlplane/Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1-labs
-ARG TERWAY_POLICY_IMAGE=registry-cn-zhangjiakou.ack.aliyuncs.com/acs/terway:policy-ff12ae92@sha256:f53c1aee4a58f1c7ed7b5d580674cad94050f38188a214cc8ccdef33ae690c61
+ARG TERWAY_POLICY_IMAGE=registry-cn-zhangjiakou.ack.aliyuncs.com/acs/terway:policy-04728dad@sha256:819a778edd3a9a3693482ff424b3686481bfb3089adee5aeeb43af57d225d4c9
 
 FROM --platform=$TARGETPLATFORM ${TERWAY_POLICY_IMAGE} AS policy-dist
 

deploy/images/terway/Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1-labs
-ARG TERWAY_POLICY_IMAGE=registry-cn-zhangjiakou.ack.aliyuncs.com/acs/terway:policy-ff12ae92@sha256:f53c1aee4a58f1c7ed7b5d580674cad94050f38188a214cc8ccdef33ae690c61
+ARG TERWAY_POLICY_IMAGE=registry-cn-zhangjiakou.ack.aliyuncs.com/acs/terway:policy-04728dad@sha256:819a778edd3a9a3693482ff424b3686481bfb3089adee5aeeb43af57d225d4c9
 ARG UBUNTU_IMAGE=registry.cn-hangzhou.aliyuncs.com/acs/ubuntu:22.04-update
 ARG CILIUM_LLVM_IMAGE=quay.io/cilium/cilium-llvm:547db7ec9a750b8f888a506709adb41f135b952e@sha256:4d6fa0aede3556c5fb5a9c71bc6b9585475ac9b1064f516d4c45c8fb691c9d9e
 ARG CILIUM_BPFTOOL_IMAGE=quay.io/cilium/cilium-bpftool:78448c1a37ff2b790d5e25c3d8b8ec3e96e6405f@sha256:99a9453a921a8de99899ef82e0822f0c03f65d97005c064e231c06247ad8597d

policy/cilium/0001-cilium-terway-datapath.patch

@@ -1,7 +1,7 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: l1b0k <[email protected]>
 Date: Wed, 9 Jun 2021 16:55:20 +0800
-Subject: [PATCH] cilium terway datapath
+Subject: cilium terway datapath
 
 Signed-off-by: l1b0k <[email protected]>
 ---
@@ -735,5 +735,5 @@ index 5eca17daeb..1ee2227373 100644
  )
 
 -- 
-2.39.1
+2.39.5 (Apple Git-154)
 

policy/cilium/0002-overwrite-endpoint-when-conflicting.patch

@@ -1,7 +1,7 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Lyt99 <[email protected]>
 Date: Wed, 9 Jun 2021 16:29:32 +0800
-Subject: [PATCH] overwrite endpoint when conflicting
+Subject: overwrite endpoint when conflicting
 
 Signed-off-by: l1b0k <[email protected]>
 ---
@@ -24,5 +24,5 @@ index 9700af0b3a..c7c4ad1637 100644
  	}
 
 -- 
-2.39.1
+2.39.5 (Apple Git-154)
 

policy/cilium/0003-run-operator.patch

@@ -1,72 +1,14 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: l1b0k <[email protected]>
 Date: Wed, 9 Jun 2021 16:32:33 +0800
-Subject: [PATCH] run operator
+Subject: run operator
 
 Signed-off-by: l1b0k <[email protected]>
 ---
- daemon/cmd/daemon_main.go |  25 +++--
- operator/Makefile         |   2 +-
- operator/main.go          | 225 --------------------------------------
- 3 files changed, 19 insertions(+), 233 deletions(-)
+ operator/Makefile |   2 +-
+ operator/main.go  | 243 ++--------------------------------------------
+ 2 files changed, 11 insertions(+), 234 deletions(-)
 
-diff --git a/daemon/cmd/daemon_main.go b/daemon/cmd/daemon_main.go
-index 28bf66a7a9..1f2796810b 100644
---- a/daemon/cmd/daemon_main.go
-+++ b/daemon/cmd/daemon_main.go
-@@ -14,13 +14,6 @@ import (
- 	"strings"
- 	"time"
- 
--	"github.com/go-openapi/loads"
--	gops "github.com/google/gops/agent"
--	"github.com/sirupsen/logrus"
--	"github.com/spf13/cobra"
--	"github.com/spf13/viper"
--	"google.golang.org/grpc"
--
- 	"github.com/cilium/cilium/api/v1/server"
- 	"github.com/cilium/cilium/api/v1/server/restapi"
- 	"github.com/cilium/cilium/pkg/aws/eni"
-@@ -28,6 +21,7 @@ import (
- 	"github.com/cilium/cilium/pkg/bgpv1/gobgp"
- 	"github.com/cilium/cilium/pkg/bpf"
- 	"github.com/cilium/cilium/pkg/cgroups"
-+	"github.com/cilium/cilium/pkg/command/exec"
- 	"github.com/cilium/cilium/pkg/common"
- 	"github.com/cilium/cilium/pkg/components"
- 	"github.com/cilium/cilium/pkg/controller"
-@@ -75,6 +69,12 @@ import (
- 	"github.com/cilium/cilium/pkg/version"
- 	wireguard "github.com/cilium/cilium/pkg/wireguard/agent"
- 	wireguardTypes "github.com/cilium/cilium/pkg/wireguard/types"
-+	"github.com/go-openapi/loads"
-+	gops "github.com/google/gops/agent"
-+	"github.com/sirupsen/logrus"
-+	"github.com/spf13/cobra"
-+	"github.com/spf13/viper"
-+	"google.golang.org/grpc"
- )
- 
- const (
-@@ -1624,6 +1624,17 @@ func (d *Daemon) initKVStore() {
- }
- 
- func runDaemon() {
-+	go func() {
-+		if os.Getenv("DISABLE_CILIUM_OPERATOR") == "true" {
-+			return
-+		}
-+		cmd := exec.CommandContext(server.ServerCtx, "cilium-operator-generic", "--skip-crd-creation", "--k8s-namespace", os.Getenv("CILIUM_K8S_NAMESPACE"), "--identity-gc-interval", "10m", "--identity-heartbeat-timeout", "20m")
-+		cmd.Stdout = os.Stdout
-+		cmd.Stderr = os.Stderr
-+		err := cmd.Run()
-+		log.Fatal(fmt.Errorf("cilium-operator exited, %w", err))
-+	}()
-+
- 	datapathConfig := linuxdatapath.DatapathConfiguration{
- 		HostDevice: defaults.HostDevice,
- 		ProcFs:     option.Config.ProcFs,
 diff --git a/operator/Makefile b/operator/Makefile
 index ebd1285e3d..1710880c84 100644
 --- a/operator/Makefile
@@ -81,7 +23,7 @@ index ebd1285e3d..1710880c84 100644
  cilium-operator-aws: GO_TAGS_FLAGS+=ipam_provider_aws
  cilium-operator-azure: GO_TAGS_FLAGS+=ipam_provider_azure
 diff --git a/operator/main.go b/operator/main.go
-index 756b218121..ec71d565dc 100644
+index 756b218121..ec0a329109 100644
 --- a/operator/main.go
 +++ b/operator/main.go
 @@ -15,30 +15,22 @@ import (
@@ -123,7 +65,33 @@ index 756b218121..ec71d565dc 100644
  	"github.com/cilium/cilium/pkg/rand"
  	"github.com/cilium/cilium/pkg/rate"
  	"github.com/cilium/cilium/pkg/version"
-@@ -250,14 +241,6 @@ func runOperator() {
+@@ -72,15 +63,17 @@ var (
+ 			}
+ 
+ 			// Open socket for using gops to get stacktraces of the agent.
+-			addr := fmt.Sprintf("127.0.0.1:%d", viper.GetInt(option.GopsPort))
+-			addrField := logrus.Fields{"address": addr}
+-			if err := gops.Listen(gops.Options{
+-				Addr:                   addr,
+-				ReuseSocketAddrAndPort: true,
+-			}); err != nil {
+-				log.WithError(err).WithFields(addrField).Fatal("Cannot start gops server")
++			if viper.GetInt(option.GopsPort) > 0 {
++				addr := fmt.Sprintf("127.0.0.1:%d", viper.GetInt(option.GopsPort))
++				addrField := logrus.Fields{"address": addr}
++				if err := gops.Listen(gops.Options{
++					Addr:                   addr,
++					ReuseSocketAddrAndPort: true,
++				}); err != nil {
++					log.WithError(err).WithFields(addrField).Fatal("Cannot start gops server")
++				}
++				log.WithFields(addrField).Info("Started gops server")
+ 			}
+-			log.WithFields(addrField).Info("Started gops server")
+ 
+ 			initEnv()
+ 			runOperator()
+@@ -250,14 +243,6 @@ func runOperator() {
  		}
  	}()
 
@@ -138,7 +106,7 @@ index 756b218121..ec71d565dc 100644
  	initK8s(k8sInitDone)
 
  	capabilities := k8sversion.Capabilities()
-@@ -367,189 +350,6 @@ func onOperatorStartLeading(ctx context.Context) {
+@@ -367,189 +352,6 @@ func onOperatorStartLeading(ctx context.Context) {
  		go cesController.Run(operatorWatchers.CiliumEndpointStore, stopCh)
  	}
 
@@ -328,7 +296,7 @@ index 756b218121..ec71d565dc 100644
  	if operatorOption.Config.IdentityGCInterval != 0 {
  		identityRateLimiter = rate.NewLimiter(
  			operatorOption.Config.IdentityGCRateInterval,
-@@ -583,31 +383,6 @@ func onOperatorStartLeading(ctx context.Context) {
+@@ -583,31 +385,6 @@ func onOperatorStartLeading(ctx context.Context) {
  		enableCiliumEndpointSyncGC(true)
  	}
 
@@ -361,5 +329,5 @@ index 756b218121..ec71d565dc 100644
 
  	<-shutdownSignal
 -- 
-2.39.1
+2.39.5 (Apple Git-154)
 

policy/cilium/0004-adapt-1.10-for-terway.patch

@@ -1,7 +1,7 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: l1b0k <[email protected]>
 Date: Thu, 10 Jun 2021 19:20:58 +0800
-Subject: [PATCH] adapt 1.10 for terway
+Subject: adapt 1.10 for terway
 
 Signed-off-by: l1b0k <[email protected]>
 ---
@@ -40,5 +40,5 @@ index 7c0eaa8932..f713067030 100644
  	c.KubeProxyReplacementHealthzBindAddr = viper.GetString(KubeProxyReplacementHealthzBindAddr)
 
 -- 
-2.39.1
+2.39.5 (Apple Git-154)
 

policy/cilium/0005-add-flag-to-control-in-cluster-loadBalance.patch

@@ -1,7 +1,7 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: l1b0k <[email protected]>
 Date: Thu, 9 Dec 2021 10:39:49 +0800
-Subject: [PATCH] add flag to control in cluster loadBalance
+Subject: add flag to control in cluster loadBalance
 
 Signed-off-by: l1b0k <[email protected]>
 ---
@@ -11,7 +11,7 @@ Signed-off-by: l1b0k <[email protected]>
  3 files changed, 11 insertions(+), 1 deletion(-)
 
 diff --git a/daemon/cmd/daemon_main.go b/daemon/cmd/daemon_main.go
-index 1f2796810b..375a11d5cf 100644
+index 28bf66a7a9..ac3c6834a3 100644
 --- a/daemon/cmd/daemon_main.go
 +++ b/daemon/cmd/daemon_main.go
 @@ -390,6 +390,9 @@ func initializeFlags() {
@@ -70,5 +70,5 @@ index f713067030..df026092f8 100644
  	c.EnableHostPort = viper.GetBool(EnableHostPort)
  	c.EnableHostLegacyRouting = viper.GetBool(EnableHostLegacyRouting)
 -- 
-2.39.1
+2.39.5 (Apple Git-154)
 

policy/cilium/0006-terway-support-kubelet-health-check.patch

@@ -1,7 +1,7 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: l1b0k <[email protected]>
 Date: Mon, 23 May 2022 17:43:50 +0800
-Subject: [PATCH] terway: support kubelet health check
+Subject: terway: support kubelet health check
 
 when package travel cross netns fw mark will lost ,this will cause bpf unable to identify host network.
 
@@ -35,5 +35,5 @@ index 6192691586..64f9169958 100644
  		}
  		cilium_dbg(ctx, info ? DBG_IP_ID_MAP_SUCCEED4 : DBG_IP_ID_MAP_FAILED4,
 -- 
-2.39.1
+2.39.5 (Apple Git-154)
 

policy/cilium/0007-add-bandwidth-for-terway-ipvlan.patch

@@ -1,7 +1,7 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: l1b0k <[email protected]>
 Date: Mon, 20 Jun 2022 10:57:16 +0800
-Subject: [PATCH] add bandwidth for terway ipvlan
+Subject: add bandwidth for terway ipvlan
 
 Signed-off-by: l1b0k <[email protected]>
 ---
@@ -93,5 +93,5 @@ index ea542dd527..0f078ad502 100644
  		ctmap.WriteBPFMacros(fw, e)
  	} else {
 -- 
-2.39.1
+2.39.5 (Apple Git-154)
 

policy/cilium/0008-adapt-1.12.patch

@@ -1,7 +1,7 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: l1b0k <[email protected]>
 Date: Wed, 28 Sep 2022 10:11:36 +0800
-Subject: [PATCH] adapt 1.12
+Subject: adapt 1.12
 
 - disable source ip check
 - disable host bpf
@@ -61,5 +61,5 @@ index d85bba9285..802384084e 100644
  	}
 
 -- 
-2.39.1
+2.39.5 (Apple Git-154)