feat(injector): add list of ignored network interfaces (#4700)

* feat(injector): add list of ignored network interfaces

This change adds a new configurable list of ignored network interface
names. Traffic received from and sent to those interfaces is not
forwarded to the sidecar container by iptables. When this list is empty
(the default), osm-injector produces the exact same iptables commands as
it did before.

The list is configured in the chart at `osm.networkInterfaceExclusionList`
and in the MeshConfig at `spec.traffic.networkInterfaceExclusionList`.

Fixes #4546

Signed-off-by: Jon Huhn <[email protected]>

* codegen

Signed-off-by: Jon Huhn <[email protected]>

* add comment

Signed-off-by: Jon Huhn <[email protected]>
Co-authored-by: Shashank Ram <[email protected]>

Author: nojnhuh

charts/osm/README.md

@@ -143,6 +143,7 @@ The following table lists the configurable parameters of the osm chart and their
 | osm.meshName | string | `"osm"` | Identifier for the instance of a service mesh within a cluster |
 | osm.multicluster | object | `{"gatewayLogLevel":"error"}` | OSM multicluster feature configuration |
 | osm.multicluster.gatewayLogLevel | string | `"error"` | Log level for the multicluster gateway |
+| osm.networkInterfaceExclusionList | list | `[]` | Specifies a global list of network interface names to exclude for inbound and outbound traffic interception by the sidecar proxy. |
 | osm.osmBootstrap.podLabels | object | `{}` | OSM bootstrap's pod labels |
 | osm.osmBootstrap.replicaCount | int | `1` | OSM bootstrap's replica count |
 | osm.osmBootstrap.resource | object | `{"limits":{"cpu":"0.5","memory":"128M"},"requests":{"cpu":"0.3","memory":"128M"}}` | OSM bootstrap's container resource parameters |

charts/osm/templates/preset-mesh-config.yaml

@@ -18,7 +18,8 @@ data:
         "outboundPortExclusionList": {{.Values.osm.outboundPortExclusionList | mustToJson}},
         "inboundPortExclusionList": {{.Values.osm.inboundPortExclusionList | mustToJson}},
         "outboundIPRangeExclusionList": {{.Values.osm.outboundIPRangeExclusionList | mustToJson}},
-        "outboundIPRangeInclusionList": {{.Values.osm.outboundIPRangeInclusionList | mustToJson}}
+        "outboundIPRangeInclusionList": {{.Values.osm.outboundIPRangeInclusionList | mustToJson}},
+        "networkInterfaceExclusionList": {{.Values.osm.networkInterfaceExclusionList | mustToJson}}
       },
       "observability": {
         "enableDebugServer": {{.Values.osm.enableDebugServer | mustToJson}},

charts/osm/values.schema.json

@@ -1131,6 +1131,21 @@
                         ]
                     ]
                 },
+                "networkInterfaceExclusionList": {
+                    "$id": "#/properties/osm/properties/networkInterfaceExclusionList",
+                    "type": "array",
+                    "title": "The networkInterfaceExclusionList schema",
+                    "description": "Network interface exluclusion list for sidecar traffic interception",
+                    "items": {
+                        "type": "string"
+                    },
+                    "examples": [
+                        [
+                            "eth0",
+                            "net1"
+                        ]
+                    ]
+                },
                 "grafana": {
                     "$id": "#/properties/osm/properties/grafana",
                     "type": "object",

charts/osm/values.yaml

@@ -262,6 +262,9 @@ osm:
   # If specified, must be a list of positive integers.
   inboundPortExclusionList: []
 
+  # -- Specifies a global list of network interface names to exclude for inbound and outbound traffic interception by the sidecar proxy.
+  networkInterfaceExclusionList: []
+
   #
   # -- OSM's sidecar injector parameters
   injector:

cmd/osm-bootstrap/crds/config_meshconfig.yaml

@@ -156,6 +156,11 @@ spec:
                         type: integer
                         minimum: 1
                         maximum: 65535
+                    networkInterfaceExclusionList:
+                      description: NetworkInterfaceExclusionList defines a global list of network interface names to exclude from inbound and outbound traffic interception by the sidecar proxy.
+                      type: array
+                      items:
+                        type: string
                     enablePermissiveTrafficPolicyMode:
                       description: True for allowing traffic to flow between client and service pods within the mesh without SMI traffic policies, i.e. no traffic policy enforcement in the mesh. If set to false, enables deny-all traffic policy in mesh i.e. an SMI Traffic Target is necessary for services to communicate.
                       type: boolean

pkg/apis/config/v1alpha2/mesh_config.go

@@ -116,6 +116,11 @@ type TrafficSpec struct {
 	// InboundExternalAuthorization defines a ruleset that, if enabled, will configure a remote external authorization endpoint
 	// for all inbound and ingress traffic in the mesh.
 	InboundExternalAuthorization ExternalAuthzSpec `json:"inboundExternalAuthorization,omitempty"`
+
+	// NetworkInterfaceExclusionList defines a global list of network interface
+	// names to exclude from inbound and outbound traffic interception by the
+	// sidecar proxy.
+	NetworkInterfaceExclusionList []string `json:"networkInterfaceExclusionList"`
 }
 
 // ObservabilitySpec is the type to represent OSM's observability configurations.

pkg/apis/config/v1alpha2/zz_generated.deepcopy.go

@@ -9,8 +9,8 @@ import (
 
 func getInitContainerSpec(containerName string, cfg configurator.Configurator, outboundIPRangeExclusionList []string,
 	outboundIPRangeInclusionList []string, outboundPortExclusionList []int,
-	inboundPortExclusionList []int, enablePrivilegedInitContainer bool, pullPolicy corev1.PullPolicy) corev1.Container {
-	iptablesInitCommand := generateIptablesCommands(outboundIPRangeExclusionList, outboundIPRangeInclusionList, outboundPortExclusionList, inboundPortExclusionList)
+	inboundPortExclusionList []int, enablePrivilegedInitContainer bool, pullPolicy corev1.PullPolicy, networkInterfaceExclusionList []string) corev1.Container {
+	iptablesInitCommand := generateIptablesCommands(outboundIPRangeExclusionList, outboundIPRangeInclusionList, outboundPortExclusionList, inboundPortExclusionList, networkInterfaceExclusionList)
 
 	return corev1.Container{
 		Name:            containerName,

pkg/injector/init_container.go

@@ -27,7 +27,7 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func()
 		It("Creates init container without ip range exclusion list", func() {
 			mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1)
 			privileged := privilegedFalse
-			actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, nil, privileged, corev1.PullAlways)
+			actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, nil, privileged, corev1.PullAlways, nil)
 
 			expected := corev1.Container{
 				Name:            "-container-name-",

pkg/injector/init_container_test.go

@@ -60,7 +60,7 @@ var iptablesInboundStaticRules = []string{
 }
 
 // generateIptablesCommands generates a list of iptables commands to set up sidecar interception and redirection
-func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundIPRangeInclusionList []string, outboundPortExclusionList []int, inboundPortExclusionList []int) string {
+func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundIPRangeInclusionList []string, outboundPortExclusionList []int, inboundPortExclusionList []int, networkInterfaceExclusionList []string) string {
 	var rules strings.Builder
 
 	fmt.Fprintln(&rules, `# OSM sidecar interception rules
@@ -74,6 +74,13 @@ func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundIPR
 	// 1. Create inbound rules
 	cmds = append(cmds, iptablesInboundStaticRules...)
 
+	// Ignore inbound traffic on specified interfaces
+	for _, iface := range networkInterfaceExclusionList {
+		// *Note: it is important to use the insert option '-I' instead of the append option '-A' to ensure the
+		// exclusion of traffic to the network interface happens before the rule that redirects traffic to the proxy
+		cmds = append(cmds, fmt.Sprintf("-I OSM_PROXY_INBOUND -i %s -j RETURN", iface))
+	}
+
 	// 2. Create dynamic inbound ports exclusion rules
 	if len(inboundPortExclusionList) > 0 {
 		var portExclusionListStr []string
@@ -88,6 +95,11 @@ func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundIPR
 	// 3. Create outbound rules
 	cmds = append(cmds, iptablesOutboundStaticRules...)
 
+	// Ignore outbound traffic in specified interfaces
+	for _, iface := range networkInterfaceExclusionList {
+		cmds = append(cmds, fmt.Sprintf("-A OSM_PROXY_OUTBOUND -o %s -j RETURN", iface))
+	}
+
 	//
 	// Create outbound exclusion and inclusion rules.
 	// *Note: exclusion rules must be applied before inclusions as order matters

pkg/injector/iptables.go

@@ -8,12 +8,13 @@ import (
 
 func TestGenerateIptablesCommands(t *testing.T) {
 	testCases := []struct {
-		name                      string
-		outboundIPRangeExclusions []string
-		outboundIPRangeInclusions []string
-		outboundPortExclusions    []int
-		inboundPortExclusions     []int
-		expected                  string
+		name                       string
+		outboundIPRangeExclusions  []string
+		outboundIPRangeInclusions  []string
+		outboundPortExclusions     []int
+		inboundPortExclusions      []int
+		networkInterfaceExclusions []string
+		expected                   string
 	}{
 		{
 			name: "no exclusions or inclusions",
@@ -45,11 +46,12 @@ EOF
 `,
 		},
 		{
-			name:                      "with exclusions and inclusions",
-			outboundIPRangeExclusions: []string{"1.1.1.1/32", "2.2.2.2/32"},
-			outboundIPRangeInclusions: []string{"3.3.3.3/32", "4.4.4.4/32"},
-			outboundPortExclusions:    []int{10, 20},
-			inboundPortExclusions:     []int{30, 40},
+			name:                       "with exclusions and inclusions",
+			outboundIPRangeExclusions:  []string{"1.1.1.1/32", "2.2.2.2/32"},
+			outboundIPRangeInclusions:  []string{"3.3.3.3/32", "4.4.4.4/32"},
+			outboundPortExclusions:     []int{10, 20},
+			inboundPortExclusions:      []int{30, 40},
+			networkInterfaceExclusions: []string{"eth0", "eth1"},
 			expected: `iptables-restore --noflush <<EOF
 # OSM sidecar interception rules
 *nat
@@ -65,6 +67,8 @@ EOF
 -A OSM_PROXY_INBOUND -p tcp --dport 15903 -j RETURN
 -A OSM_PROXY_INBOUND -p tcp --dport 15904 -j RETURN
 -A OSM_PROXY_INBOUND -p tcp -j OSM_PROXY_IN_REDIRECT
+-I OSM_PROXY_INBOUND -i eth0 -j RETURN
+-I OSM_PROXY_INBOUND -i eth1 -j RETURN
 -I OSM_PROXY_INBOUND -p tcp --match multiport --dports 30,40 -j RETURN
 -A OSM_PROXY_OUT_REDIRECT -p tcp -j REDIRECT --to-port 15001
 -A OSM_PROXY_OUT_REDIRECT -p tcp --dport 15000 -j ACCEPT
@@ -73,6 +77,8 @@ EOF
 -A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner 1500 -j RETURN
 -A OSM_PROXY_OUTBOUND -m owner --uid-owner 1500 -j RETURN
 -A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN
+-A OSM_PROXY_OUTBOUND -o eth0 -j RETURN
+-A OSM_PROXY_OUTBOUND -o eth1 -j RETURN
 -A OSM_PROXY_OUTBOUND -d 1.1.1.1/32 -j RETURN
 -A OSM_PROXY_OUTBOUND -d 2.2.2.2/32 -j RETURN
 -A OSM_PROXY_OUTBOUND -p tcp --match multiport --dports 10,20 -j RETURN
@@ -88,7 +94,7 @@ EOF
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			a := assert.New(t)
-			actual := generateIptablesCommands(tc.outboundIPRangeExclusions, tc.outboundIPRangeInclusions, tc.outboundPortExclusions, tc.inboundPortExclusions)
+			actual := generateIptablesCommands(tc.outboundIPRangeExclusions, tc.outboundIPRangeInclusions, tc.outboundPortExclusions, tc.inboundPortExclusions, tc.networkInterfaceExclusions)
 			a.Equal(tc.expected, actual)
 		})
 	}

pkg/injector/iptables_test.go

@@ -205,8 +205,10 @@ func (wh *mutatingWebhook) configurePodInit(podOS string, pod *corev1.Pod, names
 	globalOutboundIPRangeInclusionList := wh.configurator.GetMeshConfig().Spec.Traffic.OutboundIPRangeInclusionList
 	outboundIPRangeInclusionList := mergeIPRangeLists(podOutboundIPRangeInclusionList, globalOutboundIPRangeInclusionList)
 
+	networkInterfaceExclusionList := wh.configurator.GetMeshConfig().Spec.Traffic.NetworkInterfaceExclusionList
+
 	// Add the init container to the pod spec
-	initContainer := getInitContainerSpec(constants.InitContainerName, wh.configurator, outboundIPRangeExclusionList, outboundIPRangeInclusionList, outboundPortExclusionList, inboundPortExclusionList, wh.configurator.IsPrivilegedInitContainer(), wh.osmContainerPullPolicy)
+	initContainer := getInitContainerSpec(constants.InitContainerName, wh.configurator, outboundIPRangeExclusionList, outboundIPRangeInclusionList, outboundPortExclusionList, inboundPortExclusionList, wh.configurator.IsPrivilegedInitContainer(), wh.osmContainerPullPolicy, networkInterfaceExclusionList)
 	pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer)
 
 	return nil