pkg/injector: Enable podIP proxying via meshconfig setting

charts/osm/templates/preset-mesh-config.yaml

@@ -10,7 +10,8 @@ data:
         "enablePrivilegedInitContainer": {{.Values.osm.enablePrivilegedInitContainer | mustToJson}},
         "logLevel": {{.Values.osm.envoyLogLevel | mustToJson}},
         "maxDataPlaneConnections": {{.Values.osm.maxDataPlaneConnections | mustToJson}},
-        "configResyncInterval": {{.Values.osm.configResyncInterval | mustToJson}}
+        "configResyncInterval": {{.Values.osm.configResyncInterval | mustToJson}},
+        "localProxyMode": {{.Values.osm.localProxyMode | mustToJson}}
       },
       "traffic": {
         "enableEgress": {{.Values.osm.enableEgress | mustToJson}},

pkg/injector/init_container.go

@@ -10,7 +10,7 @@ import (
 func getInitContainerSpec(containerName string, cfg configurator.Configurator, outboundIPRangeExclusionList []string,
 	outboundIPRangeInclusionList []string, outboundPortExclusionList []int,
 	inboundPortExclusionList []int, enablePrivilegedInitContainer bool, pullPolicy corev1.PullPolicy, networkInterfaceExclusionList []string) corev1.Container {
-	iptablesInitCommand := generateIptablesCommands(outboundIPRangeExclusionList, outboundIPRangeInclusionList, outboundPortExclusionList, inboundPortExclusionList, networkInterfaceExclusionList)
+	iptablesInitCommand := generateIptablesCommands(cfg, outboundIPRangeExclusionList, outboundIPRangeInclusionList, outboundPortExclusionList, inboundPortExclusionList, networkInterfaceExclusionList)
 
 	return corev1.Container{
 		Name:            containerName,
@@ -32,5 +32,16 @@ func getInitContainerSpec(containerName string, cfg configurator.Configurator, o
 			"-c",
 			iptablesInitCommand,
 		},
+		Env: []corev1.EnvVar{
+			{
+				Name: "POD_IP",
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						APIVersion: "v1",
+						FieldPath:  "status.podIP",
+					},
+				},
+			},
+		},
 	}
 }

pkg/injector/init_container_test.go

@@ -7,6 +7,8 @@ import (
 	"github.com/golang/mock/gomock"
 	corev1 "k8s.io/api/core/v1"
 
+	configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
+
 	"github.com/openservicemesh/osm/pkg/configurator"
 )
 
@@ -26,6 +28,89 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func()
 	Context("test getInitContainerSpec()", func() {
 		It("Creates init container without ip range exclusion list", func() {
 			mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1)
+			mockConfigurator.EXPECT().GetMeshConfig().Return(configv1alpha2.MeshConfig{
+				Spec: configv1alpha2.MeshConfigSpec{
+					Sidecar: configv1alpha2.SidecarSpec{
+						LocalProxyMode: configv1alpha2.LocalProxyModeLocalhost,
+					},
+				},
+			}).Times(1)
+			privileged := privilegedFalse
+			actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, nil, privileged, corev1.PullAlways, nil)
+
+			expected := corev1.Container{
+				Name:            "-container-name-",
+				Image:           "-init-container-image-",
+				ImagePullPolicy: corev1.PullAlways,
+				Command:         []string{"/bin/sh"},
+				Args: []string{
+					"-c",
+					`iptables-restore --noflush <<EOF
+# OSM sidecar interception rules
+*nat
+:OSM_PROXY_INBOUND - [0:0]
+:OSM_PROXY_IN_REDIRECT - [0:0]
+:OSM_PROXY_OUTBOUND - [0:0]
+:OSM_PROXY_OUT_REDIRECT - [0:0]
+-A OSM_PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003
+-A PREROUTING -p tcp -j OSM_PROXY_INBOUND
+-A OSM_PROXY_INBOUND -p tcp --dport 15010 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15901 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15902 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15903 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15904 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp -j OSM_PROXY_IN_REDIRECT
+-A OSM_PROXY_OUT_REDIRECT -p tcp -j REDIRECT --to-port 15001
+-A OSM_PROXY_OUT_REDIRECT -p tcp --dport 15000 -j ACCEPT
+-A OUTPUT -p tcp -j OSM_PROXY_OUTBOUND
+-A OSM_PROXY_OUTBOUND -o lo ! -d 127.0.0.1/32 -m owner --uid-owner 1500 -j OSM_PROXY_IN_REDIRECT
+-A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner 1500 -j RETURN
+-A OSM_PROXY_OUTBOUND -m owner --uid-owner 1500 -j RETURN
+-A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN
+-A OSM_PROXY_OUTBOUND -j OSM_PROXY_OUT_REDIRECT
+COMMIT
+EOF
+`,
+				},
+				WorkingDir: "",
+				Resources:  corev1.ResourceRequirements{},
+				SecurityContext: &corev1.SecurityContext{
+					Capabilities: &corev1.Capabilities{
+						Add: []corev1.Capability{
+							"NET_ADMIN",
+						},
+					},
+					Privileged:   &privilegedFalse,
+					RunAsNonRoot: &runAsNonRootFalse,
+					RunAsUser:    &runAsUserID,
+				},
+				Env: []corev1.EnvVar{
+					{
+						Name: "POD_IP",
+						ValueFrom: &corev1.EnvVarSource{
+							FieldRef: &corev1.ObjectFieldSelector{
+								APIVersion: "v1",
+								FieldPath:  "status.podIP",
+							},
+						},
+					},
+				},
+				Stdin:     false,
+				StdinOnce: false,
+				TTY:       false,
+			}
+
+			Expect(actual).To(Equal(expected))
+		})
+		It("Sets podIP DNAT rule if set in meshconfig", func() {
+			mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1)
+			mockConfigurator.EXPECT().GetMeshConfig().Return(configv1alpha2.MeshConfig{
+				Spec: configv1alpha2.MeshConfigSpec{
+					Sidecar: configv1alpha2.SidecarSpec{
+						LocalProxyMode: configv1alpha2.LocalProxyModePodIP,
+					},
+				},
+			}).Times(1)
 			privileged := privilegedFalse
 			actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, nil, privileged, corev1.PullAlways, nil)
 
@@ -53,6 +138,7 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func()
 -A OSM_PROXY_INBOUND -p tcp -j OSM_PROXY_IN_REDIRECT
 -A OSM_PROXY_OUT_REDIRECT -p tcp -j REDIRECT --to-port 15001
 -A OSM_PROXY_OUT_REDIRECT -p tcp --dport 15000 -j ACCEPT
+-A OUTPUT -p tcp -o lo -d 127.0.0.1/32 -m owner --uid-owner 1500 -j DNAT --to-destination $POD_IP
 -A OUTPUT -p tcp -j OSM_PROXY_OUTBOUND
 -A OSM_PROXY_OUTBOUND -o lo ! -d 127.0.0.1/32 -m owner --uid-owner 1500 -j OSM_PROXY_IN_REDIRECT
 -A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner 1500 -j RETURN
@@ -75,6 +161,17 @@ EOF
 					RunAsNonRoot: &runAsNonRootFalse,
 					RunAsUser:    &runAsUserID,
 				},
+				Env: []corev1.EnvVar{
+					{
+						Name: "POD_IP",
+						ValueFrom: &corev1.EnvVarSource{
+							FieldRef: &corev1.ObjectFieldSelector{
+								APIVersion: "v1",
+								FieldPath:  "status.podIP",
+							},
+						},
+					},
+				},
 				Stdin:     false,
 				StdinOnce: false,
 				TTY:       false,

pkg/injector/iptables.go

@@ -5,34 +5,50 @@ import (
 	"strconv"
 	"strings"
 
+	configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
+
+	"github.com/openservicemesh/osm/pkg/configurator"
 	"github.com/openservicemesh/osm/pkg/constants"
 )
 
-// iptablesOutboundStaticRules is the list of iptables rules related to outbound traffic interception and redirection
-var iptablesOutboundStaticRules = []string{
-	// Redirects outbound TCP traffic hitting OSM_PROXY_OUT_REDIRECT chain to Envoy's outbound listener port
-	fmt.Sprintf("-A OSM_PROXY_OUT_REDIRECT -p tcp -j REDIRECT --to-port %d", constants.EnvoyOutboundListenerPort),
+func genIPTablesOutboundStaticRules(cfg configurator.Configurator) []string {
+	// iptablesOutboundStaticRules is the list of iptables rules related to outbound traffic interception and redirection
+	iptablesOutboundStaticRules := []string{
+		// Redirects outbound TCP traffic hitting OSM_PROXY_OUT_REDIRECT chain to Envoy's outbound listener port
+		fmt.Sprintf("-A OSM_PROXY_OUT_REDIRECT -p tcp -j REDIRECT --to-port %d", constants.EnvoyOutboundListenerPort),
+
+		// Traffic to the Proxy Admin port flows to the Proxy -- not redirected
+		fmt.Sprintf("-A OSM_PROXY_OUT_REDIRECT -p tcp --dport %d -j ACCEPT", constants.EnvoyAdminPort),
+	}
 
-	// Traffic to the Proxy Admin port flows to the Proxy -- not redirected
-	fmt.Sprintf("-A OSM_PROXY_OUT_REDIRECT -p tcp --dport %d -j ACCEPT", constants.EnvoyAdminPort),
+	localProxyMode := cfg.GetMeshConfig().Spec.Sidecar.LocalProxyMode
 
-	// For outbound TCP traffic jump from OUTPUT chain to OSM_PROXY_OUTBOUND chain
-	"-A OUTPUT -p tcp -j OSM_PROXY_OUTBOUND",
+	if localProxyMode == configv1alpha2.LocalProxyModePodIP {
+		// For envoy -> local service container proxying, send traffic to pod IP instead of localhost
+		iptablesOutboundStaticRules = append(iptablesOutboundStaticRules, fmt.Sprintf("-A OUTPUT -p tcp -o lo -d 127.0.0.1/32 -m owner --uid-owner %d -j DNAT --to-destination $POD_IP", constants.EnvoyUID))
+	}
 
-	// Outbound traffic from Envoy to the local app over the loopback interface should jump to the inbound proxy redirect chain.
-	// So when an app directs traffic to itself via the k8s service, traffic flows as follows:
-	// app -> local envoy's outbound listener -> iptables -> local envoy's inbound listener -> app
-	fmt.Sprintf("-A OSM_PROXY_OUTBOUND -o lo ! -d 127.0.0.1/32 -m owner --uid-owner %d -j OSM_PROXY_IN_REDIRECT", constants.EnvoyUID),
+	iptablesOutboundStaticRules = append(iptablesOutboundStaticRules, []string{
+		// For all other outbound TCP traffic jump from OUTPUT chain to OSM_PROXY_OUTBOUND chain
+		"-A OUTPUT -p tcp -j OSM_PROXY_OUTBOUND",
 
-	// Outbound traffic from the app to itself over the loopback interface is not be redirected via the proxy.
-	// E.g. when app sends traffic to itself via the pod IP.
-	fmt.Sprintf("-A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner %d -j RETURN", constants.EnvoyUID),
+		// Outbound traffic from Envoy to the local app over the loopback interface should jump to the inbound proxy redirect chain.
+		// So when an app directs traffic to itself via the k8s service, traffic flows as follows:
+		// app -> local envoy's outbound listener -> iptables -> local envoy's inbound listener -> app
+		fmt.Sprintf("-A OSM_PROXY_OUTBOUND -o lo ! -d 127.0.0.1/32 -m owner --uid-owner %d -j OSM_PROXY_IN_REDIRECT", constants.EnvoyUID),
 
-	// Don't redirect Envoy traffic back to itself, return it to the next chain for processing
-	fmt.Sprintf("-A OSM_PROXY_OUTBOUND -m owner --uid-owner %d -j RETURN", constants.EnvoyUID),
+		// Outbound traffic from the app to itself over the loopback interface is not be redirected via the proxy.
+		// E.g. when app sends traffic to itself via the pod IP.
+		fmt.Sprintf("-A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner %d -j RETURN", constants.EnvoyUID),
 
-	// Skip localhost traffic, doesn't need to be routed via the proxy
-	"-A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN",
+		// Don't redirect Envoy traffic back to itself, return it to the next chain for processing
+		fmt.Sprintf("-A OSM_PROXY_OUTBOUND -m owner --uid-owner %d -j RETURN", constants.EnvoyUID),
+
+		// Skip localhost traffic, doesn't need to be routed via the proxy
+		"-A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN",
+	}...)
+
+	return iptablesOutboundStaticRules
 }
 
 // iptablesInboundStaticRules is the list of iptables rules related to inbound traffic interception and redirection
@@ -60,7 +76,7 @@ var iptablesInboundStaticRules = []string{
 }
 
 // generateIptablesCommands generates a list of iptables commands to set up sidecar interception and redirection
-func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundIPRangeInclusionList []string, outboundPortExclusionList []int, inboundPortExclusionList []int, networkInterfaceExclusionList []string) string {
+func generateIptablesCommands(cfg configurator.Configurator, outboundIPRangeExclusionList []string, outboundIPRangeInclusionList []string, outboundPortExclusionList []int, inboundPortExclusionList []int, networkInterfaceExclusionList []string) string {
 	var rules strings.Builder
 
 	fmt.Fprintln(&rules, `# OSM sidecar interception rules
@@ -92,6 +108,8 @@ func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundIPR
 		cmds = append(cmds, rule)
 	}
 
+	iptablesOutboundStaticRules := genIPTablesOutboundStaticRules(cfg)
+
 	// 3. Create outbound rules
 	cmds = append(cmds, iptablesOutboundStaticRules...)
 

pkg/injector/iptables_test.go

@@ -3,12 +3,19 @@ package injector
 import (
 	"testing"
 
+	"github.com/golang/mock/gomock"
+	. "github.com/onsi/ginkgo"
 	"github.com/stretchr/testify/assert"
+
+	configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
+
+	"github.com/openservicemesh/osm/pkg/configurator"
 )
 
 func TestGenerateIptablesCommands(t *testing.T) {
 	testCases := []struct {
 		name                       string
+		proxyMode                  configv1alpha2.LocalProxyMode
 		outboundIPRangeExclusions  []string
 		outboundIPRangeInclusions  []string
 		outboundPortExclusions     []int
@@ -87,14 +94,55 @@ EOF
 -A OSM_PROXY_OUTBOUND -j RETURN
 COMMIT
 EOF
+`,
+		},
+		{
+			name:      "proxy mode pod ip",
+			proxyMode: configv1alpha2.LocalProxyModePodIP,
+			expected: `iptables-restore --noflush <<EOF
+# OSM sidecar interception rules
+*nat
+:OSM_PROXY_INBOUND - [0:0]
+:OSM_PROXY_IN_REDIRECT - [0:0]
+:OSM_PROXY_OUTBOUND - [0:0]
+:OSM_PROXY_OUT_REDIRECT - [0:0]
+-A OSM_PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003
+-A PREROUTING -p tcp -j OSM_PROXY_INBOUND
+-A OSM_PROXY_INBOUND -p tcp --dport 15010 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15901 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15902 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15903 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15904 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp -j OSM_PROXY_IN_REDIRECT
+-A OSM_PROXY_OUT_REDIRECT -p tcp -j REDIRECT --to-port 15001
+-A OSM_PROXY_OUT_REDIRECT -p tcp --dport 15000 -j ACCEPT
+-A OUTPUT -p tcp -o lo -d 127.0.0.1/32 -m owner --uid-owner 1500 -j DNAT --to-destination $POD_IP
+-A OUTPUT -p tcp -j OSM_PROXY_OUTBOUND
+-A OSM_PROXY_OUTBOUND -o lo ! -d 127.0.0.1/32 -m owner --uid-owner 1500 -j OSM_PROXY_IN_REDIRECT
+-A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner 1500 -j RETURN
+-A OSM_PROXY_OUTBOUND -m owner --uid-owner 1500 -j RETURN
+-A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN
+-A OSM_PROXY_OUTBOUND -j OSM_PROXY_OUT_REDIRECT
+COMMIT
+EOF
 `,
 		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			a := assert.New(t)
-			actual := generateIptablesCommands(tc.outboundIPRangeExclusions, tc.outboundIPRangeInclusions, tc.outboundPortExclusions, tc.inboundPortExclusions, tc.networkInterfaceExclusions)
+			mockCtrl := gomock.NewController(GinkgoT())
+			mockConfigurator := configurator.NewMockConfigurator(mockCtrl)
+			mockConfigurator.EXPECT().GetMeshConfig().Return(configv1alpha2.MeshConfig{
+				Spec: configv1alpha2.MeshConfigSpec{
+					Sidecar: configv1alpha2.SidecarSpec{
+						LocalProxyMode: tc.proxyMode,
+					},
+				},
+			}).Times(1)
+
+			actual := generateIptablesCommands(mockConfigurator, tc.outboundIPRangeExclusions, tc.outboundIPRangeInclusions, tc.outboundPortExclusions, tc.inboundPortExclusions, tc.networkInterfaceExclusions)
 			a.Equal(tc.expected, actual)
 		})
 	}

tests/e2e/e2e_pod_client_server_test.go

@@ -8,10 +8,11 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
+
 	"github.com/openservicemesh/osm/pkg/constants"
 	. "github.com/openservicemesh/osm/tests/framework"
 )
@@ -25,26 +26,33 @@ var _ = OSMDescribe("Test HTTP traffic from 1 pod client -> 1 pod server",
 	func() {
 		Context("Test traffic flowing from client to server with a Kubernetes Service for the Source: HTTP", func() {
 			withSourceKubernetesService := true
-			testTraffic(withSourceKubernetesService)
+			testTraffic(withSourceKubernetesService, PodCommandDefault)
 		})
 
 		Context("Test traffic flowing from client to server without a Kubernetes Service for the Source: HTTP", func() {
 			// Prior iterations of OSM required that a source pod belong to a Kubernetes service
 			// for the Envoy proxy to be configured for outbound traffic to some remote server.
 			// This test ensures we test this scenario: client Pod is not associated w/ a service.
 			withSourceKubernetesService := false
-			testTraffic(withSourceKubernetesService)
+			testTraffic(withSourceKubernetesService, PodCommandDefault)
+		})
+
+		Context("Test traffic flowing from client to a server with a podIP bind", func() {
+			// Prior iterations of OSM didn't allow mesh services to bind to the podIP
+			// This test ensures that that behavior is configurable via MeshConfig
+			withSourceKubernetesService := true
+			testTraffic(withSourceKubernetesService, []string{"gunicorn", "-b", "$(POD_IP):14001", "httpbin:app", "-k", "gevent"}, WithLocalProxyMode(v1alpha2.LocalProxyModePodIP))
 		})
 	})
 
-func testTraffic(withSourceKubernetesService bool) {
+func testTraffic(withSourceKubernetesService bool, destPodCommand []string, installOpts ...InstallOsmOpt) {
 	const sourceName = "client"
 	const destName = "server"
 	var ns = []string{sourceName, destName}
 
 	It("Tests HTTP traffic for client pod -> server pod", func() {
 		// Install OSM
-		Expect(Td.InstallOSM(Td.GetOSMInstallOpts())).To(Succeed())
+		Expect(Td.InstallOSM(Td.GetOSMInstallOpts(installOpts...))).To(Succeed())
 
 		// Create Test NS
 		for _, n := range ns {
@@ -53,7 +61,7 @@ func testTraffic(withSourceKubernetesService bool) {
 		}
 
 		// Get simple pod definitions for the HTTP server
-		svcAccDef, podDef, svcDef, err := Td.GetOSSpecificHTTPBinPod(destName, destName)
+		svcAccDef, podDef, svcDef, err := Td.GetOSSpecificHTTPBinPod(destName, destName, destPodCommand...)
 		Expect(err).NotTo(HaveOccurred())
 
 		_, err = Td.CreateServiceAccount(destName, &svcAccDef)