pkg/injector: Enable podIP proxying via meshconfig setting

charts/osm/templates/preset-mesh-config.yaml

@@ -10,7 +10,8 @@ data:
         "enablePrivilegedInitContainer": {{.Values.osm.enablePrivilegedInitContainer | mustToJson}},
         "logLevel": {{.Values.osm.envoyLogLevel | mustToJson}},
         "maxDataPlaneConnections": {{.Values.osm.maxDataPlaneConnections | mustToJson}},
-        "configResyncInterval": {{.Values.osm.configResyncInterval | mustToJson}}
+        "configResyncInterval": {{.Values.osm.configResyncInterval | mustToJson}},
+        "localProxyMode": {{.Values.osm.localProxyMode | mustToJson}}
       },
       "traffic": {
         "enableEgress": {{.Values.osm.enableEgress | mustToJson}},

pkg/injector/init_container.go

@@ -10,7 +10,8 @@ import (
 func getInitContainerSpec(containerName string, cfg configurator.Configurator, outboundIPRangeExclusionList []string,
 	outboundIPRangeInclusionList []string, outboundPortExclusionList []int,
 	inboundPortExclusionList []int, enablePrivilegedInitContainer bool, pullPolicy corev1.PullPolicy, networkInterfaceExclusionList []string) corev1.Container {
-	iptablesInitCommand := generateIptablesCommands(outboundIPRangeExclusionList, outboundIPRangeInclusionList, outboundPortExclusionList, inboundPortExclusionList, networkInterfaceExclusionList)
+	proxyMode := cfg.GetMeshConfig().Spec.Sidecar.LocalProxyMode
+	iptablesInitCommand := generateIptablesCommands(proxyMode, outboundIPRangeExclusionList, outboundIPRangeInclusionList, outboundPortExclusionList, inboundPortExclusionList, networkInterfaceExclusionList)
 
 	return corev1.Container{
 		Name:            containerName,
@@ -32,5 +33,16 @@ func getInitContainerSpec(containerName string, cfg configurator.Configurator, o
 			"-c",
 			iptablesInitCommand,
 		},
+		Env: []corev1.EnvVar{
+			{
+				Name: "POD_IP",
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						APIVersion: "v1",
+						FieldPath:  "status.podIP",
+					},
+				},
+			},
+		},
 	}
 }

pkg/injector/init_container_test.go

@@ -7,6 +7,8 @@ import (
 	"github.com/golang/mock/gomock"
 	corev1 "k8s.io/api/core/v1"
 
+	configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
+
 	"github.com/openservicemesh/osm/pkg/configurator"
 )
 
@@ -26,6 +28,89 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func()
 	Context("test getInitContainerSpec()", func() {
 		It("Creates init container without ip range exclusion list", func() {
 			mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1)
+			mockConfigurator.EXPECT().GetMeshConfig().Return(configv1alpha2.MeshConfig{
+				Spec: configv1alpha2.MeshConfigSpec{
+					Sidecar: configv1alpha2.SidecarSpec{
+						LocalProxyMode: configv1alpha2.LocalProxyModeLocalhost,
+					},
+				},
+			}).Times(1)
+			privileged := privilegedFalse
+			actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, nil, privileged, corev1.PullAlways, nil)
+
+			expected := corev1.Container{
+				Name:            "-container-name-",
+				Image:           "-init-container-image-",
+				ImagePullPolicy: corev1.PullAlways,
+				Command:         []string{"/bin/sh"},
+				Args: []string{
+					"-c",
+					`iptables-restore --noflush <<EOF
+# OSM sidecar interception rules
+*nat
+:OSM_PROXY_INBOUND - [0:0]
+:OSM_PROXY_IN_REDIRECT - [0:0]
+:OSM_PROXY_OUTBOUND - [0:0]
+:OSM_PROXY_OUT_REDIRECT - [0:0]
+-A OSM_PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003
+-A PREROUTING -p tcp -j OSM_PROXY_INBOUND
+-A OSM_PROXY_INBOUND -p tcp --dport 15010 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15901 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15902 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15903 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15904 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp -j OSM_PROXY_IN_REDIRECT
+-A OSM_PROXY_OUT_REDIRECT -p tcp -j REDIRECT --to-port 15001
+-A OSM_PROXY_OUT_REDIRECT -p tcp --dport 15000 -j ACCEPT
+-A OUTPUT -p tcp -j OSM_PROXY_OUTBOUND
+-A OSM_PROXY_OUTBOUND -o lo ! -d 127.0.0.1/32 -m owner --uid-owner 1500 -j OSM_PROXY_IN_REDIRECT
+-A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner 1500 -j RETURN
+-A OSM_PROXY_OUTBOUND -m owner --uid-owner 1500 -j RETURN
+-A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN
+-A OSM_PROXY_OUTBOUND -j OSM_PROXY_OUT_REDIRECT
+COMMIT
+EOF
+`,
+				},
+				WorkingDir: "",
+				Resources:  corev1.ResourceRequirements{},
+				SecurityContext: &corev1.SecurityContext{
+					Capabilities: &corev1.Capabilities{
+						Add: []corev1.Capability{
+							"NET_ADMIN",
+						},
+					},
+					Privileged:   &privilegedFalse,
+					RunAsNonRoot: &runAsNonRootFalse,
+					RunAsUser:    &runAsUserID,
+				},
+				Env: []corev1.EnvVar{
+					{
+						Name: "POD_IP",
+						ValueFrom: &corev1.EnvVarSource{
+							FieldRef: &corev1.ObjectFieldSelector{
+								APIVersion: "v1",
+								FieldPath:  "status.podIP",
+							},
+						},
+					},
+				},
+				Stdin:     false,
+				StdinOnce: false,
+				TTY:       false,
+			}
+
+			Expect(actual).To(Equal(expected))
+		})
+		It("Sets podIP DNAT rule if set in meshconfig", func() {
+			mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1)
+			mockConfigurator.EXPECT().GetMeshConfig().Return(configv1alpha2.MeshConfig{
+				Spec: configv1alpha2.MeshConfigSpec{
+					Sidecar: configv1alpha2.SidecarSpec{
+						LocalProxyMode: configv1alpha2.LocalProxyModePodIP,
+					},
+				},
+			}).Times(1)
 			privileged := privilegedFalse
 			actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, nil, privileged, corev1.PullAlways, nil)
 
@@ -58,6 +143,7 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func()
 -A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner 1500 -j RETURN
 -A OSM_PROXY_OUTBOUND -m owner --uid-owner 1500 -j RETURN
 -A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN
+-I OUTPUT -p tcp -o lo -d 127.0.0.1/32 -m owner --uid-owner 1500 -j DNAT --to-destination $POD_IP
 -A OSM_PROXY_OUTBOUND -j OSM_PROXY_OUT_REDIRECT
 COMMIT
 EOF
@@ -75,6 +161,17 @@ EOF
 					RunAsNonRoot: &runAsNonRootFalse,
 					RunAsUser:    &runAsUserID,
 				},
+				Env: []corev1.EnvVar{
+					{
+						Name: "POD_IP",
+						ValueFrom: &corev1.EnvVarSource{
+							FieldRef: &corev1.ObjectFieldSelector{
+								APIVersion: "v1",
+								FieldPath:  "status.podIP",
+							},
+						},
+					},
+				},
 				Stdin:     false,
 				StdinOnce: false,
 				TTY:       false,

pkg/injector/iptables.go

@@ -5,6 +5,8 @@ import (
 	"strconv"
 	"strings"
 
+	configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
+
 	"github.com/openservicemesh/osm/pkg/constants"
 )
 
@@ -60,7 +62,7 @@ var iptablesInboundStaticRules = []string{
 }
 
 // generateIptablesCommands generates a list of iptables commands to set up sidecar interception and redirection
-func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundIPRangeInclusionList []string, outboundPortExclusionList []int, inboundPortExclusionList []int, networkInterfaceExclusionList []string) string {
+func generateIptablesCommands(proxyMode configv1alpha2.LocalProxyMode, outboundIPRangeExclusionList []string, outboundIPRangeInclusionList []string, outboundPortExclusionList []int, inboundPortExclusionList []int, networkInterfaceExclusionList []string) string {
 	var rules strings.Builder
 
 	fmt.Fprintln(&rules, `# OSM sidecar interception rules
@@ -95,6 +97,13 @@ func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundIPR
 	// 3. Create outbound rules
 	cmds = append(cmds, iptablesOutboundStaticRules...)
 
+	if proxyMode == configv1alpha2.LocalProxyModePodIP {
+		// For envoy -> local service container proxying, send traffic to pod IP instead of localhost
+		// *Note: it is important to use the insert option '-I' instead of the append option '-A' to ensure the
+		// DNAT to the pod ip for envoy -> localhost traffic happens before the rule that redirects traffic to the proxy
+		cmds = append(cmds, fmt.Sprintf("-I OUTPUT -p tcp -o lo -d 127.0.0.1/32 -m owner --uid-owner %d -j DNAT --to-destination $POD_IP", constants.EnvoyUID))
+	}
+
 	// Ignore outbound traffic in specified interfaces
 	for _, iface := range networkInterfaceExclusionList {
 		cmds = append(cmds, fmt.Sprintf("-A OSM_PROXY_OUTBOUND -o %s -j RETURN", iface))

pkg/injector/iptables_test.go

@@ -4,11 +4,14 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
 )
 
 func TestGenerateIptablesCommands(t *testing.T) {
 	testCases := []struct {
 		name                       string
+		proxyMode                  configv1alpha2.LocalProxyMode
 		outboundIPRangeExclusions  []string
 		outboundIPRangeInclusions  []string
 		outboundPortExclusions     []int
@@ -87,14 +90,46 @@ EOF
 -A OSM_PROXY_OUTBOUND -j RETURN
 COMMIT
 EOF
+`,
+		},
+		{
+			name:      "proxy mode pod ip",
+			proxyMode: configv1alpha2.LocalProxyModePodIP,
+			expected: `iptables-restore --noflush <<EOF
+# OSM sidecar interception rules
+*nat
+:OSM_PROXY_INBOUND - [0:0]
+:OSM_PROXY_IN_REDIRECT - [0:0]
+:OSM_PROXY_OUTBOUND - [0:0]
+:OSM_PROXY_OUT_REDIRECT - [0:0]
+-A OSM_PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003
+-A PREROUTING -p tcp -j OSM_PROXY_INBOUND
+-A OSM_PROXY_INBOUND -p tcp --dport 15010 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15901 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15902 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15903 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp --dport 15904 -j RETURN
+-A OSM_PROXY_INBOUND -p tcp -j OSM_PROXY_IN_REDIRECT
+-A OSM_PROXY_OUT_REDIRECT -p tcp -j REDIRECT --to-port 15001
+-A OSM_PROXY_OUT_REDIRECT -p tcp --dport 15000 -j ACCEPT
+-A OUTPUT -p tcp -j OSM_PROXY_OUTBOUND
+-A OSM_PROXY_OUTBOUND -o lo ! -d 127.0.0.1/32 -m owner --uid-owner 1500 -j OSM_PROXY_IN_REDIRECT
+-A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner 1500 -j RETURN
+-A OSM_PROXY_OUTBOUND -m owner --uid-owner 1500 -j RETURN
+-A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN
+-I OUTPUT -p tcp -o lo -d 127.0.0.1/32 -m owner --uid-owner 1500 -j DNAT --to-destination $POD_IP
+-A OSM_PROXY_OUTBOUND -j OSM_PROXY_OUT_REDIRECT
+COMMIT
+EOF
 `,
 		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			a := assert.New(t)
-			actual := generateIptablesCommands(tc.outboundIPRangeExclusions, tc.outboundIPRangeInclusions, tc.outboundPortExclusions, tc.inboundPortExclusions, tc.networkInterfaceExclusions)
+
+			actual := generateIptablesCommands(tc.proxyMode, tc.outboundIPRangeExclusions, tc.outboundIPRangeInclusions, tc.outboundPortExclusions, tc.inboundPortExclusions, tc.networkInterfaceExclusions)
 			a.Equal(tc.expected, actual)
 		})
 	}

tests/e2e/e2e_pod_client_server_test.go → tests/e2e/e2e_client_server_connectivity_test.go

@@ -8,10 +8,11 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
+
 	"github.com/openservicemesh/osm/pkg/constants"
 	. "github.com/openservicemesh/osm/tests/framework"
 )
@@ -25,26 +26,33 @@ var _ = OSMDescribe("Test HTTP traffic from 1 pod client -> 1 pod server",
 	func() {
 		Context("Test traffic flowing from client to server with a Kubernetes Service for the Source: HTTP", func() {
 			withSourceKubernetesService := true
-			testTraffic(withSourceKubernetesService)
+			testTraffic(withSourceKubernetesService, PodCommandDefault)
 		})
 
 		Context("Test traffic flowing from client to server without a Kubernetes Service for the Source: HTTP", func() {
 			// Prior iterations of OSM required that a source pod belong to a Kubernetes service
 			// for the Envoy proxy to be configured for outbound traffic to some remote server.
 			// This test ensures we test this scenario: client Pod is not associated w/ a service.
 			withSourceKubernetesService := false
-			testTraffic(withSourceKubernetesService)
+			testTraffic(withSourceKubernetesService, PodCommandDefault)
+		})
+
+		Context("Test traffic flowing from client to a server with a podIP bind", func() {
+			// Prior iterations of OSM didn't allow mesh services to bind to the podIP
+			// This test ensures that that behavior is configurable via MeshConfig
+			withSourceKubernetesService := true
+			testTraffic(withSourceKubernetesService, []string{"gunicorn", "-b", "$(POD_IP):80", "httpbin:app", "-k", "gevent"}, WithLocalProxyMode(v1alpha2.LocalProxyModePodIP))
 		})
 	})
 
-func testTraffic(withSourceKubernetesService bool) {
+func testTraffic(withSourceKubernetesService bool, destPodCommand []string, installOpts ...InstallOsmOpt) {
 	const sourceName = "client"
 	const destName = "server"
 	var ns = []string{sourceName, destName}
 
 	It("Tests HTTP traffic for client pod -> server pod", func() {
 		// Install OSM
-		Expect(Td.InstallOSM(Td.GetOSMInstallOpts())).To(Succeed())
+		Expect(Td.InstallOSM(Td.GetOSMInstallOpts(installOpts...))).To(Succeed())
 
 		// Create Test NS
 		for _, n := range ns {
@@ -53,7 +61,7 @@ func testTraffic(withSourceKubernetesService bool) {
 		}
 
 		// Get simple pod definitions for the HTTP server
-		svcAccDef, podDef, svcDef, err := Td.GetOSSpecificHTTPBinPod(destName, destName)
+		svcAccDef, podDef, svcDef, err := Td.GetOSSpecificHTTPBinPod(destName, destName, destPodCommand...)
 		Expect(err).NotTo(HaveOccurred())
 
 		_, err = Td.CreateServiceAccount(destName, &svcAccDef)

tests/framework/common.go

@@ -328,13 +328,21 @@ nodeRegistration:
 	return nil
 }
 
+// WithLocalProxyMode sets the LocalProxyMode for OSM
+func WithLocalProxyMode(mode configv1alpha2.LocalProxyMode) InstallOsmOpt {
+	return func(opts *InstallOSMOpts) {
+		opts.LocalProxyMode = mode
+	}
+}
+
 // GetOSMInstallOpts initializes install options for OSM
-func (td *OsmTestData) GetOSMInstallOpts() InstallOSMOpts {
+func (td *OsmTestData) GetOSMInstallOpts(options ...InstallOsmOpt) InstallOSMOpts {
 	enablePrivilegedInitContainer := false
 	if td.DeployOnOpenShift {
 		enablePrivilegedInitContainer = true
 	}
-	return InstallOSMOpts{
+
+	baseOpts := InstallOSMOpts{
 		ControlPlaneNS:          td.OsmNamespace,
 		CertManager:             defaultCertManager,
 		ContainerRegistryLoc:    td.CtrRegistryServer,
@@ -364,6 +372,12 @@ func (td *OsmTestData) GetOSMInstallOpts() InstallOSMOpts {
 		EnablePrivilegedInitContainer: enablePrivilegedInitContainer,
 		EnableIngressBackendPolicy:    true,
 	}
+
+	for _, opt := range options {
+		opt(&baseOpts)
+	}
+
+	return baseOpts
 }
 
 // LoadImagesToKind loads the list of images to the node for Kind clusters
@@ -415,7 +429,7 @@ func (td *OsmTestData) LoadImagesToKind(imageNames []string) error {
 	return nil
 }
 
-func setMeshConfigToDefault(instOpts InstallOSMOpts, meshConfig *configv1alpha2.MeshConfig) (defaultConfig *configv1alpha2.MeshConfig) {
+func setMeshConfigToDefault(instOpts InstallOSMOpts, meshConfig *configv1alpha2.MeshConfig) *configv1alpha2.MeshConfig {
 	meshConfig.Spec.Traffic.EnableEgress = instOpts.EgressEnabled
 	meshConfig.Spec.Traffic.EnablePermissiveTrafficPolicyMode = instOpts.EnablePermissiveMode
 	meshConfig.Spec.Traffic.OutboundPortExclusionList = []int{}
@@ -491,6 +505,10 @@ func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error {
 		fmt.Sprintf("osm.enableReconciler=%v", instOpts.EnableReconciler),
 	)
 
+	if instOpts.LocalProxyMode != "" {
+		instOpts.SetOverrides = append(instOpts.SetOverrides, fmt.Sprintf("osm.localProxyMode=%s", instOpts.LocalProxyMode))
+	}
+
 	switch instOpts.CertManager {
 	case "vault":
 		if err := td.installVault(instOpts); err != nil {