Skip to content

Commit cbdcfe1

Browse files
keithmattixkeithmattix
committed
pkg/injector: Enable podIP proxying via meshconfig setting (openservicemesh#4701)
* pkg/injector: Enable podIP proxying via meshconfig setting Fixes openservicemesh#4653. Allows customers to choose how their traffic gets proxied Signed-off-by: Keith Mattix II <[email protected]> * Fix linting and test errors Signed-off-by: Keith Mattix II <[email protected]> * Inject pod_ip into initcontainer Signed-off-by: Keith Mattix II <[email protected]> * Fix port in pod command Signed-off-by: Keith Mattix II <[email protected]> * Reduce leaky abstraction by not passing configurator to generateIPTables Signed-off-by: Keith Mattix II <[email protected]> * Use iptables -I to make the rules dynamic Signed-off-by: Keith Mattix II <[email protected]> * Fix array append Signed-off-by: Keith Mattix II <[email protected]>
1 parent 868c132 commit cbdcfe1

File tree

9 files changed

+232
-24
lines changed

9 files changed

+232
-24
lines changed

‎charts/osm/templates/preset-mesh-config.yaml

+2-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ data:
1010
"enablePrivilegedInitContainer": {{.Values.osm.enablePrivilegedInitContainer | mustToJson}},
1111
"logLevel": {{.Values.osm.envoyLogLevel | mustToJson}},
1212
"maxDataPlaneConnections": {{.Values.osm.maxDataPlaneConnections | mustToJson}},
13-
"configResyncInterval": {{.Values.osm.configResyncInterval | mustToJson}}
13+
"configResyncInterval": {{.Values.osm.configResyncInterval | mustToJson}},
14+
"localProxyMode": {{.Values.osm.localProxyMode | mustToJson}}
1415
},
1516
"traffic": {
1617
"enableEgress": {{.Values.osm.enableEgress | mustToJson}},

‎pkg/injector/init_container.go

+13-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ import (
1010
func getInitContainerSpec(containerName string, cfg configurator.Configurator, outboundIPRangeExclusionList []string,
1111
outboundIPRangeInclusionList []string, outboundPortExclusionList []int,
1212
inboundPortExclusionList []int, enablePrivilegedInitContainer bool, pullPolicy corev1.PullPolicy, networkInterfaceExclusionList []string) corev1.Container {
13-
iptablesInitCommand := generateIptablesCommands(outboundIPRangeExclusionList, outboundIPRangeInclusionList, outboundPortExclusionList, inboundPortExclusionList, networkInterfaceExclusionList)
13+
proxyMode := cfg.GetMeshConfig().Spec.Sidecar.LocalProxyMode
14+
iptablesInitCommand := generateIptablesCommands(proxyMode, outboundIPRangeExclusionList, outboundIPRangeInclusionList, outboundPortExclusionList, inboundPortExclusionList, networkInterfaceExclusionList)
1415

1516
return corev1.Container{
1617
Name: containerName,
@@ -32,5 +33,16 @@ func getInitContainerSpec(containerName string, cfg configurator.Configurator, o
3233
"-c",
3334
iptablesInitCommand,
3435
},
36+
Env: []corev1.EnvVar{
37+
{
38+
Name: "POD_IP",
39+
ValueFrom: &corev1.EnvVarSource{
40+
FieldRef: &corev1.ObjectFieldSelector{
41+
APIVersion: "v1",
42+
FieldPath: "status.podIP",
43+
},
44+
},
45+
},
46+
},
3547
}
3648
}

‎pkg/injector/init_container_test.go

+97
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import (
77
"github.com/golang/mock/gomock"
88
corev1 "k8s.io/api/core/v1"
99

10+
configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
11+
1012
"github.com/openservicemesh/osm/pkg/configurator"
1113
)
1214

@@ -26,6 +28,89 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func()
2628
Context("test getInitContainerSpec()", func() {
2729
It("Creates init container without ip range exclusion list", func() {
2830
mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1)
31+
mockConfigurator.EXPECT().GetMeshConfig().Return(configv1alpha2.MeshConfig{
32+
Spec: configv1alpha2.MeshConfigSpec{
33+
Sidecar: configv1alpha2.SidecarSpec{
34+
LocalProxyMode: configv1alpha2.LocalProxyModeLocalhost,
35+
},
36+
},
37+
}).Times(1)
38+
privileged := privilegedFalse
39+
actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, nil, privileged, corev1.PullAlways, nil)
40+
41+
expected := corev1.Container{
42+
Name: "-container-name-",
43+
Image: "-init-container-image-",
44+
ImagePullPolicy: corev1.PullAlways,
45+
Command: []string{"/bin/sh"},
46+
Args: []string{
47+
"-c",
48+
`iptables-restore --noflush <<EOF
49+
# OSM sidecar interception rules
50+
*nat
51+
:OSM_PROXY_INBOUND - [0:0]
52+
:OSM_PROXY_IN_REDIRECT - [0:0]
53+
:OSM_PROXY_OUTBOUND - [0:0]
54+
:OSM_PROXY_OUT_REDIRECT - [0:0]
55+
-A OSM_PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003
56+
-A PREROUTING -p tcp -j OSM_PROXY_INBOUND
57+
-A OSM_PROXY_INBOUND -p tcp --dport 15010 -j RETURN
58+
-A OSM_PROXY_INBOUND -p tcp --dport 15901 -j RETURN
59+
-A OSM_PROXY_INBOUND -p tcp --dport 15902 -j RETURN
60+
-A OSM_PROXY_INBOUND -p tcp --dport 15903 -j RETURN
61+
-A OSM_PROXY_INBOUND -p tcp --dport 15904 -j RETURN
62+
-A OSM_PROXY_INBOUND -p tcp -j OSM_PROXY_IN_REDIRECT
63+
-A OSM_PROXY_OUT_REDIRECT -p tcp -j REDIRECT --to-port 15001
64+
-A OSM_PROXY_OUT_REDIRECT -p tcp --dport 15000 -j ACCEPT
65+
-A OUTPUT -p tcp -j OSM_PROXY_OUTBOUND
66+
-A OSM_PROXY_OUTBOUND -o lo ! -d 127.0.0.1/32 -m owner --uid-owner 1500 -j OSM_PROXY_IN_REDIRECT
67+
-A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner 1500 -j RETURN
68+
-A OSM_PROXY_OUTBOUND -m owner --uid-owner 1500 -j RETURN
69+
-A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN
70+
-A OSM_PROXY_OUTBOUND -j OSM_PROXY_OUT_REDIRECT
71+
COMMIT
72+
EOF
73+
`,
74+
},
75+
WorkingDir: "",
76+
Resources: corev1.ResourceRequirements{},
77+
SecurityContext: &corev1.SecurityContext{
78+
Capabilities: &corev1.Capabilities{
79+
Add: []corev1.Capability{
80+
"NET_ADMIN",
81+
},
82+
},
83+
Privileged: &privilegedFalse,
84+
RunAsNonRoot: &runAsNonRootFalse,
85+
RunAsUser: &runAsUserID,
86+
},
87+
Env: []corev1.EnvVar{
88+
{
89+
Name: "POD_IP",
90+
ValueFrom: &corev1.EnvVarSource{
91+
FieldRef: &corev1.ObjectFieldSelector{
92+
APIVersion: "v1",
93+
FieldPath: "status.podIP",
94+
},
95+
},
96+
},
97+
},
98+
Stdin: false,
99+
StdinOnce: false,
100+
TTY: false,
101+
}
102+
103+
Expect(actual).To(Equal(expected))
104+
})
105+
It("Sets podIP DNAT rule if set in meshconfig", func() {
106+
mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1)
107+
mockConfigurator.EXPECT().GetMeshConfig().Return(configv1alpha2.MeshConfig{
108+
Spec: configv1alpha2.MeshConfigSpec{
109+
Sidecar: configv1alpha2.SidecarSpec{
110+
LocalProxyMode: configv1alpha2.LocalProxyModePodIP,
111+
},
112+
},
113+
}).Times(1)
29114
privileged := privilegedFalse
30115
actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, nil, privileged, corev1.PullAlways, nil)
31116

@@ -58,6 +143,7 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func()
58143
-A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner 1500 -j RETURN
59144
-A OSM_PROXY_OUTBOUND -m owner --uid-owner 1500 -j RETURN
60145
-A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN
146+
-I OUTPUT -p tcp -o lo -d 127.0.0.1/32 -m owner --uid-owner 1500 -j DNAT --to-destination $POD_IP
61147
-A OSM_PROXY_OUTBOUND -j OSM_PROXY_OUT_REDIRECT
62148
COMMIT
63149
EOF
@@ -75,6 +161,17 @@ EOF
75161
RunAsNonRoot: &runAsNonRootFalse,
76162
RunAsUser: &runAsUserID,
77163
},
164+
Env: []corev1.EnvVar{
165+
{
166+
Name: "POD_IP",
167+
ValueFrom: &corev1.EnvVarSource{
168+
FieldRef: &corev1.ObjectFieldSelector{
169+
APIVersion: "v1",
170+
FieldPath: "status.podIP",
171+
},
172+
},
173+
},
174+
},
78175
Stdin: false,
79176
StdinOnce: false,
80177
TTY: false,

‎pkg/injector/iptables.go

+10-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"strconv"
66
"strings"
77

8+
configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
9+
810
"github.com/openservicemesh/osm/pkg/constants"
911
)
1012

@@ -60,7 +62,7 @@ var iptablesInboundStaticRules = []string{
6062
}
6163

6264
// generateIptablesCommands generates a list of iptables commands to set up sidecar interception and redirection
63-
func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundIPRangeInclusionList []string, outboundPortExclusionList []int, inboundPortExclusionList []int, networkInterfaceExclusionList []string) string {
65+
func generateIptablesCommands(proxyMode configv1alpha2.LocalProxyMode, outboundIPRangeExclusionList []string, outboundIPRangeInclusionList []string, outboundPortExclusionList []int, inboundPortExclusionList []int, networkInterfaceExclusionList []string) string {
6466
var rules strings.Builder
6567

6668
fmt.Fprintln(&rules, `# OSM sidecar interception rules
@@ -95,6 +97,13 @@ func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundIPR
9597
// 3. Create outbound rules
9698
cmds = append(cmds, iptablesOutboundStaticRules...)
9799

100+
if proxyMode == configv1alpha2.LocalProxyModePodIP {
101+
// For envoy -> local service container proxying, send traffic to pod IP instead of localhost
102+
// *Note: it is important to use the insert option '-I' instead of the append option '-A' to ensure the
103+
// DNAT to the pod ip for envoy -> localhost traffic happens before the rule that redirects traffic to the proxy
104+
cmds = append(cmds, fmt.Sprintf("-I OUTPUT -p tcp -o lo -d 127.0.0.1/32 -m owner --uid-owner %d -j DNAT --to-destination $POD_IP", constants.EnvoyUID))
105+
}
106+
98107
// Ignore outbound traffic in specified interfaces
99108
for _, iface := range networkInterfaceExclusionList {
100109
cmds = append(cmds, fmt.Sprintf("-A OSM_PROXY_OUTBOUND -o %s -j RETURN", iface))

‎pkg/injector/iptables_test.go

+36-1
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,14 @@ import (
44
"testing"
55

66
"github.com/stretchr/testify/assert"
7+
8+
configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
79
)
810

911
func TestGenerateIptablesCommands(t *testing.T) {
1012
testCases := []struct {
1113
name string
14+
proxyMode configv1alpha2.LocalProxyMode
1215
outboundIPRangeExclusions []string
1316
outboundIPRangeInclusions []string
1417
outboundPortExclusions []int
@@ -87,14 +90,46 @@ EOF
8790
-A OSM_PROXY_OUTBOUND -j RETURN
8891
COMMIT
8992
EOF
93+
`,
94+
},
95+
{
96+
name: "proxy mode pod ip",
97+
proxyMode: configv1alpha2.LocalProxyModePodIP,
98+
expected: `iptables-restore --noflush <<EOF
99+
# OSM sidecar interception rules
100+
*nat
101+
:OSM_PROXY_INBOUND - [0:0]
102+
:OSM_PROXY_IN_REDIRECT - [0:0]
103+
:OSM_PROXY_OUTBOUND - [0:0]
104+
:OSM_PROXY_OUT_REDIRECT - [0:0]
105+
-A OSM_PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003
106+
-A PREROUTING -p tcp -j OSM_PROXY_INBOUND
107+
-A OSM_PROXY_INBOUND -p tcp --dport 15010 -j RETURN
108+
-A OSM_PROXY_INBOUND -p tcp --dport 15901 -j RETURN
109+
-A OSM_PROXY_INBOUND -p tcp --dport 15902 -j RETURN
110+
-A OSM_PROXY_INBOUND -p tcp --dport 15903 -j RETURN
111+
-A OSM_PROXY_INBOUND -p tcp --dport 15904 -j RETURN
112+
-A OSM_PROXY_INBOUND -p tcp -j OSM_PROXY_IN_REDIRECT
113+
-A OSM_PROXY_OUT_REDIRECT -p tcp -j REDIRECT --to-port 15001
114+
-A OSM_PROXY_OUT_REDIRECT -p tcp --dport 15000 -j ACCEPT
115+
-A OUTPUT -p tcp -j OSM_PROXY_OUTBOUND
116+
-A OSM_PROXY_OUTBOUND -o lo ! -d 127.0.0.1/32 -m owner --uid-owner 1500 -j OSM_PROXY_IN_REDIRECT
117+
-A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner 1500 -j RETURN
118+
-A OSM_PROXY_OUTBOUND -m owner --uid-owner 1500 -j RETURN
119+
-A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN
120+
-I OUTPUT -p tcp -o lo -d 127.0.0.1/32 -m owner --uid-owner 1500 -j DNAT --to-destination $POD_IP
121+
-A OSM_PROXY_OUTBOUND -j OSM_PROXY_OUT_REDIRECT
122+
COMMIT
123+
EOF
90124
`,
91125
},
92126
}
93127

94128
for _, tc := range testCases {
95129
t.Run(tc.name, func(t *testing.T) {
96130
a := assert.New(t)
97-
actual := generateIptablesCommands(tc.outboundIPRangeExclusions, tc.outboundIPRangeInclusions, tc.outboundPortExclusions, tc.inboundPortExclusions, tc.networkInterfaceExclusions)
131+
132+
actual := generateIptablesCommands(tc.proxyMode, tc.outboundIPRangeExclusions, tc.outboundIPRangeInclusions, tc.outboundPortExclusions, tc.inboundPortExclusions, tc.networkInterfaceExclusions)
98133
a.Equal(tc.expected, actual)
99134
})
100135
}

‎tests/e2e/e2e_pod_client_server_test.go ‎tests/e2e/e2e_client_server_connectivity_test.go

+14-6
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,11 @@ import (
88

99
. "github.com/onsi/ginkgo"
1010
. "github.com/onsi/gomega"
11-
1211
v1 "k8s.io/api/core/v1"
1312
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1413

14+
"github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
15+
1516
"github.com/openservicemesh/osm/pkg/constants"
1617
. "github.com/openservicemesh/osm/tests/framework"
1718
)
@@ -25,26 +26,33 @@ var _ = OSMDescribe("Test HTTP traffic from 1 pod client -> 1 pod server",
2526
func() {
2627
Context("Test traffic flowing from client to server with a Kubernetes Service for the Source: HTTP", func() {
2728
withSourceKubernetesService := true
28-
testTraffic(withSourceKubernetesService)
29+
testTraffic(withSourceKubernetesService, PodCommandDefault)
2930
})
3031

3132
Context("Test traffic flowing from client to server without a Kubernetes Service for the Source: HTTP", func() {
3233
// Prior iterations of OSM required that a source pod belong to a Kubernetes service
3334
// for the Envoy proxy to be configured for outbound traffic to some remote server.
3435
// This test ensures we test this scenario: client Pod is not associated w/ a service.
3536
withSourceKubernetesService := false
36-
testTraffic(withSourceKubernetesService)
37+
testTraffic(withSourceKubernetesService, PodCommandDefault)
38+
})
39+
40+
Context("Test traffic flowing from client to a server with a podIP bind", func() {
41+
// Prior iterations of OSM didn't allow mesh services to bind to the podIP
42+
// This test ensures that that behavior is configurable via MeshConfig
43+
withSourceKubernetesService := true
44+
testTraffic(withSourceKubernetesService, []string{"gunicorn", "-b", "$(POD_IP):80", "httpbin:app", "-k", "gevent"}, WithLocalProxyMode(v1alpha2.LocalProxyModePodIP))
3745
})
3846
})
3947

40-
func testTraffic(withSourceKubernetesService bool) {
48+
func testTraffic(withSourceKubernetesService bool, destPodCommand []string, installOpts ...InstallOsmOpt) {
4149
const sourceName = "client"
4250
const destName = "server"
4351
var ns = []string{sourceName, destName}
4452

4553
It("Tests HTTP traffic for client pod -> server pod", func() {
4654
// Install OSM
47-
Expect(Td.InstallOSM(Td.GetOSMInstallOpts())).To(Succeed())
55+
Expect(Td.InstallOSM(Td.GetOSMInstallOpts(installOpts...))).To(Succeed())
4856

4957
// Create Test NS
5058
for _, n := range ns {
@@ -53,7 +61,7 @@ func testTraffic(withSourceKubernetesService bool) {
5361
}
5462

5563
// Get simple pod definitions for the HTTP server
56-
svcAccDef, podDef, svcDef, err := Td.GetOSSpecificHTTPBinPod(destName, destName)
64+
svcAccDef, podDef, svcDef, err := Td.GetOSSpecificHTTPBinPod(destName, destName, destPodCommand...)
5765
Expect(err).NotTo(HaveOccurred())
5866

5967
_, err = Td.CreateServiceAccount(destName, &svcAccDef)

‎tests/framework/common.go

+21-3
Original file line numberDiff line numberDiff line change
@@ -328,13 +328,21 @@ nodeRegistration:
328328
return nil
329329
}
330330

331+
// WithLocalProxyMode sets the LocalProxyMode for OSM
332+
func WithLocalProxyMode(mode configv1alpha2.LocalProxyMode) InstallOsmOpt {
333+
return func(opts *InstallOSMOpts) {
334+
opts.LocalProxyMode = mode
335+
}
336+
}
337+
331338
// GetOSMInstallOpts initializes install options for OSM
332-
func (td *OsmTestData) GetOSMInstallOpts() InstallOSMOpts {
339+
func (td *OsmTestData) GetOSMInstallOpts(options ...InstallOsmOpt) InstallOSMOpts {
333340
enablePrivilegedInitContainer := false
334341
if td.DeployOnOpenShift {
335342
enablePrivilegedInitContainer = true
336343
}
337-
return InstallOSMOpts{
344+
345+
baseOpts := InstallOSMOpts{
338346
ControlPlaneNS: td.OsmNamespace,
339347
CertManager: defaultCertManager,
340348
ContainerRegistryLoc: td.CtrRegistryServer,
@@ -364,6 +372,12 @@ func (td *OsmTestData) GetOSMInstallOpts() InstallOSMOpts {
364372
EnablePrivilegedInitContainer: enablePrivilegedInitContainer,
365373
EnableIngressBackendPolicy: true,
366374
}
375+
376+
for _, opt := range options {
377+
opt(&baseOpts)
378+
}
379+
380+
return baseOpts
367381
}
368382

369383
// LoadImagesToKind loads the list of images to the node for Kind clusters
@@ -415,7 +429,7 @@ func (td *OsmTestData) LoadImagesToKind(imageNames []string) error {
415429
return nil
416430
}
417431

418-
func setMeshConfigToDefault(instOpts InstallOSMOpts, meshConfig *configv1alpha2.MeshConfig) (defaultConfig *configv1alpha2.MeshConfig) {
432+
func setMeshConfigToDefault(instOpts InstallOSMOpts, meshConfig *configv1alpha2.MeshConfig) *configv1alpha2.MeshConfig {
419433
meshConfig.Spec.Traffic.EnableEgress = instOpts.EgressEnabled
420434
meshConfig.Spec.Traffic.EnablePermissiveTrafficPolicyMode = instOpts.EnablePermissiveMode
421435
meshConfig.Spec.Traffic.OutboundPortExclusionList = []int{}
@@ -491,6 +505,10 @@ func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error {
491505
fmt.Sprintf("osm.enableReconciler=%v", instOpts.EnableReconciler),
492506
)
493507

508+
if instOpts.LocalProxyMode != "" {
509+
instOpts.SetOverrides = append(instOpts.SetOverrides, fmt.Sprintf("osm.localProxyMode=%s", instOpts.LocalProxyMode))
510+
}
511+
494512
switch instOpts.CertManager {
495513
case "vault":
496514
if err := td.installVault(instOpts); err != nil {

0 commit comments

Comments
 (0)