Skip to content

Commit 4f42b6f

Browse files
jchen6585Joseph Chen
jchen6585
and
Joseph Chen
authored
Fix coredns failing during custom networking tests (#2844)
Co-authored-by: Joseph Chen <[email protected]>
1 parent f2c3f73 commit 4f42b6f

File tree

6 files changed

+134
-49
lines changed

6 files changed

+134
-49
lines changed

‎test/framework/resources/aws/services/ec2.go

+64-36
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@ type EC2 interface {
2828
DescribeInstance(instanceID string) (*ec2.Instance, error)
2929
DescribeVPC(vpcID string) (*ec2.DescribeVpcsOutput, error)
3030
DescribeNetworkInterface(interfaceIDs []string) (*ec2.DescribeNetworkInterfacesOutput, error)
31-
AuthorizeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error
32-
RevokeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error
31+
AuthorizeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string, sourceSG bool) error
32+
RevokeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string, sourceSG bool) error
3333
AuthorizeSecurityGroupEgress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error
3434
RevokeSecurityGroupEgress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error
3535
AssociateVPCCIDRBlock(vpcId string, cidrBlock string) (*ec2.AssociateVpcCidrBlockOutput, error)
@@ -96,30 +96,44 @@ func (d *defaultEC2) DescribeInstance(instanceID string) (*ec2.Instance, error)
9696
return describeInstanceOutput.Reservations[0].Instances[0], nil
9797
}
9898

99-
func (d *defaultEC2) AuthorizeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error {
99+
func (d *defaultEC2) AuthorizeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string, sourceSG bool) error {
100100
var ipv4Ranges []*ec2.IpRange
101101
var ipv6Ranges []*ec2.Ipv6Range
102-
if strings.Contains(cidrIP, ":") {
103-
ipv6Ranges = []*ec2.Ipv6Range{
104-
{
105-
CidrIpv6: aws.String(cidrIP),
106-
},
102+
var ipPermissions *ec2.IpPermission
103+
if !sourceSG {
104+
if strings.Contains(cidrIP, ":") {
105+
ipv6Ranges = []*ec2.Ipv6Range{
106+
{
107+
CidrIpv6: aws.String(cidrIP),
108+
},
109+
}
110+
} else {
111+
ipv4Ranges = []*ec2.IpRange{
112+
{
113+
CidrIp: aws.String(cidrIP),
114+
},
115+
}
116+
}
117+
118+
ipPermissions = &ec2.IpPermission{
119+
FromPort: aws.Int64(int64(fromPort)),
120+
ToPort: aws.Int64(int64(toPort)),
121+
IpProtocol: aws.String(protocol),
122+
IpRanges: ipv4Ranges,
123+
Ipv6Ranges: ipv6Ranges,
107124
}
108125
} else {
109-
ipv4Ranges = []*ec2.IpRange{
110-
{
111-
CidrIp: aws.String(cidrIP),
126+
ipPermissions = &ec2.IpPermission{
127+
FromPort: aws.Int64(int64(fromPort)),
128+
ToPort: aws.Int64(int64(toPort)),
129+
IpProtocol: aws.String(protocol),
130+
UserIdGroupPairs: []*ec2.UserIdGroupPair{
131+
{
132+
GroupId: aws.String(cidrIP),
133+
},
112134
},
113135
}
114136
}
115-
116-
ipPermissions := &ec2.IpPermission{
117-
FromPort: aws.Int64(int64(fromPort)),
118-
ToPort: aws.Int64(int64(toPort)),
119-
IpProtocol: aws.String(protocol),
120-
IpRanges: ipv4Ranges,
121-
Ipv6Ranges: ipv6Ranges,
122-
}
123137
authorizeSecurityGroupIngressInput := &ec2.AuthorizeSecurityGroupIngressInput{
124138
GroupId: aws.String(groupID),
125139
IpPermissions: []*ec2.IpPermission{ipPermissions},
@@ -128,30 +142,44 @@ func (d *defaultEC2) AuthorizeSecurityGroupIngress(groupID string, protocol stri
128142
return err
129143
}
130144

131-
func (d *defaultEC2) RevokeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error {
145+
func (d *defaultEC2) RevokeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string, sourceSG bool) error {
132146
var ipv4Ranges []*ec2.IpRange
133147
var ipv6Ranges []*ec2.Ipv6Range
134-
if strings.Contains(cidrIP, ":") {
135-
ipv6Ranges = []*ec2.Ipv6Range{
136-
{
137-
CidrIpv6: aws.String(cidrIP),
138-
},
148+
var ipPermissions *ec2.IpPermission
149+
if !sourceSG {
150+
if strings.Contains(cidrIP, ":") {
151+
ipv6Ranges = []*ec2.Ipv6Range{
152+
{
153+
CidrIpv6: aws.String(cidrIP),
154+
},
155+
}
156+
} else {
157+
ipv4Ranges = []*ec2.IpRange{
158+
{
159+
CidrIp: aws.String(cidrIP),
160+
},
161+
}
162+
}
163+
164+
ipPermissions = &ec2.IpPermission{
165+
FromPort: aws.Int64(int64(fromPort)),
166+
ToPort: aws.Int64(int64(toPort)),
167+
IpProtocol: aws.String(protocol),
168+
IpRanges: ipv4Ranges,
169+
Ipv6Ranges: ipv6Ranges,
139170
}
140171
} else {
141-
ipv4Ranges = []*ec2.IpRange{
142-
{
143-
CidrIp: aws.String(cidrIP),
172+
ipPermissions = &ec2.IpPermission{
173+
FromPort: aws.Int64(int64(fromPort)),
174+
ToPort: aws.Int64(int64(toPort)),
175+
IpProtocol: aws.String(protocol),
176+
UserIdGroupPairs: []*ec2.UserIdGroupPair{
177+
{
178+
GroupId: aws.String(cidrIP),
179+
},
144180
},
145181
}
146182
}
147-
148-
ipPermissions := &ec2.IpPermission{
149-
FromPort: aws.Int64(int64(fromPort)),
150-
ToPort: aws.Int64(int64(toPort)),
151-
IpProtocol: aws.String(protocol),
152-
IpRanges: ipv4Ranges,
153-
Ipv6Ranges: ipv6Ranges,
154-
}
155183
revokeSecurityGroupIngressInput := &ec2.RevokeSecurityGroupIngressInput{
156184
GroupId: aws.String(groupID),
157185
IpPermissions: []*ec2.IpPermission{ipPermissions},

‎test/integration/cni/pod_traffic_across_az_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ var _ = Describe("[STATIC_CANARY] test pod networking", FlakeAttempts(retries),
101101
JustBeforeEach(func() {
102102
By("authorizing security group ingress on instance security group")
103103
err = f.CloudServices.EC2().
104-
AuthorizeSecurityGroupIngress(instanceSecurityGroupID, protocol, serverPort, serverPort, "0.0.0.0/0")
104+
AuthorizeSecurityGroupIngress(instanceSecurityGroupID, protocol, serverPort, serverPort, "0.0.0.0/0", false)
105105
Expect(err).ToNot(HaveOccurred())
106106

107107
By("authorizing security group egress on instance security group")
@@ -140,7 +140,7 @@ var _ = Describe("[STATIC_CANARY] test pod networking", FlakeAttempts(retries),
140140
JustAfterEach(func() {
141141
By("revoking security group ingress on instance security group")
142142
err = f.CloudServices.EC2().
143-
RevokeSecurityGroupIngress(instanceSecurityGroupID, protocol, serverPort, serverPort, "0.0.0.0/0")
143+
RevokeSecurityGroupIngress(instanceSecurityGroupID, protocol, serverPort, serverPort, "0.0.0.0/0", false)
144144
Expect(err).ToNot(HaveOccurred())
145145

146146
By("revoking security group egress on instance security group")

‎test/integration/cni/pod_traffic_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ var _ = Describe("test pod networking", func() {
7272
JustBeforeEach(func() {
7373
By("authorizing security group ingress on instance security group")
7474
err = f.CloudServices.EC2().
75-
AuthorizeSecurityGroupIngress(instanceSecurityGroupID, protocol, serverPort, serverPort, "0.0.0.0/0")
75+
AuthorizeSecurityGroupIngress(instanceSecurityGroupID, protocol, serverPort, serverPort, "0.0.0.0/0", false)
7676
Expect(err).ToNot(HaveOccurred())
7777

7878
By("authorizing security group egress on instance security group")
@@ -139,7 +139,7 @@ var _ = Describe("test pod networking", func() {
139139
JustAfterEach(func() {
140140
By("revoking security group ingress on instance security group")
141141
err = f.CloudServices.EC2().
142-
RevokeSecurityGroupIngress(instanceSecurityGroupID, protocol, serverPort, serverPort, "0.0.0.0/0")
142+
RevokeSecurityGroupIngress(instanceSecurityGroupID, protocol, serverPort, serverPort, "0.0.0.0/0", false)
143143
Expect(err).ToNot(HaveOccurred())
144144

145145
By("revoking security group egress on instance security group")

‎test/integration/custom-networking/custom_networking_suite_test.go

+59-2
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,17 @@ import (
1919
"net"
2020
"testing"
2121

22+
"github.com/apparentlymart/go-cidr/cidr"
2223
"github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
2324
"github.com/aws/amazon-vpc-cni-k8s/test/framework"
2425
awsUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/utils"
2526
"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
2627
k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
2728
"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
29+
"github.com/aws/amazon-vpc-cni-k8s/test/integration/common"
2830
"github.com/prometheus/client_golang/prometheus"
31+
corev1 "k8s.io/api/core/v1"
2932

30-
"github.com/apparentlymart/go-cidr/cidr"
3133
. "github.com/onsi/ginkgo/v2"
3234
. "github.com/onsi/gomega"
3335
)
@@ -51,6 +53,9 @@ var (
5153
customNetworkingSGID string
5254
customNetworkingSGOpenPort = 8080
5355
customNetworkingSubnetIDList []string
56+
corednsSGOpenPort = 53
57+
primaryENISGID string
58+
primaryENISGList []string
5459
// List of ENIConfig per Availability Zone
5560
eniConfigList []*v1alpha1.ENIConfig
5661
eniConfigBuilderList []*manifest.ENIConfigBuilder
@@ -85,7 +90,53 @@ var _ = BeforeSuite(func() {
8590
f.CloudServices.EC2().AuthorizeSecurityGroupEgress(customNetworkingSGID, "TCP",
8691
customNetworkingSGOpenPort, customNetworkingSGOpenPort, "0.0.0.0/0")
8792
f.CloudServices.EC2().AuthorizeSecurityGroupIngress(customNetworkingSGID, "TCP",
88-
customNetworkingSGOpenPort, customNetworkingSGOpenPort, "0.0.0.0/0")
93+
customNetworkingSGOpenPort, customNetworkingSGOpenPort, "0.0.0.0/0", false)
94+
f.CloudServices.EC2().AuthorizeSecurityGroupEgress(customNetworkingSGID, "UDP",
95+
corednsSGOpenPort, corednsSGOpenPort, "0.0.0.0/0")
96+
f.CloudServices.EC2().AuthorizeSecurityGroupIngress(customNetworkingSGID, "UDP",
97+
corednsSGOpenPort, corednsSGOpenPort, "0.0.0.0/0", false)
98+
f.CloudServices.EC2().AuthorizeSecurityGroupEgress(customNetworkingSGID, "TCP",
99+
corednsSGOpenPort, corednsSGOpenPort, "0.0.0.0/0")
100+
f.CloudServices.EC2().AuthorizeSecurityGroupIngress(customNetworkingSGID, "TCP",
101+
corednsSGOpenPort, corednsSGOpenPort, "0.0.0.0/0", false)
102+
103+
By("Adding custom networking security group ingress rule from primary eni")
104+
nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey,
105+
f.Options.NgNameLabelVal)
106+
Expect(err).ToNot(HaveOccurred())
107+
108+
var primaryNode *corev1.Node
109+
for _, n := range nodeList.Items {
110+
if len(n.Spec.Taints) == 0 {
111+
primaryNode = &n
112+
break
113+
}
114+
}
115+
Expect(primaryNode).To(Not(BeNil()), "expected to find a non-tainted node")
116+
117+
instanceID := k8sUtils.GetInstanceIDFromNode(*primaryNode)
118+
primaryInstance, err := f.CloudServices.EC2().DescribeInstance(instanceID)
119+
Expect(err).ToNot(HaveOccurred())
120+
121+
instance, err := f.CloudServices.EC2().DescribeInstance(*primaryInstance.InstanceId)
122+
Expect(err).ToNot(HaveOccurred())
123+
124+
var primaryENIID string
125+
for _, nwInterface := range instance.NetworkInterfaces {
126+
primaryENI := common.IsPrimaryENI(nwInterface, instance.PrivateIpAddress)
127+
if primaryENI {
128+
primaryENIID = *nwInterface.NetworkInterfaceId
129+
break
130+
}
131+
}
132+
133+
eniOutput, err := f.CloudServices.EC2().DescribeNetworkInterface([]string{primaryENIID})
134+
Expect(err).ToNot(HaveOccurred())
135+
for _, sg := range eniOutput.NetworkInterfaces[0].Groups {
136+
primaryENISGList = append(primaryENISGList, *sg.GroupId)
137+
f.CloudServices.EC2().AuthorizeSecurityGroupIngress(*sg.GroupId, "-1",
138+
-1, -1, customNetworkingSGID, true)
139+
}
89140

90141
By("associating cidr range to the VPC")
91142
association, err := f.CloudServices.EC2().AssociateVPCCIDRBlock(f.Options.AWSVPCID, cidrRange.String())
@@ -160,6 +211,12 @@ var _ = AfterSuite(func() {
160211
By("terminating instances")
161212
errs.Append(awsUtils.TerminateInstances(f))
162213

214+
By("Removing custom networking security group ingress rule from primary eni")
215+
for _, sg := range primaryENISGList {
216+
f.CloudServices.EC2().RevokeSecurityGroupIngress(sg, "-1",
217+
-1, -1, customNetworkingSGID, true)
218+
}
219+
163220
By("deleting security group")
164221
errs.Append(f.CloudServices.EC2().DeleteSecurityGroup(customNetworkingSGID))
165222

‎test/integration/pod-eni/security_group_per_pod_suite_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -86,11 +86,11 @@ var _ = BeforeSuite(func() {
8686
By("authorizing egress and ingress on security group for client-server communication")
8787
if isIPv4Cluster {
8888
f.CloudServices.EC2().AuthorizeSecurityGroupEgress(securityGroupId, "tcp", openPort, openPort, v4Zero)
89-
f.CloudServices.EC2().AuthorizeSecurityGroupIngress(securityGroupId, "tcp", openPort, openPort, v4Zero)
89+
f.CloudServices.EC2().AuthorizeSecurityGroupIngress(securityGroupId, "tcp", openPort, openPort, v4Zero, false)
9090
} else {
9191
f.CloudServices.EC2().AuthorizeSecurityGroupEgress(securityGroupId, "tcp", openPort, openPort, v6Zero)
92-
f.CloudServices.EC2().AuthorizeSecurityGroupIngress(securityGroupId, "tcp", openPort, openPort, v6Zero)
93-
f.CloudServices.EC2().AuthorizeSecurityGroupIngress(securityGroupId, "icmpv6", -1, -1, v6Zero)
92+
f.CloudServices.EC2().AuthorizeSecurityGroupIngress(securityGroupId, "tcp", openPort, openPort, v6Zero, false)
93+
f.CloudServices.EC2().AuthorizeSecurityGroupIngress(securityGroupId, "icmpv6", -1, -1, v6Zero, false)
9494
}
9595

9696
By("getting branch ENI limits")

‎test/integration/pod-eni/security_group_per_pod_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -126,10 +126,10 @@ var _ = Describe("Security Group for Pods Test", func() {
126126
// 8080: metric-pod listener port
127127
By("Adding an additional Ingress Rule on NodeSecurityGroupID to allow client-to-metric traffic")
128128
if isIPv4Cluster {
129-
err := f.CloudServices.EC2().AuthorizeSecurityGroupIngress(clusterSGID, "TCP", metricsPort, metricsPort, v4Zero)
129+
err := f.CloudServices.EC2().AuthorizeSecurityGroupIngress(clusterSGID, "TCP", metricsPort, metricsPort, v4Zero, false)
130130
Expect(err).ToNot(HaveOccurred())
131131
} else {
132-
err := f.CloudServices.EC2().AuthorizeSecurityGroupIngress(clusterSGID, "TCP", metricsPort, metricsPort, v6Zero)
132+
err := f.CloudServices.EC2().AuthorizeSecurityGroupIngress(clusterSGID, "TCP", metricsPort, metricsPort, v6Zero, false)
133133
Expect(err).ToNot(HaveOccurred())
134134
}
135135
})
@@ -160,10 +160,10 @@ var _ = Describe("Security Group for Pods Test", func() {
160160
// Revoke the Ingress rule for traffic from client pods added to Node Security Group
161161
By("Revoking the additional Ingress rule added to allow client-to-metric traffic")
162162
if isIPv4Cluster {
163-
err := f.CloudServices.EC2().RevokeSecurityGroupIngress(clusterSGID, "TCP", metricsPort, metricsPort, v4Zero)
163+
err := f.CloudServices.EC2().RevokeSecurityGroupIngress(clusterSGID, "TCP", metricsPort, metricsPort, v4Zero, false)
164164
Expect(err).ToNot(HaveOccurred())
165165
} else {
166-
err := f.CloudServices.EC2().RevokeSecurityGroupIngress(clusterSGID, "TCP", metricsPort, metricsPort, v6Zero)
166+
err := f.CloudServices.EC2().RevokeSecurityGroupIngress(clusterSGID, "TCP", metricsPort, metricsPort, v6Zero, false)
167167
Expect(err).ToNot(HaveOccurred())
168168
}
169169
})

0 commit comments

Comments
 (0)