Skip to content

Commit 58092b9

Browse files
cPu1cPu1
committed
Attach Auto Mode policies to cluster role when enabling Auto Mode
Signed-off-by: cpu1 <[email protected]>
1 parent 7351e74 commit 58092b9

35 files changed

+711
-494
lines changed

‎.mockery.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -108,3 +108,4 @@ packages:
108108
interfaces:
109109
RoleManager: {}
110110
NodeGroupDrainer: {}
111+
ClusterRoleManager: {}

‎examples/42-auto-mode.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# A sample ClusterConfig file that creates a cluster with Auto Mode enabled.
2+
apiVersion: eksctl.io/v1alpha5
3+
kind: ClusterConfig
4+
5+
metadata:
6+
name: auto-mode-cluster
7+
region: us-west-2
8+
9+
autoModeConfig:
10+
# defaults to false
11+
enabled: true
12+
# optional, defaults to [general-purpose, system].
13+
# To disable creation of nodePools, set it to the empty array ([]).
14+
nodePools: [general-purpose, system]
15+
# optional, eksctl creates a new role if this is not supplied
16+
# and nodePools are present.
17+
# nodeRoleARN: ""

‎go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -772,7 +772,9 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2K
772772
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
773773
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
774774
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
775+
github.com/aws/smithy-go v1.4.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
775776
github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
777+
github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
776778
github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
777779
github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
778780
github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240508073157-fbfa1bc129f5 h1:F80UWAvCDH3PgWIkMhwhKN7FRlkn9MhI+nBHFq739ZM=

‎integration/tests/auto_mode/auto_mode_test.go

+5-7
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,17 @@ package auto_mode
66

77
import (
88
"context"
9-
"fmt"
109
"testing"
1110
"time"
1211

13-
"github.com/aws/aws-sdk-go-v2/aws"
14-
awseks "github.com/aws/aws-sdk-go-v2/service/eks"
15-
ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types"
1612
. "github.com/onsi/ginkgo/v2"
1713
. "github.com/onsi/gomega"
1814
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1915

16+
"github.com/aws/aws-sdk-go-v2/aws"
17+
awseks "github.com/aws/aws-sdk-go-v2/service/eks"
18+
ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types"
19+
2020
. "github.com/weaveworks/eksctl/integration/runner"
2121
"github.com/weaveworks/eksctl/integration/tests"
2222
clusterutils "github.com/weaveworks/eksctl/integration/utilities/cluster"
@@ -93,9 +93,7 @@ var _ = Describe("Auto Mode", Ordered, func() {
9393
test, err := kube.NewTest(params.KubeconfigPath)
9494
Expect(err).NotTo(HaveOccurred())
9595
d := test.CreateDeploymentFromFile(test.Namespace, "../../data/podinfo.yaml")
96-
start := time.Now()
9796
test.WaitForDeploymentReady(d, 30*time.Minute)
98-
fmt.Println("dep ready after", time.Since(start))
9997
deployment, err := test.GetDeployment(test.Namespace, "podinfo")
10098
Expect(err).NotTo(HaveOccurred())
10199
nodeList := test.ListNodes(metav1.ListOptions{})
@@ -104,7 +102,7 @@ var _ = Describe("Auto Mode", Ordered, func() {
104102
const labelName = "eks.amazonaws.com/compute-type"
105103
computeType, ok := node.Labels[labelName]
106104
Expect(ok).To(BeTrue(), "expected to find label %s on node %s", labelName, node.Name)
107-
Expect(computeType).To(Equal("eks-managed"))
105+
Expect(computeType).To(Equal("auto"))
108106
podList := test.ListPodsFromDeployment(deployment)
109107
Expect(podList.Items).To(HaveLen(2))
110108
for _, pod := range podList.Items {

‎pkg/actions/automode/auto_mode_updater.go

+25-20
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,15 @@ import (
77
"slices"
88
"time"
99

10-
"github.com/aws/aws-sdk-go-v2/aws"
11-
"github.com/aws/aws-sdk-go-v2/service/eks"
12-
ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types"
1310
"github.com/kris-nova/logger"
1411
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1512
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
1613

14+
"github.com/aws/aws-sdk-go-v2/aws"
15+
"github.com/aws/aws-sdk-go-v2/service/eks"
16+
ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types"
17+
1718
api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
18-
"github.com/weaveworks/eksctl/pkg/automode"
1919
"github.com/weaveworks/eksctl/pkg/eks/waiter"
2020
)
2121

@@ -41,13 +41,19 @@ type RoleManager interface {
4141
DeleteIfRequired(ctx context.Context) error
4242
}
4343

44+
// A ClusterRoleManager manages the cluster role.
45+
type ClusterRoleManager interface {
46+
UpdateRoleForAutoMode(ctx context.Context) error
47+
DeleteAutoModePolicies(ctx context.Context) error
48+
}
49+
4450
// An Updater enables or disables Auto Mode.
4551
type Updater struct {
46-
RoleManager RoleManager
47-
CoreV1Interface corev1client.CoreV1Interface
48-
EKSUpdater EKSUpdater
49-
Drainer NodeGroupDrainer
50-
RBACApplier *automode.RBACApplier
52+
RoleManager RoleManager
53+
ClusterRoleManager ClusterRoleManager
54+
PodsGetter corev1client.PodsGetter
55+
EKSUpdater EKSUpdater
56+
Drainer NodeGroupDrainer
5157
}
5258

5359
// Update updates the cluster to match the autoModeConfig settings supplied in clusterConfig.
@@ -57,8 +63,8 @@ func (u *Updater) Update(ctx context.Context, clusterConfig *api.ClusterConfig,
5763
return cc != nil && *cc.Enabled
5864
}
5965
if clusterConfig.IsAutoModeEnabled() {
66+
amc := clusterConfig.AutoModeConfig
6067
if autoModeEnabled() {
61-
amc := clusterConfig.AutoModeConfig
6268
if !amc.NodeRoleARN.IsZero() && currentCluster.ComputeConfig.NodeRoleArn != nil &&
6369
*currentCluster.ComputeConfig.NodeRoleArn != amc.NodeRoleARN.String() {
6470
return errors.New("autoModeConfig.nodeRoleARN cannot be modified")
@@ -73,17 +79,13 @@ func (u *Updater) Update(ctx context.Context, clusterConfig *api.ClusterConfig,
7379
} else {
7480
logger.Info("enabling Auto Mode")
7581
}
76-
if err := u.enableAutoMode(ctx, clusterConfig.AutoModeConfig, currentCluster.ComputeConfig, clusterConfig.Metadata.Name); err != nil {
82+
if err := u.enableAutoMode(ctx, amc, currentCluster.ComputeConfig, clusterConfig.Metadata.Name); err != nil {
7783
return fmt.Errorf("enabling Auto Mode: %w", err)
7884
}
79-
if clusterConfig.AutoModeConfig.HasNodePools() {
85+
if amc.HasNodePools() {
8086
logger.Info("cluster subnets will be used for nodes launched by Auto Mode; please create a new NodeClass " +
8187
"resource if you do not want to use cluster subnets")
8288
}
83-
logger.Info("applying node RBAC resources for Auto Mode")
84-
if err := u.RBACApplier.ApplyRBACResources(); err != nil {
85-
return err
86-
}
8789
logger.Info("Auto Mode enabled successfully")
8890
return nil
8991
}
@@ -94,9 +96,6 @@ func (u *Updater) Update(ctx context.Context, clusterConfig *api.ClusterConfig,
9496
if err := u.disableAutoMode(ctx, clusterConfig.Metadata.Name); err != nil {
9597
return fmt.Errorf("disabling Auto Mode: %w", err)
9698
}
97-
if err := u.RBACApplier.DeleteRBACResources(); err != nil {
98-
return err
99-
}
10099
logger.Info("Auto Mode disabled successfully")
101100
return nil
102101
}
@@ -105,6 +104,9 @@ func (u *Updater) enableAutoMode(ctx context.Context, autoModeConfig *api.AutoMo
105104
if err := u.preflightCheck(ctx); err != nil {
106105
return err
107106
}
107+
if err := u.ClusterRoleManager.UpdateRoleForAutoMode(ctx); err != nil {
108+
return fmt.Errorf("updating cluster role to use Auto Mode: %w", err)
109+
}
108110
computeConfigReq := &ekstypes.ComputeConfigRequest{
109111
Enabled: aws.Bool(true),
110112
NodePools: *autoModeConfig.NodePools,
@@ -177,6 +179,9 @@ func (u *Updater) disableAutoMode(ctx context.Context, clusterName string) error
177179
if err := u.RoleManager.DeleteIfRequired(ctx); err != nil {
178180
return fmt.Errorf("deleting IAM resources for Auto Mode: %w", err)
179181
}
182+
if err := u.ClusterRoleManager.DeleteAutoModePolicies(ctx); err != nil {
183+
return fmt.Errorf("deleting Auto Mode policies from cluster role: %w", err)
184+
}
180185
return nil
181186
}
182187

@@ -204,7 +209,7 @@ func (u *Updater) preflightCheck(ctx context.Context) error {
204209
}
205210
knownKarpenterNamespaces := []string{metav1.NamespaceSystem, "karpenter"}
206211
for _, ns := range knownKarpenterNamespaces {
207-
podList, err := u.CoreV1Interface.Pods(ns).List(ctx, metav1.ListOptions{
212+
podList, err := u.PodsGetter.Pods(ns).List(ctx, metav1.ListOptions{
208213
LabelSelector: "app.kubernetes.io/instance=karpenter",
209214
})
210215
if err != nil {

‎pkg/actions/automode/auto_mode_updater_test.go

+17-20
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,20 @@ package automode_test
33
import (
44
"context"
55

6-
"github.com/aws/aws-sdk-go-v2/aws"
7-
awseks "github.com/aws/aws-sdk-go-v2/service/eks"
8-
ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types"
96
. "github.com/onsi/ginkgo/v2"
107
. "github.com/onsi/gomega"
118
"github.com/stretchr/testify/mock"
129
corev1 "k8s.io/api/core/v1"
1310
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1411
kubernetesfake "k8s.io/client-go/kubernetes/fake"
1512

13+
"github.com/aws/aws-sdk-go-v2/aws"
14+
awseks "github.com/aws/aws-sdk-go-v2/service/eks"
15+
ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types"
16+
1617
automodeactions "github.com/weaveworks/eksctl/pkg/actions/automode"
1718
"github.com/weaveworks/eksctl/pkg/actions/automode/mocks"
1819
api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
19-
"github.com/weaveworks/eksctl/pkg/automode"
20-
automodemocks "github.com/weaveworks/eksctl/pkg/automode/mocks"
2120
"github.com/weaveworks/eksctl/pkg/eks/mocksv2"
2221
)
2322

@@ -32,11 +31,11 @@ type updaterTest struct {
3231
}
3332

3433
type updaterMocks struct {
35-
roleManager mocks.RoleManager
36-
drainer mocks.NodeGroupDrainer
37-
eksUpdater mocksv2.EKS
38-
rawClient automodemocks.RawClient
39-
clientSet *kubernetesfake.Clientset
34+
roleManager mocks.RoleManager
35+
clusterRoleManager mocks.ClusterRoleManager
36+
drainer mocks.NodeGroupDrainer
37+
eksUpdater mocksv2.EKS
38+
clientSet *kubernetesfake.Clientset
4039
}
4140

4241
var _ = DescribeTable("Auto Mode Updater", func(t updaterTest) {
@@ -50,12 +49,10 @@ var _ = DescribeTable("Auto Mode Updater", func(t updaterTest) {
5049
clusterConfig.AutoModeConfig = t.autoModeConfig
5150
clusterConfig.VPC = t.vpc
5251
updater := &automodeactions.Updater{
53-
RoleManager: &um.roleManager,
54-
CoreV1Interface: um.clientSet.CoreV1(),
55-
EKSUpdater: &um.eksUpdater,
56-
RBACApplier: &automode.RBACApplier{
57-
RawClient: &um.rawClient,
58-
},
52+
RoleManager: &um.roleManager,
53+
ClusterRoleManager: &um.clusterRoleManager,
54+
PodsGetter: um.clientSet.CoreV1(),
55+
EKSUpdater: &um.eksUpdater,
5956
}
6057
if t.drainNodes {
6158
updater.Drainer = &um.drainer
@@ -70,9 +67,9 @@ var _ = DescribeTable("Auto Mode Updater", func(t updaterTest) {
7067
AssertExpectations(t mock.TestingT) bool
7168
}{
7269
&um.roleManager,
70+
&um.clusterRoleManager,
7371
&um.eksUpdater,
7472
&um.drainer,
75-
&um.rawClient,
7673
} {
7774
asserter.AssertExpectations(GinkgoT())
7875
}
@@ -151,12 +148,12 @@ var _ = DescribeTable("Auto Mode Updater", func(t updaterTest) {
151148
},
152149
},
153150
updateMocks: func(u *updaterMocks) {
151+
u.clusterRoleManager.EXPECT().UpdateRoleForAutoMode(mock.Anything).Return(nil).Once()
154152
mockUpdateClusterConfig(u, &ekstypes.ComputeConfigRequest{
155153
Enabled: aws.Bool(true),
156154
NodePools: []string{api.AutoModeNodePoolGeneralPurpose},
157155
NodeRoleArn: aws.String("arn:aws:iam::000:role/CustomNodeRole"),
158156
})
159-
u.rawClient.EXPECT().CreateOrReplace(mock.Anything, false).Return(nil)
160157
},
161158
}),
162159
Entry("disabling Auto Mode", updaterTest{
@@ -173,7 +170,7 @@ var _ = DescribeTable("Auto Mode Updater", func(t updaterTest) {
173170
Enabled: aws.Bool(false),
174171
})
175172
u.roleManager.EXPECT().DeleteIfRequired(mock.Anything).Return(nil).Once()
176-
u.rawClient.EXPECT().Delete(mock.Anything).Return(nil)
173+
u.clusterRoleManager.EXPECT().DeleteAutoModePolicies(mock.Anything).Return(nil).Once()
177174
},
178175
}),
179176
Entry("Karpenter pods exist in the cluster when enabling Auto Mode", updaterTest{
@@ -221,12 +218,12 @@ var _ = DescribeTable("Auto Mode Updater", func(t updaterTest) {
221218
)
222219

223220
func mockEnableAutoMode(u *updaterMocks, nodeRoleARN string, nodePools []string) {
221+
u.clusterRoleManager.EXPECT().UpdateRoleForAutoMode(mock.Anything).Return(nil).Once()
224222
mockUpdateClusterConfig(u, &ekstypes.ComputeConfigRequest{
225223
Enabled: aws.Bool(true),
226224
NodePools: nodePools,
227225
NodeRoleArn: aws.String(nodeRoleARN),
228226
})
229-
u.rawClient.EXPECT().CreateOrReplace(mock.Anything, false).Return(nil)
230227
}
231228

232229
func mockUpdateClusterConfig(u *updaterMocks, computeConfig *ekstypes.ComputeConfigRequest) {

0 commit comments

Comments
 (0)