Skip to content

refactor: instance profile provider #7971

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 9, 2025
Merged

refactor: instance profile provider #7971

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions pkg/apis/v1/ec2nodeclass.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

"github.com/mitchellh/hashstructure/v2"
"github.com/samber/lo"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
Expand Down Expand Up @@ -491,11 +492,12 @@ func (in *EC2NodeClass) InstanceProfileRole() string {
return in.Spec.Role
}

func (in *EC2NodeClass) InstanceProfileTags(clusterName string) map[string]string {
func (in *EC2NodeClass) InstanceProfileTags(clusterName string, region string) map[string]string {
return lo.Assign(in.Spec.Tags, map[string]string{
fmt.Sprintf("kubernetes.io/cluster/%s", clusterName): "owned",
EKSClusterNameTagKey: clusterName,
LabelNodeClass: in.Name,
EKSClusterNameTagKey: clusterName,
LabelNodeClass: in.Name,
v1.LabelTopologyRegion: region,
})
}

Expand Down
6 changes: 3 additions & 3 deletions pkg/cloudprovider/suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1186,7 +1186,7 @@ var _ = Describe("CloudProvider", func() {
{SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int32(100),
Tags: []ec2types.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}},
}})
controller := nodeclass.NewController(awsEnv.Clock, env.Client, recorder, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider, awsEnv.CapacityReservationProvider, awsEnv.EC2API, awsEnv.ValidationCache, awsEnv.AMIResolver)
controller := nodeclass.NewController(awsEnv.Clock, env.Client, recorder, fake.DefaultRegion, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider, awsEnv.CapacityReservationProvider, awsEnv.EC2API, awsEnv.ValidationCache, awsEnv.AMIResolver)
ExpectApplied(ctx, env.Client, nodePool, nodeClass)
ExpectObjectReconciled(ctx, env.Client, controller, nodeClass)
pod := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{corev1.LabelTopologyZone: "test-zone-1a"}})
Expand All @@ -1203,7 +1203,7 @@ var _ = Describe("CloudProvider", func() {
{SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int32(11),
Tags: []ec2types.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}},
}})
controller := nodeclass.NewController(awsEnv.Clock, env.Client, recorder, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider, awsEnv.CapacityReservationProvider, awsEnv.EC2API, awsEnv.ValidationCache, awsEnv.AMIResolver)
controller := nodeclass.NewController(awsEnv.Clock, env.Client, recorder, fake.DefaultRegion, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider, awsEnv.CapacityReservationProvider, awsEnv.EC2API, awsEnv.ValidationCache, awsEnv.AMIResolver)
nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{
MaxPods: aws.Int32(1),
}
Expand Down Expand Up @@ -1242,7 +1242,7 @@ var _ = Describe("CloudProvider", func() {
Tags: []ec2types.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}})
nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{{Tags: map[string]string{"Name": "test-subnet-1"}}}
ExpectApplied(ctx, env.Client, nodePool, nodeClass)
controller := nodeclass.NewController(awsEnv.Clock, env.Client, recorder, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider, awsEnv.CapacityReservationProvider, awsEnv.EC2API, awsEnv.ValidationCache, awsEnv.AMIResolver)
controller := nodeclass.NewController(awsEnv.Clock, env.Client, recorder, fake.DefaultRegion, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider, awsEnv.CapacityReservationProvider, awsEnv.EC2API, awsEnv.ValidationCache, awsEnv.AMIResolver)
ExpectObjectReconciled(ctx, env.Client, controller, nodeClass)
podSubnet1 := coretest.UnschedulablePod()
ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, podSubnet1)
Expand Down
2 changes: 1 addition & 1 deletion pkg/controllers/controllers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func NewControllers(
) []controller.Controller {
controllers := []controller.Controller{
nodeclasshash.NewController(kubeClient),
nodeclass.NewController(clk, kubeClient, recorder, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, launchTemplateProvider, capacityReservationProvider, ec2api, validationCache, amiResolver),
nodeclass.NewController(clk, kubeClient, recorder, cfg.Region, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, launchTemplateProvider, capacityReservationProvider, ec2api, validationCache, amiResolver),
nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider),
nodeclaimtagging.NewController(kubeClient, cloudProvider, instanceProvider),
controllerspricing.NewController(pricingProvider),
Expand Down
4 changes: 2 additions & 2 deletions pkg/controllers/nodeclass/capacityreservation_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ var _ = Describe("NodeClass Capacity Reservation Reconciler", func() {
InstanceMatchCriteria: ec2types.InstanceMatchCriteriaTargeted,
CapacityReservationId: lo.ToPtr("cr-m5.large-1a-2"),
AvailableInstanceCount: lo.ToPtr[int32](10),
Tags: utils.MergeTags(discoveryTags),
Tags: utils.EC2MergeTags(discoveryTags),
State: ec2types.CapacityReservationStateActive,
},
{
Expand All @@ -74,7 +74,7 @@ var _ = Describe("NodeClass Capacity Reservation Reconciler", func() {
InstanceMatchCriteria: ec2types.InstanceMatchCriteriaTargeted,
CapacityReservationId: lo.ToPtr("cr-m5.large-1b-2"),
AvailableInstanceCount: lo.ToPtr[int32](15),
Tags: utils.MergeTags(discoveryTags),
Tags: utils.EC2MergeTags(discoveryTags),
State: ec2types.CapacityReservationStateActive,
},
},
Expand Down
12 changes: 8 additions & 4 deletions pkg/controllers/nodeclass/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/utils/clock"
"sigs.k8s.io/karpenter/pkg/operator/injection"
"sigs.k8s.io/karpenter/pkg/operator/options"
karpoptions "sigs.k8s.io/karpenter/pkg/operator/options"
nodeclaimutils "sigs.k8s.io/karpenter/pkg/utils/nodeclaim"
"sigs.k8s.io/karpenter/pkg/utils/result"

Expand All @@ -48,6 +48,7 @@ import (

v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
sdk "github.com/aws/karpenter-provider-aws/pkg/aws"
"github.com/aws/karpenter-provider-aws/pkg/operator/options"
"github.com/aws/karpenter-provider-aws/pkg/providers/amifamily"
"github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation"
"github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile"
Expand All @@ -59,6 +60,7 @@ import (
type Controller struct {
kubeClient client.Client
recorder events.Recorder
region string
launchTemplateProvider launchtemplate.Provider
instanceProfileProvider instanceprofile.Provider
validation *Validation
Expand All @@ -69,6 +71,7 @@ func NewController(
clk clock.Clock,
kubeClient client.Client,
recorder events.Recorder,
region string,
subnetProvider subnet.Provider,
securityGroupProvider securitygroup.Provider,
amiProvider amifamily.Provider,
Expand All @@ -83,6 +86,7 @@ func NewController(
return &Controller{
kubeClient: kubeClient,
recorder: recorder,
region: region,
launchTemplateProvider: launchTemplateProvider,
instanceProfileProvider: instanceProfileProvider,
validation: validation,
Expand All @@ -91,7 +95,7 @@ func NewController(
NewCapacityReservationReconciler(clk, capacityReservationProvider),
NewSubnetReconciler(subnetProvider),
NewSecurityGroupReconciler(securityGroupProvider),
NewInstanceProfileReconciler(instanceProfileProvider),
NewInstanceProfileReconciler(instanceProfileProvider, region),
validation,
NewReadinessReconciler(launchTemplateProvider),
},
Expand Down Expand Up @@ -129,7 +133,7 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass)
var results []reconcile.Result
var errs error
for _, reconciler := range c.reconcilers {
if _, ok := reconciler.(*CapacityReservation); ok && !options.FromContext(ctx).FeatureGates.ReservedCapacity {
if _, ok := reconciler.(*CapacityReservation); ok && !karpoptions.FromContext(ctx).FeatureGates.ReservedCapacity {
continue
}
res, err := reconciler.Reconcile(ctx, nodeClass)
Expand Down Expand Up @@ -168,7 +172,7 @@ func (c *Controller) finalize(ctx context.Context, nodeClass *v1.EC2NodeClass) (
return reconcile.Result{RequeueAfter: time.Minute * 10}, nil // periodically fire the event
}
if nodeClass.Spec.Role != "" {
if err := c.instanceProfileProvider.Delete(ctx, nodeClass); err != nil {
if err := c.instanceProfileProvider.Delete(ctx, nodeClass.InstanceProfileName(options.FromContext(ctx).ClusterName, c.region)); err != nil {
return reconcile.Result{}, fmt.Errorf("deleting instance profile, %w", err)
}
}
Expand Down
16 changes: 12 additions & 4 deletions pkg/controllers/nodeclass/instanceprofile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,26 +22,34 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile"

v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
"github.com/aws/karpenter-provider-aws/pkg/operator/options"
"github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile"
)

type InstanceProfile struct {
instanceProfileProvider instanceprofile.Provider
region string
}

func NewInstanceProfileReconciler(instanceProfileProvider instanceprofile.Provider) *InstanceProfile {
func NewInstanceProfileReconciler(instanceProfileProvider instanceprofile.Provider, region string) *InstanceProfile {
return &InstanceProfile{
instanceProfileProvider: instanceProfileProvider,
region: region,
}
}

func (ip *InstanceProfile) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconcile.Result, error) {
if nodeClass.Spec.Role != "" {
name, err := ip.instanceProfileProvider.Create(ctx, nodeClass)
if err != nil {
profileName := nodeClass.InstanceProfileName(options.FromContext(ctx).ClusterName, ip.region)
if err := ip.instanceProfileProvider.Create(
ctx,
profileName,
nodeClass.InstanceProfileRole(),
nodeClass.InstanceProfileTags(options.FromContext(ctx).ClusterName, ip.region),
); err != nil {
return reconcile.Result{}, fmt.Errorf("creating instance profile, %w", err)
}
nodeClass.Status.InstanceProfile = name
nodeClass.Status.InstanceProfile = profileName
} else {
nodeClass.Status.InstanceProfile = lo.FromPtr(nodeClass.Spec.InstanceProfile)
}
Expand Down
1 change: 1 addition & 0 deletions pkg/controllers/nodeclass/suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ var _ = BeforeSuite(func() {
awsEnv.Clock,
env.Client,
events.NewRecorder(&record.FakeRecorder{}),
fake.DefaultRegion,
awsEnv.SubnetProvider,
awsEnv.SecurityGroupProvider,
awsEnv.AMIProvider,
Expand Down
2 changes: 1 addition & 1 deletion pkg/operator/operator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont

subnetProvider := subnet.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), cache.New(awscache.AvailableIPAddressTTL, awscache.DefaultCleanupInterval), cache.New(awscache.AssociatePublicIPAddressTTL, awscache.DefaultCleanupInterval))
securityGroupProvider := securitygroup.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval))
instanceProfileProvider := instanceprofile.NewDefaultProvider(cfg.Region, iam.NewFromConfig(cfg), cache.New(awscache.InstanceProfileTTL, awscache.DefaultCleanupInterval))
instanceProfileProvider := instanceprofile.NewDefaultProvider(iam.NewFromConfig(cfg), cache.New(awscache.InstanceProfileTTL, awscache.DefaultCleanupInterval))
pricingProvider := pricing.NewDefaultProvider(
ctx,
pricing.NewAPI(cfg),
Expand Down
4 changes: 2 additions & 2 deletions pkg/providers/capacityreservation/suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ var _ = Describe("Capacity Reservation Provider", func() {
InstanceMatchCriteria: ec2types.InstanceMatchCriteriaTargeted,
CapacityReservationId: lo.ToPtr("cr-m5.large-1a-1"),
AvailableInstanceCount: lo.ToPtr[int32](10),
Tags: utils.MergeTags(discoveryTags),
Tags: utils.EC2MergeTags(discoveryTags),
State: ec2types.CapacityReservationStateActive,
},
{
Expand All @@ -83,7 +83,7 @@ var _ = Describe("Capacity Reservation Provider", func() {
InstanceMatchCriteria: ec2types.InstanceMatchCriteriaTargeted,
CapacityReservationId: lo.ToPtr("cr-m5.large-1a-2"),
AvailableInstanceCount: lo.ToPtr[int32](15),
Tags: utils.MergeTags(discoveryTags),
Tags: utils.EC2MergeTags(discoveryTags),
State: ec2types.CapacityReservationStateActive,
},
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/providers/instance/instance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,9 +309,9 @@ func GetCreateFleetInput(nodeClass *v1.EC2NodeClass, capacityType string, tags m
TotalTargetCapacity: aws.Int32(1),
},
TagSpecifications: []ec2types.TagSpecification{
{ResourceType: ec2types.ResourceTypeInstance, Tags: utils.MergeTags(tags)},
{ResourceType: ec2types.ResourceTypeVolume, Tags: utils.MergeTags(tags)},
{ResourceType: ec2types.ResourceTypeFleet, Tags: utils.MergeTags(tags)},
{ResourceType: ec2types.ResourceTypeInstance, Tags: utils.EC2MergeTags(tags)},
{ResourceType: ec2types.ResourceTypeVolume, Tags: utils.EC2MergeTags(tags)},
{ResourceType: ec2types.ResourceTypeFleet, Tags: utils.EC2MergeTags(tags)},
},
}
}
Expand Down
Loading