Skip to content

Commit a697e9b

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request #9718 from patrickdillon/OCPBUGS-56008-azure-image-pull
OCPBUGS-56008: default Azure to create VM user-assigned identities
2 parents 94c2b43 + 8e54931 commit a697e9b

File tree

96 files changed

+23911
-34
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

96 files changed

+23911
-34
lines changed

‎data/data/install.openshift.io_installconfigs.yaml

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -249,14 +249,16 @@ spec:
249249
host.
250250
type: boolean
251251
identity:
252-
default: None
252+
default: UserAssigned
253253
description: |-
254254
Identity is the type of identity used for the virtual machine.
255255
The type 'SystemAssigned' is an implicitly created identity.
256256
The generated identity will be assigned a Subscription contributor role.
257257
The type 'UserAssigned' is a standalone Azure resource provided by the user
258-
and assigned to the VM
258+
and assigned to the VM.
259259
Identity can only be set for control-plane nodes.
260+
The default identity is a user-assigned identity, generated by the installer.
261+
This default is expected to change in future releases.
260262
properties:
261263
type:
262264
description: Type specifies the type of identity to be
@@ -1571,14 +1573,16 @@ spec:
15711573
VM host.
15721574
type: boolean
15731575
identity:
1574-
default: None
1576+
default: UserAssigned
15751577
description: |-
15761578
Identity is the type of identity used for the virtual machine.
15771579
The type 'SystemAssigned' is an implicitly created identity.
15781580
The generated identity will be assigned a Subscription contributor role.
15791581
The type 'UserAssigned' is a standalone Azure resource provided by the user
1580-
and assigned to the VM
1582+
and assigned to the VM.
15811583
Identity can only be set for control-plane nodes.
1584+
The default identity is a user-assigned identity, generated by the installer.
1585+
This default is expected to change in future releases.
15821586
properties:
15831587
type:
15841588
description: Type specifies the type of identity to
@@ -2832,14 +2836,16 @@ spec:
28322836
host.
28332837
type: boolean
28342838
identity:
2835-
default: None
2839+
default: UserAssigned
28362840
description: |-
28372841
Identity is the type of identity used for the virtual machine.
28382842
The type 'SystemAssigned' is an implicitly created identity.
28392843
The generated identity will be assigned a Subscription contributor role.
28402844
The type 'UserAssigned' is a standalone Azure resource provided by the user
2841-
and assigned to the VM
2845+
and assigned to the VM.
28422846
Identity can only be set for control-plane nodes.
2847+
The default identity is a user-assigned identity, generated by the installer.
2848+
This default is expected to change in future releases.
28432849
properties:
28442850
type:
28452851
description: Type specifies the type of identity to be
@@ -4567,14 +4573,16 @@ spec:
45674573
host.
45684574
type: boolean
45694575
identity:
4570-
default: None
4576+
default: UserAssigned
45714577
description: |-
45724578
Identity is the type of identity used for the virtual machine.
45734579
The type 'SystemAssigned' is an implicitly created identity.
45744580
The generated identity will be assigned a Subscription contributor role.
45754581
The type 'UserAssigned' is a standalone Azure resource provided by the user
4576-
and assigned to the VM
4582+
and assigned to the VM.
45774583
Identity can only be set for control-plane nodes.
4584+
The default identity is a user-assigned identity, generated by the installer.
4585+
This default is expected to change in future releases.
45784586
properties:
45794587
type:
45804588
description: Type specifies the type of identity to be

‎go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,12 @@ require (
1212
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
1313
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
1414
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
15+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.2
1516
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1
1617
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0
1718
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns v1.2.0
1819
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0
20+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0
1921
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0
2022
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1
2123
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0

‎go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH
3131
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
3232
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
3333
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
34+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.2 h1:qiir/pptnHqp6hV8QwV+IExYIf6cPsXBfUDUXQ27t2Y=
35+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.2/go.mod h1:jVRrRDLCOuif95HDYC23ADTMlvahB7tMdl519m9Iyjc=
3436
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1 h1:UPeCRD+XY7QlaGQte2EVI2iOcWvUYA2XY8w5T/8v0NQ=
3537
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1/go.mod h1:oGV6NlB0cvi1ZbYRR2UN44QHxWFyGk+iylgD0qaMXjA=
3638
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4=
@@ -47,6 +49,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.
4749
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0/go.mod h1:StGsLbuJh06Bd8IBfnAlIFV3fLb+gkczONWf15hpX2E=
4850
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o=
4951
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0/go.mod h1:mLfWfj8v3jfWKsL9G4eoBoXVcsqcIUTapmdKy7uGOp0=
52+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 h1:z4YeiSXxnUI+PqB46Yj6MZA3nwb1CcJIkEMDrzUd8Cs=
53+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0/go.mod h1:rko9SzMxcMk0NJsNAxALEGaTYyy79bNRwxgJfrH0Spw=
5054
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 h1:QM6sE5k2ZT/vI5BEe0r7mqjsUSnhVBFbOsVkEuaEfiA=
5155
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0/go.mod h1:243D9iHbcQXoFUtgHJwL7gl2zx1aDuDMjvBZVGr2uW0=
5256
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 h1:bWh0Z2rOEDfB/ywv/l0iHN1JgyazE6kW/aIA89+CEK0=

‎pkg/asset/machines/azure/azuremachines.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,17 @@ func GenerateMachines(clusterID, resourceGroup, subscriptionID string, session *
146146
for i, id := range mpool.Identity.UserAssignedIdentities {
147147
userAssignedIdentities[i] = capz.UserAssignedIdentity{ProviderID: id.ProviderID()}
148148
}
149+
150+
// If identity type is UserAssigned, but no identities are provided, the installer
151+
// will create one. Populate the manifest with a reference to that identity.
152+
if mpool.Identity.Type == capz.VMIdentityUserAssigned && len(userAssignedIdentities) == 0 {
153+
userAssignedIdentities = []capz.UserAssignedIdentity{
154+
{
155+
ProviderID: fmt.Sprintf("/subscriptions/%s/resourcegroups/%s/providers/Microsoft.ManagedIdentity/userAssignedIdentities/%s-identity", subscriptionID, resourceGroup, clusterID),
156+
},
157+
}
158+
}
159+
149160
storageAccountName := aztypes.GetStorageAccountName(clusterID)
150161

151162
defaultDiag := &capz.Diagnostics{

‎pkg/asset/machines/azure/machines.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
corev1 "k8s.io/api/core/v1"
1111
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1212
"k8s.io/apimachinery/pkg/runtime"
13-
"sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
13+
capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
1414

1515
v1 "github.com/openshift/api/config/v1"
1616
machinev1 "github.com/openshift/api/machine/v1"
@@ -216,6 +216,9 @@ func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string
216216
managedIdentity := ""
217217
if len(mpool.Identity.UserAssignedIdentities) > 0 {
218218
managedIdentity = mpool.Identity.UserAssignedIdentities[0].ProviderID()
219+
} else if mpool.Identity.Type == capz.VMIdentityUserAssigned {
220+
// In this case, the installer will create the user-assigned identity.
221+
managedIdentity = fmt.Sprintf("%s-identity", clusterID)
219222
}
220223

221224
var diskEncryptionSet *machineapi.DiskEncryptionSetParameters
@@ -299,11 +302,11 @@ func getBootDiagnosticObject(diag *azure.BootDiagnostics, cloudName string, role
299302
}
300303
return nil
301304
}
302-
if diag.Type == v1beta1.DisabledDiagnosticsStorage {
305+
if diag.Type == capz.DisabledDiagnosticsStorage {
303306
return nil
304307
}
305308
bootDiagnostics := &machineapi.AzureDiagnostics{Boot: &machineapi.AzureBootDiagnostics{}}
306-
if diag.Type == v1beta1.ManagedDiagnosticsStorage {
309+
if diag.Type == capz.ManagedDiagnosticsStorage {
307310
bootDiagnostics.Boot.StorageAccountType = machineapi.AzureManagedAzureDiagnosticsStorage
308311
} else {
309312
bootDiagnostics.Boot.StorageAccountType = machineapi.CustomerManagedAzureDiagnosticsStorage

‎pkg/asset/machines/clusterapi.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ func (c *ClusterAPI) Generate(ctx context.Context, dependencies asset.Parents) e
216216
}
217217
c.FileList = append(c.FileList, bootstrapAWSMachine...)
218218
case azuretypes.Name:
219-
mpool := defaultAzureMachinePoolPlatform()
219+
mpool := defaultAzureMachinePoolPlatform(installConfig.Config.Platform.Azure.CloudName)
220220
mpool.InstanceType = azuredefaults.ControlPlaneInstanceType(
221221
installConfig.Config.Platform.Azure.CloudName,
222222
installConfig.Config.Platform.Azure.Region,

‎pkg/asset/machines/master.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ func (m *Master) Generate(ctx context.Context, dependencies asset.Parents) error
339339
}
340340
openstack.ConfigMasters(machines, clusterID.InfraID)
341341
case azuretypes.Name:
342-
mpool := defaultAzureMachinePoolPlatform()
342+
mpool := defaultAzureMachinePoolPlatform(installConfig.Config.Platform.Azure.CloudName)
343343
mpool.InstanceType = azuredefaults.ControlPlaneInstanceType(
344344
installConfig.Config.Platform.Azure.CloudName,
345345
installConfig.Config.Platform.Azure.Region,

‎pkg/asset/machines/worker.go

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,13 +114,18 @@ func defaultAWSMachinePoolPlatform(poolName string) awstypes.MachinePool {
114114
}
115115
}
116116

117-
func defaultAzureMachinePoolPlatform() azuretypes.MachinePool {
117+
func defaultAzureMachinePoolPlatform(env azuretypes.CloudEnvironment) azuretypes.MachinePool {
118+
idType := capz.VMIdentityUserAssigned
119+
if env == azuretypes.StackCloud {
120+
idType = capz.VMIdentityNone
121+
}
122+
118123
return azuretypes.MachinePool{
119124
OSDisk: azuretypes.OSDisk{
120125
DiskSizeGB: powerOfTwoRootVolumeSize,
121126
DiskType: azuretypes.DefaultDiskType,
122127
},
123-
Identity: &azuretypes.VMIdentity{Type: capz.VMIdentityNone},
128+
Identity: &azuretypes.VMIdentity{Type: idType},
124129
}
125130
}
126131

@@ -486,7 +491,7 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error
486491
machineSets = append(machineSets, set)
487492
}
488493
case azuretypes.Name:
489-
mpool := defaultAzureMachinePoolPlatform()
494+
mpool := defaultAzureMachinePoolPlatform(installConfig.Config.Platform.Azure.CloudName)
490495
mpool.InstanceType = azuredefaults.ComputeInstanceType(
491496
installConfig.Config.Platform.Azure.CloudName,
492497
installConfig.Config.Platform.Azure.Region,

‎pkg/asset/manifests/azure/cloudproviderconfig.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ type CloudProviderConfig struct {
2121
SubnetName string
2222
ResourceManagerEndpoint string
2323
ARO bool
24+
UseManagedIdentity bool
2425
}
2526

2627
// JSON generates the cloud provider json config for the azure platform.
@@ -57,6 +58,16 @@ func (params CloudProviderConfig) JSON() (string, error) {
5758
ExcludeMasterFromStandardLB: &excludeMasterFromStandardLB,
5859
}
5960

61+
if params.UseManagedIdentity {
62+
config.UseManagedIdentityExtension = true
63+
// The cloud provider needs the clientID which is only known after terraform has run.
64+
// When left empty, the existing managed identity on the VM will be used.
65+
// By leaving it empty, we don't have to create the identity before running the installer.
66+
// We only need to know that there will be one assigned to the VM, and we control this.
67+
// ref: https://github.com/kubernetes/kubernetes/blob/4b7c607ba47928a7be77fadef1550d6498397a4c/staging/src/k8s.io/legacy-cloud-providers/azure/auth/azure_auth.go#L69
68+
config.UserAssignedIdentityID = ""
69+
}
70+
6071
if params.CloudName == azure.StackCloud {
6172
config.authConfig.ResourceManagerEndpoint = params.ResourceManagerEndpoint
6273
config.UseInstanceMetadata = false

‎pkg/asset/manifests/cloudproviderconfig.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ func (cpc *CloudProviderConfig) Generate(ctx context.Context, dependencies asset
152152
VirtualNetworkName: vnet,
153153
SubnetName: subnet,
154154
ResourceManagerEndpoint: installConfig.Config.Azure.ARMEndpoint,
155+
UseManagedIdentity: installConfig.Config.CreateAzureIdentity(),
155156
}.JSON()
156157
if err != nil {
157158
return errors.Wrap(err, "could not create cloud provider config")

‎pkg/infrastructure/azure/azure.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,23 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
135135
p.clientOptions = opts
136136
p.computeClientOptions = computeClientOpts
137137

138+
if err = handleIdentity(ctx, identityInput{
139+
installConfig: installConfig,
140+
region: platform.Region,
141+
resourceGroupName: p.ResourceGroupName,
142+
subscriptionID: subscriptionID,
143+
tokenCredential: tokenCredential,
144+
infraID: in.InfraID,
145+
clientOpts: p.clientOptions,
146+
tags: p.Tags,
147+
}); err != nil {
148+
errMsg := "error creating user-assigned identity: please ensure your user credentials " +
149+
"have the User Access Admin Role or if you are not utilizing an Azure Container Registry " +
150+
"you can set installconfig.platform.azure.defaultMachinePlatform.identity: None to skip " +
151+
"the creation of the identity: creation failed with: %w"
152+
return fmt.Errorf(errMsg, err)
153+
}
154+
138155
// Creating a dummy nsg for existing vnets installation to appease the ingress operator.
139156
if in.InstallConfig.Config.Azure.VirtualNetwork != "" {
140157
networkClientFactory, err := armnetwork.NewClientFactory(subscriptionID, tokenCredential, p.clientOptions)

0 commit comments

Comments
 (0)