Skip to content

Commit a2d2828

Browse files
zarvdzarvd
committed
Introduce KeyVaultCredential
1 parent c7384e7 commit a2d2828

File tree

7 files changed

+214
-7
lines changed

7 files changed

+214
-7
lines changed

‎pkg/azclient/arm_conf.go

+4
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,7 @@ func GetAzCoreClientOption(armConfig *ARMClientConfig) (*policy.ClientOptions, e
6262
}
6363
return &azCoreClientConfig, nil
6464
}
65+
66+
func IsMultiTenant(armConfig *ARMClientConfig) bool {
67+
return armConfig != nil && armConfig.NetworkResourceTenantID != "" && !strings.EqualFold(armConfig.NetworkResourceTenantID, armConfig.GetTenantID())
68+
}

‎pkg/azclient/armauth/auxiliary_auth_policy.go

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package armauth
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"net/http"
23+
"strings"
24+
25+
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
26+
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
27+
)
28+
29+
const (
30+
HeaderAuthorizationAuxiliary = "x-ms-authorization-auxiliary"
31+
)
32+
33+
type AuxiliaryAuthPolicy struct {
34+
credentials []azcore.TokenCredential
35+
scope string
36+
}
37+
38+
func NewAuxiliaryAuthPolicy(credentials []azcore.TokenCredential, scope string) *AuxiliaryAuthPolicy {
39+
return &AuxiliaryAuthPolicy{
40+
credentials: credentials,
41+
scope: scope,
42+
}
43+
}
44+
45+
func (p *AuxiliaryAuthPolicy) Do(req *policy.Request) (*http.Response, error) {
46+
tokens := make([]string, 0, len(p.credentials))
47+
48+
for _, cred := range p.credentials {
49+
token, err := cred.GetToken(context.TODO(), policy.TokenRequestOptions{
50+
Scopes: []string{p.scope},
51+
})
52+
if err != nil {
53+
return nil, err
54+
}
55+
tokens = append(tokens, fmt.Sprintf("Bearer %s", token.Token))
56+
}
57+
req.Raw().Header.Set(HeaderAuthorizationAuxiliary, strings.Join(tokens, ", "))
58+
return req.Next()
59+
}

‎pkg/azclient/armauth/keyvault_credential.go

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package armauth
18+
19+
import (
20+
"context"
21+
"encoding/json"
22+
"fmt"
23+
"time"
24+
25+
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
26+
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
27+
"github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets"
28+
)
29+
30+
type KeyVaultCredential struct {
31+
secretClient *azsecrets.Client
32+
secretPath string
33+
34+
token *azcore.AccessToken
35+
}
36+
37+
type KeyVaultCredentialSecret struct {
38+
AccessToken string `json:"access_token"`
39+
ExpiresOn time.Time `json:"expires_on"`
40+
}
41+
42+
func NewKeyVaultCredential(
43+
msiCredential azcore.TokenCredential,
44+
keyVaultURL string,
45+
secretName string,
46+
) (*KeyVaultCredential, error) {
47+
cli, err := azsecrets.NewClient(keyVaultURL, msiCredential, nil)
48+
if err != nil {
49+
return nil, fmt.Errorf("create KeyVault client: %w", err)
50+
}
51+
52+
rv := &KeyVaultCredential{
53+
secretClient: cli,
54+
secretPath: secretName,
55+
}
56+
57+
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
58+
defer cancel()
59+
if err := rv.refreshToken(ctx); err != nil {
60+
return nil, fmt.Errorf("refresh token: %w", err)
61+
}
62+
63+
return rv, nil
64+
}
65+
66+
func (c *KeyVaultCredential) refreshToken(ctx context.Context) error {
67+
const LatestVersion = ""
68+
69+
resp, err := c.secretClient.GetSecret(ctx, c.secretPath, LatestVersion, nil)
70+
if err != nil {
71+
return err
72+
}
73+
if resp.Value == nil {
74+
return fmt.Errorf("secret value is nil")
75+
}
76+
77+
var secret KeyVaultCredentialSecret
78+
if err := json.Unmarshal([]byte(*resp.Value), &secret); err != nil {
79+
return fmt.Errorf("unmarshal secret value `%s`: %w", *resp.Value, err)
80+
}
81+
82+
c.token = &azcore.AccessToken{
83+
Token: secret.AccessToken,
84+
ExpiresOn: secret.ExpiresOn,
85+
}
86+
87+
return nil
88+
}
89+
90+
func (c *KeyVaultCredential) GetToken(ctx context.Context, opts policy.TokenRequestOptions) (azcore.AccessToken, error) {
91+
const RefreshTokenOffset = 5 * time.Minute
92+
93+
if c.token != nil && c.token.ExpiresOn.Add(RefreshTokenOffset).Before(time.Now()) {
94+
return *c.token, nil
95+
}
96+
97+
if err := c.refreshToken(ctx); err != nil {
98+
return azcore.AccessToken{}, fmt.Errorf("refresh token: %w", err)
99+
}
100+
101+
return *c.token, nil
102+
}

‎pkg/azclient/auth.go

+31-7
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,21 @@ import (
2424
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
2525
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
2626
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
27+
28+
"sigs.k8s.io/cloud-provider-azure/pkg/azclient/armauth"
2729
)
2830

2931
type AuthProvider struct {
30-
FederatedIdentityCredential azcore.TokenCredential
31-
ManagedIdentityCredential azcore.TokenCredential
32-
ClientSecretCredential azcore.TokenCredential
32+
FederatedIdentityCredential azcore.TokenCredential
33+
34+
ManagedIdentityCredential azcore.TokenCredential
35+
ClientSecretCredential azcore.TokenCredential
36+
ClientCertificateCredential azcore.TokenCredential
37+
38+
NetworkTokenCredential azcore.TokenCredential
3339
NetworkClientSecretCredential azcore.TokenCredential
34-
MultiTenantCredential azcore.TokenCredential
35-
ClientCertificateCredential azcore.TokenCredential
40+
41+
MultiTenantCredential azcore.TokenCredential
3642
}
3743

3844
func NewAuthProvider(armConfig *ARMClientConfig, config *AzureAuthConfig, clientOptionsMutFn ...func(option *policy.ClientOptions)) (*AuthProvider, error) {
@@ -76,6 +82,20 @@ func NewAuthProvider(armConfig *ARMClientConfig, config *AzureAuthConfig, client
7682
}
7783
}
7884

85+
var (
86+
networkTokenCredential azcore.TokenCredential
87+
)
88+
if config.UseManagedIdentityExtension && config.AuxiliaryTokenProvider != nil && IsMultiTenant(armConfig) {
89+
networkTokenCredential, err = armauth.NewKeyVaultCredential(
90+
managedIdentityCredential,
91+
config.AuxiliaryTokenProvider.KeyVaultURL,
92+
config.AuxiliaryTokenProvider.SecretName,
93+
)
94+
if err != nil {
95+
return nil, fmt.Errorf("create KeyVaultCredential for auxiliary token provider: %w", err)
96+
}
97+
}
98+
7999
// ClientSecretCredential is used for client secret
80100
var clientSecretCredential azcore.TokenCredential
81101
var networkClientSecretCredential azcore.TokenCredential
@@ -88,7 +108,7 @@ func NewAuthProvider(armConfig *ARMClientConfig, config *AzureAuthConfig, client
88108
if err != nil {
89109
return nil, err
90110
}
91-
if len(armConfig.NetworkResourceTenantID) > 0 && !strings.EqualFold(armConfig.NetworkResourceTenantID, armConfig.GetTenantID()) {
111+
if IsMultiTenant(armConfig) {
92112
credOptions := &azidentity.ClientSecretCredentialOptions{
93113
ClientOptions: *clientOption,
94114
}
@@ -128,7 +148,7 @@ func NewAuthProvider(armConfig *ARMClientConfig, config *AzureAuthConfig, client
128148
if err != nil {
129149
return nil, err
130150
}
131-
if len(armConfig.NetworkResourceTenantID) > 0 && !strings.EqualFold(armConfig.NetworkResourceTenantID, armConfig.GetTenantID()) {
151+
if IsMultiTenant(armConfig) {
132152
networkClientSecretCredential, err = azidentity.NewClientCertificateCredential(armConfig.NetworkResourceTenantID, config.GetAADClientID(), certificate, privateKey, credOptions)
133153
if err != nil {
134154
return nil, err
@@ -150,6 +170,7 @@ func NewAuthProvider(armConfig *ARMClientConfig, config *AzureAuthConfig, client
150170
ClientSecretCredential: clientSecretCredential,
151171
ClientCertificateCredential: clientCertificateCredential,
152172
NetworkClientSecretCredential: networkClientSecretCredential,
173+
NetworkTokenCredential: networkTokenCredential,
153174
MultiTenantCredential: multiTenantCredential,
154175
}, nil
155176
}
@@ -173,6 +194,9 @@ func (factory *AuthProvider) GetNetworkAzIdentity() azcore.TokenCredential {
173194
if factory.NetworkClientSecretCredential != nil {
174195
return factory.NetworkClientSecretCredential
175196
}
197+
if factory.NetworkTokenCredential != nil {
198+
return factory.NetworkTokenCredential
199+
}
176200
return nil
177201
}
178202

‎pkg/azclient/auth_conf.go

+8
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,14 @@ type AzureAuthConfig struct {
4242
AADFederatedTokenFile string `json:"aadFederatedTokenFile,omitempty" yaml:"aadFederatedTokenFile,omitempty"`
4343
// Use workload identity federation for the virtual machine to access Azure ARM APIs
4444
UseFederatedWorkloadIdentityExtension bool `json:"useFederatedWorkloadIdentityExtension,omitempty" yaml:"useFederatedWorkloadIdentityExtension,omitempty"`
45+
// Auxiliary token provider for accessing resources from network tenant
46+
// Require MSI to be enabled and have permission to access the KeyVault
47+
AuxiliaryTokenProvider *AzureAuthAuxiliaryTokenProvider `json:"auxiliaryTokenProvider,omitempty" yaml:"auxiliaryTokenProvider,omitempty"`
48+
}
49+
50+
type AzureAuthAuxiliaryTokenProvider struct {
51+
KeyVaultURL string `json:"keyVaultURL,omitempty" yaml:"keyVaultURL,omitempty"`
52+
SecretName string `json:"secretName" yaml:"secretName"`
4553
}
4654

4755
func (config *AzureAuthConfig) GetAADClientID() string {

‎pkg/azclient/go.mod

+5
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ go 1.20
55
require (
66
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.0
77
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
8+
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0
89
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0
910
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0
1011
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.8.0
@@ -16,6 +17,7 @@ require (
1617
github.com/google/uuid v1.6.0
1718
github.com/onsi/ginkgo/v2 v2.17.1
1819
github.com/onsi/gomega v1.32.0
20+
github.com/stretchr/testify v1.8.4
1921
go.uber.org/mock v0.4.0
2022
golang.org/x/crypto v0.21.0
2123
golang.org/x/sync v0.6.0
@@ -26,7 +28,9 @@ require (
2628

2729
require (
2830
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
31+
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
2932
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
33+
github.com/davecgh/go-spew v1.1.1 // indirect
3034
github.com/go-logr/logr v1.4.1 // indirect
3135
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
3236
github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
@@ -35,6 +39,7 @@ require (
3539
github.com/kr/pretty v0.3.1 // indirect
3640
github.com/kylelemons/godebug v1.1.0 // indirect
3741
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
42+
github.com/pmezard/go-difflib v1.0.0 // indirect
3843
github.com/rogpeppe/go-internal v1.10.0 // indirect
3944
golang.org/x/net v0.22.0 // indirect
4045
golang.org/x/sys v0.18.0 // indirect

‎pkg/azclient/go.sum

+5
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+
44
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
55
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
66
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
7+
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 h1:xnO4sFyG8UH2fElBkcqLTOZsAajvKfnSlgBBW8dXYjw=
8+
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0/go.mod h1:XD3DIOOVgBCO03OleB1fHjgktVRFxlT++KwKgIOewdM=
9+
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw=
10+
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
711
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0 h1:ui3YNbxfW7J3tTFIZMH6LIGRjCngp+J+nIFlnizfNTE=
812
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0/go.mod h1:gZmgV+qBqygoznvqo2J9oKZAFziqhLZ2xE/WVUmzkHA=
913
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 h1:DWlwvVV5r/Wy1561nZ3wrpI1/vDIBRY/Wd1HWaRBZWA=
@@ -65,6 +69,7 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj
6569
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
6670
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
6771
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
72+
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
6873
go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
6974
go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
7075
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=

0 commit comments

Comments
 (0)