Skip to content

feat: added regional secret support for secret-manager #3365

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
b45b0fc
secret-manager: added regional secret support
abheda-crest Oct 30, 2024
c75d512
Merge branch 'main' into secret-manager-regional-support
abheda-crest Nov 4, 2024
9557d43
secret-manager: updated pom.xml of sample
abheda-crest Nov 4, 2024
6f23874
Merge branch 'main' into secret-manager-regional-support
abheda-crest Nov 11, 2024
9f6bf38
secret-manager: replaced junit assertion with assertj
abheda-crest Nov 11, 2024
e47d196
Merge branch 'main' into secret-manager-regional-support
abheda-crest Feb 10, 2025
ca13e36
secret-manager: updated regional sample code based on the changes in …
abheda-crest Feb 10, 2025
50c0f80
Merge branch 'main' into secret-manager-regional-support
abheda-crest Feb 12, 2025
1bcd340
secret-manager: addressed the review comments
abheda-crest Feb 12, 2025
9deaa78
secret-manager: addressed the review comments
abheda-crest Feb 16, 2025
a35f8f2
secret-manager: addressed the review comments
abheda-crest Feb 16, 2025
0f45864
secret-manager: addressed the review comments
abheda-crest Feb 16, 2025
e175a35
Merge remote-tracking branch 'origin/secret-manager-regional-support'…
abheda-crest Feb 16, 2025
64215f5
secret-manager: addressed the review comments
abheda-crest Feb 20, 2025
ab33af2
secret-manager: addressed the review comments
abheda-crest Mar 3, 2025
4bbf654
secret-manager: added multi-region support for SecretManagerTemplate
abheda-crest Mar 5, 2025
56f3ead
secret-manager: added multi-region support
abheda-crest Mar 6, 2025
5db414a
secret-manager: modified unit-tests
abheda-crest Mar 6, 2025
ead04ca
secret-manager: minor fixes
abheda-crest Mar 6, 2025
fa72e52
Merge branch 'main' into secret-manager-regional-support
meltsufin Mar 7, 2025
e83f844
secret-manager: addressed review comments
abheda-crest Mar 11, 2025
2208a5c
secret-manager: addressed review comments
abheda-crest Mar 11, 2025
7683612
secret-manager: addressed review comments
abheda-crest Mar 11, 2025
65f6e17
Merge branch 'main' into secret-manager-regional-support
abheda-crest Mar 21, 2025
d3dca72
secret-manager: fixed test cases
abheda-crest Mar 21, 2025
9bfe41a
secret-manager: addressed review comments
abheda-crest Mar 26, 2025
edfc0a9
Merge branch 'main' into secret-manager-regional-support
zhumin8 Apr 4, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions docs/src/main/asciidoc/secretmanager.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,22 @@ sm@<secret-id>/<version>

# 5. Shortest form - specify secret ID, use default project and latest version.
sm@<secret-id>

# 6. Long form - specify project ID, location ID, secret ID, and version
sm@projects/<project-id>/locations/<location-id>/secrets/<secret-id>/versions/<version-id>

# 7. Long form - specify project ID, location ID, secret ID, and use latest version
sm@projects/<project-id>/locations/<location-id>/secrets/<secret-id>

# 8. Short form - specify project ID, location ID, secret ID, and version
sm@<project-id>/<location-id>/<secret-id>/<version-id>

# 9. Short form - specify location ID, secret ID,
version and use default project
sm@locations/<location-id>/<secret-id>/<version>

# 10. Shortest form - specify location ID, secret ID, and use default project and latest version
sm@locations/<location-id>/<secret-id>
----

You can use this syntax in the following places:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
/*
* Copyright 2025 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.cloud.spring.autoconfigure.secretmanager;

import com.google.api.gax.core.CredentialsProvider;
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
import com.google.cloud.spring.core.UserAgentHeaderProvider;
import com.google.cloud.spring.secretmanager.SecretManagerServiceClientFactory;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.springframework.stereotype.Component;
import org.springframework.util.ObjectUtils;

/**
* A default implementation of the {@link SecretManagerServiceClientFactory} interface.
*
* <p>This factory provides a caching layer for {@link SecretManagerServiceClient} instances.
* Clients are created using the provided {@link CredentialsProvider} and a {@link
* UserAgentHeaderProvider} that adds the Spring Cloud GCP agent header to the client.
*
*/
@Component
public class DefaultSecretManagerServiceClientFactory implements SecretManagerServiceClientFactory {

private final CredentialsProvider credentialsProvider;
private final Map<String, SecretManagerServiceClient> clientCache = new ConcurrentHashMap<>();

public DefaultSecretManagerServiceClientFactory(CredentialsProvider credentialsProvider) {
this.credentialsProvider = credentialsProvider;
}

@Override
public SecretManagerServiceClient getClient(String location) {
if (ObjectUtils.isEmpty(location)) {
return getClient();
}
return clientCache.computeIfAbsent(location, loc -> {
try {
String endpoint = String.format("secretmanager.%s.rep.googleapis.com:443", loc);
SecretManagerServiceSettings settings = SecretManagerServiceSettings.newBuilder()
.setCredentialsProvider(credentialsProvider)
.setHeaderProvider(new UserAgentHeaderProvider(SecretManagerConfigDataLoader.class))
.setEndpoint(endpoint).build();
return SecretManagerServiceClient.create(settings);
} catch (IOException e) {
throw new RuntimeException(
"Failed to create SecretManagerServiceClient for location: " + loc, e);
}
});
}

@Override
public SecretManagerServiceClient getClient() {
return clientCache.computeIfAbsent("", loc -> {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you just re-use getClient(String) with some modification to avoid code duplication?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated

try {
SecretManagerServiceSettings settings = SecretManagerServiceSettings.newBuilder()
.setCredentialsProvider(credentialsProvider)
.setHeaderProvider(new UserAgentHeaderProvider(SecretManagerConfigDataLoader.class)
).build();

return SecretManagerServiceClient.create(settings);
} catch (IOException e) {
throw new RuntimeException("Failed to create SecretManagerServiceClient", e);
}
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,17 @@
package com.google.cloud.spring.autoconfigure.secretmanager;

import com.google.api.gax.core.CredentialsProvider;
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
import com.google.cloud.spring.core.GcpProjectIdProvider;
import com.google.cloud.spring.core.UserAgentHeaderProvider;
import com.google.cloud.spring.secretmanager.SecretManagerServiceClientFactory;
import com.google.cloud.spring.secretmanager.SecretManagerTemplate;
import java.io.IOException;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;


/**
* Autoconfiguration for GCP Secret Manager.
*
Expand Down Expand Up @@ -60,22 +58,14 @@ public GcpSecretManagerAutoConfiguration(

@Bean
@ConditionalOnMissingBean
public SecretManagerServiceClient secretManagerClient()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Customers may already be using this one or defining a custom secretManagerClient bean and expecting it to be used. Consider leaving this bean and passing it down into clientFactory. I understand that it might not be used for apps that only use regional secrets, but it would be more backwards-compatible.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated. Also checked sample application by using the SecretManagerServiceClient with the SecretManagerTemplate.

throws IOException {
SecretManagerServiceSettings settings =
SecretManagerServiceSettings.newBuilder()
.setCredentialsProvider(this.credentialsProvider)
.setHeaderProvider(
new UserAgentHeaderProvider(GcpSecretManagerAutoConfiguration.class))
.build();

return SecretManagerServiceClient.create(settings);
public SecretManagerServiceClientFactory clientFactory() {
return new DefaultSecretManagerServiceClientFactory(this.credentialsProvider);
}

@Bean
@ConditionalOnMissingBean
public SecretManagerTemplate secretManagerTemplate(SecretManagerServiceClient client) {
return new SecretManagerTemplate(client, this.gcpProjectIdProvider)
public SecretManagerTemplate secretManagerTemplate(SecretManagerServiceClientFactory clientFactory) {
return new SecretManagerTemplate(clientFactory, this.gcpProjectIdProvider)
.setAllowDefaultSecretValue(this.properties.isAllowDefaultSecret());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ public ConfigData load(
GcpProjectIdProvider projectIdProvider = context.getBootstrapContext()
.get(GcpProjectIdProvider.class);

return new ConfigData(Collections.singleton(new SecretManagerPropertySource(
"spring-cloud-gcp-secret-manager", secretManagerTemplate, projectIdProvider)));
SecretManagerPropertySource secretManagerPropertySource = new SecretManagerPropertySource(
"spring-cloud-gcp-secret-manager", secretManagerTemplate, projectIdProvider);
return new ConfigData(Collections.singleton(secretManagerPropertySource));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,17 @@
import static com.google.cloud.spring.secretmanager.SecretManagerSyntaxUtils.getMatchedPrefixes;
import static com.google.cloud.spring.secretmanager.SecretManagerSyntaxUtils.warnIfUsingDeprecatedSyntax;

import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
import autovalue.shaded.com.google.common.annotations.VisibleForTesting;
import com.google.api.gax.core.CredentialsProvider;
import com.google.cloud.spring.core.DefaultCredentialsProvider;
import com.google.cloud.spring.core.DefaultGcpProjectIdProvider;
import com.google.cloud.spring.core.GcpProjectIdProvider;
import com.google.cloud.spring.core.UserAgentHeaderProvider;
import com.google.cloud.spring.secretmanager.SecretManagerServiceClientFactory;
import com.google.cloud.spring.secretmanager.SecretManagerTemplate;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import org.apache.arrow.util.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
Expand All @@ -44,42 +43,42 @@
public class SecretManagerConfigDataLocationResolver implements
ConfigDataLocationResolver<SecretManagerConfigDataResource> {

private static final Logger logger = LoggerFactory.getLogger(SecretManagerConfigDataLocationResolver.class);
private static final Logger logger =
LoggerFactory.getLogger(SecretManagerConfigDataLocationResolver.class);

/**
* A static client to avoid creating another client after refreshing.
* A static client factory to avoid creating another client after refreshing.
*/
private static SecretManagerServiceClient secretManagerServiceClient;
private static SecretManagerServiceClientFactory secretManagerServiceClientFactory;

@Override
public boolean isResolvable(ConfigDataLocationResolverContext context,
ConfigDataLocation location) {
public boolean isResolvable(
ConfigDataLocationResolverContext context, ConfigDataLocation location) {
Optional<String> matchedPrefix = getMatchedPrefixes(location::hasPrefix);
warnIfUsingDeprecatedSyntax(logger, matchedPrefix.orElse(""));
return matchedPrefix.isPresent();
}

@Override
public List<SecretManagerConfigDataResource> resolve(ConfigDataLocationResolverContext context,
ConfigDataLocation location)
public List<SecretManagerConfigDataResource> resolve(
ConfigDataLocationResolverContext context, ConfigDataLocation location)
throws ConfigDataLocationNotFoundException, ConfigDataResourceNotFoundException {
registerSecretManagerBeans(context);

return Collections.singletonList(
new SecretManagerConfigDataResource(location));
return Collections.singletonList(new SecretManagerConfigDataResource(location));
}

private static void registerSecretManagerBeans(ConfigDataLocationResolverContext context) {
// Register the Secret Manager properties.
registerBean(
context, GcpSecretManagerProperties.class, getSecretManagerProperties(context));
// Register the Secret Manager client.
registerBean(context, GcpSecretManagerProperties.class, getSecretManagerProperties(context));
// Register the CredentialsProvider.
registerBean(context, CredentialsProvider.class, getCredentialsProvider(context));
// Register the Secret Manager client factory.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@abheda-crest Sorry this was missed in earlier reviews.
Merging this PR caused IT tests failures across modules. I have reverted this change. More context here

I believe the root cause is here: This SecretManagerConfigDataLocationResolver is used regardless if secretmanager dependency is present. When not present, for example if you run the vision sample app, this code createSecretManagerServiceClientFactory() calls DefaultSecretManagerServiceClientFactory that eventually gives "NoClassDefFoundError" for "com/google/cloud/spring/secretmanager/SecretManagerServiceClientFactory"

Note that secretmanager is optional dependency for autoconfig. You will need to guard this logic to only run when secretmanager is present.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have fixed the issue

registerAndPromoteBean(
context,
SecretManagerServiceClient.class,
// lazy register the client solely for unit test.
BootstrapRegistry.InstanceSupplier.from(() -> createSecretManagerClient(context)));
// Register the GCP Project ID provider.
SecretManagerServiceClientFactory.class,
BootstrapRegistry.InstanceSupplier.from(
() -> createSecretManagerServiceClientFactory(context)));
registerAndPromoteBean(
context,
GcpProjectIdProvider.class,
Expand All @@ -98,44 +97,43 @@ private static GcpSecretManagerProperties getSecretManagerProperties(
.orElse(new GcpSecretManagerProperties());
}

private static CredentialsProvider getCredentialsProvider(
ConfigDataLocationResolverContext context) {
try {
GcpSecretManagerProperties properties =
context.getBootstrapContext().get(GcpSecretManagerProperties.class);
return context.getBinder()
.bind(GcpSecretManagerProperties.PREFIX, CredentialsProvider.class)
.orElse(new DefaultCredentialsProvider(properties));
} catch (IOException e) {
throw new RuntimeException(
"Failed to create the Secret Manager Client Factory for ConfigData loading.", e);
}
}

private static GcpProjectIdProvider createProjectIdProvider(
ConfigDataLocationResolverContext context) {
GcpSecretManagerProperties properties = context.getBootstrapContext()
.get(GcpSecretManagerProperties.class);
return properties.getProjectId() != null
? properties::getProjectId : new DefaultGcpProjectIdProvider();
? properties::getProjectId
: new DefaultGcpProjectIdProvider();
}

@VisibleForTesting
static synchronized SecretManagerServiceClient createSecretManagerClient(
static synchronized SecretManagerServiceClientFactory createSecretManagerServiceClientFactory(
ConfigDataLocationResolverContext context) {
if (secretManagerServiceClient != null && !secretManagerServiceClient.isTerminated()) {
return secretManagerServiceClient;
}

try {
GcpSecretManagerProperties properties = context.getBootstrapContext()
.get(GcpSecretManagerProperties.class);
DefaultCredentialsProvider credentialsProvider =
new DefaultCredentialsProvider(properties);
SecretManagerServiceSettings settings = SecretManagerServiceSettings.newBuilder()
.setCredentialsProvider(credentialsProvider)
.setHeaderProvider(
new UserAgentHeaderProvider(SecretManagerConfigDataLoader.class))
.build();
secretManagerServiceClient = SecretManagerServiceClient.create(settings);

return secretManagerServiceClient;
} catch (IOException e) {
throw new RuntimeException(
"Failed to create the Secret Manager Client for ConfigData loading.", e);
if (secretManagerServiceClientFactory != null) {
return secretManagerServiceClientFactory;
}
return new DefaultSecretManagerServiceClientFactory(
context.getBootstrapContext().get(CredentialsProvider.class));
}

private static SecretManagerTemplate createSecretManagerTemplate(
ConfigDataLocationResolverContext context) {
SecretManagerServiceClient client = context.getBootstrapContext()
.get(SecretManagerServiceClient.class);
SecretManagerServiceClientFactory client = context.getBootstrapContext()
.get(SecretManagerServiceClientFactory.class);
GcpProjectIdProvider projectIdProvider = context.getBootstrapContext()
.get(GcpProjectIdProvider.class);
GcpSecretManagerProperties properties = context.getBootstrapContext()
Expand All @@ -153,7 +151,8 @@ private static SecretManagerTemplate createSecretManagerTemplate(
*/
private static <T> void registerBean(
ConfigDataLocationResolverContext context, Class<T> type, T instance) {
context.getBootstrapContext()
context
.getBootstrapContext()
.registerIfAbsent(type, BootstrapRegistry.InstanceSupplier.of(instance));
}

Expand All @@ -162,7 +161,8 @@ private static <T> void registerBean(
* application context.
*/
private static <T> void registerAndPromoteBean(
ConfigDataLocationResolverContext context, Class<T> type,
ConfigDataLocationResolverContext context,
Class<T> type,
BootstrapRegistry.InstanceSupplier<T> supplier) {
context.getBootstrapContext().registerIfAbsent(type, supplier);
context.getBootstrapContext().addCloseListener(event -> {
Expand All @@ -176,7 +176,8 @@ private static <T> void registerAndPromoteBean(
}

@VisibleForTesting
static void setSecretManagerServiceClient(SecretManagerServiceClient client) {
secretManagerServiceClient = client;
static void setSecretManagerServiceClientFactory(
SecretManagerServiceClientFactory clientFactory) {
secretManagerServiceClientFactory = clientFactory;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,12 @@
package com.google.cloud.spring.autoconfigure.secretmanager;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;

import com.google.api.gax.core.CredentialsProvider;
import com.google.auth.Credentials;
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
import com.google.cloud.spring.autoconfigure.TestUtils;
import com.google.cloud.spring.autoconfigure.core.GcpContextAutoConfiguration;
import com.google.cloud.spring.secretmanager.SecretManagerServiceClientFactory;
import com.google.cloud.spring.secretmanager.SecretManagerTemplate;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -61,9 +60,9 @@ void testProjectIdWithGcpProperties() {
}

@Test
void testSecretManagerServiceClientExists() {
void testSecretManagerServiceClientFactoryExists() {
contextRunner.run(
ctx -> assertThat(ctx.getBean(SecretManagerServiceClient.class))
ctx -> assertThat(ctx.getBean(SecretManagerServiceClientFactory.class))
.isNotNull());
}

Expand Down
Loading