From 929844ea2bcc91186d862d161f3f8f1e24d88de2 Mon Sep 17 00:00:00 2001 From: mpeddada1 Date: Tue, 27 Aug 2024 20:32:09 +0000 Subject: [PATCH 01/12] feat: add properties to set universe domain and endpoint in bigquery --- .../GcpBigQueryAutoConfiguration.java | 29 +++++--- .../bigquery/GcpBigQueryProperties.java | 19 ++++++ .../GcpBigQueryAutoConfigurationTests.java | 68 +++++++++++++++++++ 3 files changed, 108 insertions(+), 8 deletions(-) diff --git a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java index 88111ffe8e..ba80f573e7 100644 --- a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java +++ b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java @@ -57,6 +57,9 @@ public class GcpBigQueryAutoConfiguration { private int threadPoolSize; + private String universeDomain; + private String endpoint; + GcpBigQueryAutoConfiguration( GcpBigQueryProperties gcpBigQueryProperties, GcpProjectIdProvider projectIdProvider, @@ -78,6 +81,9 @@ public class GcpBigQueryAutoConfiguration { this.jsonWriterBatchSize = gcpBigQueryProperties.getJsonWriterBatchSize(); this.threadPoolSize = getThreadPoolSize(gcpBigQueryProperties.getThreadPoolSize()); + + this.universeDomain = gcpBigQueryProperties.getUniverseDomain(); + this.endpoint = gcpBigQueryProperties.getEndpoint(); } /** @@ -97,25 +103,32 @@ private int getThreadPoolSize(int threadPoolSize) { @Bean @ConditionalOnMissingBean public BigQuery bigQuery() throws IOException { - BigQueryOptions bigQueryOptions = + BigQueryOptions.Builder bigQueryOptionsBuilder = BigQueryOptions.newBuilder() .setProjectId(this.projectId) .setCredentials(this.credentialsProvider.getCredentials()) - .setHeaderProvider(new UserAgentHeaderProvider(GcpBigQueryAutoConfiguration.class)) - .build(); - return bigQueryOptions.getService(); + .setHeaderProvider(new UserAgentHeaderProvider(GcpBigQueryAutoConfiguration.class)); + if (this.universeDomain != null) { + bigQueryOptionsBuilder.setUniverseDomain(this.universeDomain); + } + return bigQueryOptionsBuilder.build().getService(); } @Bean @ConditionalOnMissingBean public BigQueryWriteClient bigQueryWriteClient() throws IOException { - BigQueryWriteSettings bigQueryWriteSettings = + BigQueryWriteSettings.Builder bigQueryWriteSettingsBuilder = BigQueryWriteSettings.newBuilder() .setCredentialsProvider(this.credentialsProvider) .setQuotaProjectId(this.projectId) - .setHeaderProvider(new UserAgentHeaderProvider(GcpBigQueryAutoConfiguration.class)) - .build(); - return BigQueryWriteClient.create(bigQueryWriteSettings); + .setHeaderProvider(new UserAgentHeaderProvider(GcpBigQueryAutoConfiguration.class)); + if (this.universeDomain != null) { + bigQueryWriteSettingsBuilder.setUniverseDomain(this.universeDomain); + } + if (this.endpoint != null) { + bigQueryWriteSettingsBuilder.setEndpoint(this.endpoint); + } + return BigQueryWriteClient.create(bigQueryWriteSettingsBuilder.build()); } @Bean diff --git a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java index 1bac1dd742..dde500bad8 100644 --- a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java +++ b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java @@ -45,6 +45,9 @@ public class GcpBigQueryProperties implements CredentialsSupplier { /** The size of thread pool of ThreadPoolTaskScheduler used by GcpBigQueryAutoConfiguration */ private int threadPoolSize; + private String universeDomain; + private String endpoint; + public int getJsonWriterBatchSize() { return jsonWriterBatchSize; } @@ -80,4 +83,20 @@ public String getDatasetName() { public void setDatasetName(String datasetName) { this.datasetName = datasetName; } + + public String getUniverseDomain() { + return universeDomain; + } + + public void setUniverseDomain(String universeDomain) { + this.universeDomain = universeDomain; + } + + public String getEndpoint() { + return endpoint; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } } diff --git a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java index 07d62ce5f5..c26ec4ba9a 100644 --- a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java +++ b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java @@ -23,6 +23,7 @@ import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; import com.google.cloud.spring.autoconfigure.core.GcpContextAutoConfiguration; import com.google.cloud.spring.bigquery.core.BigQueryTemplate; import org.junit.jupiter.api.Test; @@ -60,6 +61,73 @@ void testSettingBigQueryOptions() { }); } + @Test + void testBigQuery_universeDomain() { + this.contextRunner + .withPropertyValues("spring.cloud.gcp.bigquery.universe-domain=myUniverseDomain") + .run( + ctx -> { + BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); + assertThat(options.getUniverseDomain()).isNull(); + }); + } + + @Test + void testBigQuery_noUniverseDomainSet_useClientDefault() { + this.contextRunner.run( + ctx -> { + BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); + assertThat(options.getUniverseDomain()).isEqualTo("googleapis.com"); + }); + } + + @Test + void testBigQueryWrite_universeDomain() { + this.contextRunner + .withPropertyValues("spring.cloud.gcp.bigquery.universe-domain=myUniverseDomain") + .run( + ctx -> { + BigQueryWriteClient writeClient = ctx.getBean(BigQueryWriteClient.class); + assertThat(writeClient.getSettings().getUniverseDomain()) + .isEqualTo("myUniverseDomain"); + }); + } + + @Test + void testBigQueryWrite_endpoint() { + this.contextRunner + .withPropertyValues("spring.cloud.gcp.bigquery.endpoint=kms.example.com:123") + .run( + ctx -> { + BigQueryWriteClient client = ctx.getBean(BigQueryWriteClient.class); + assertThat(client.getSettings().getEndpoint()).isEqualTo("kms.example.com:123"); + }); + } + + @Test + void testBigQueryWrite_bothUniverseDomainAndEndpointSet() { + this.contextRunner + .withPropertyValues("spring.cloud.gcp.bigquery.universe-domain=myUniverseDomain") + .withPropertyValues("spring.cloud.gcp.bigquery.endpoint=kms.example.com:123") + .run( + ctx -> { + BigQueryWriteClient client = ctx.getBean(BigQueryWriteClient.class); + assertThat(client.getSettings().getUniverseDomain()).isEqualTo("myUniverseDomain"); + assertThat(client.getSettings().getEndpoint()).isEqualTo("kms.example.com:123"); + }); + } + + @Test + void testBigQueryWrite_noUniverseDomainOrEndpointSet_useClientDefault() { + this.contextRunner.run( + ctx -> { + BigQueryWriteClient client = ctx.getBean(BigQueryWriteClient.class); + assertThat(client.getSettings().getUniverseDomain()).isEqualTo("googleapis.com"); + assertThat(client.getSettings().getEndpoint()) + .isEqualTo("bigquerystorage.googleapis.com:443"); + }); + } + /** Spring Boot config for tests. */ @AutoConfigurationPackage static class TestConfiguration { From bf6887b079161226fa77d10d5023ceb60004a2a1 Mon Sep 17 00:00:00 2001 From: mpeddada1 Date: Tue, 27 Aug 2024 20:38:11 +0000 Subject: [PATCH 02/12] fix test --- .../bigquery/GcpBigQueryAutoConfigurationTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java index c26ec4ba9a..0695c586d7 100644 --- a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java +++ b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java @@ -68,7 +68,7 @@ void testBigQuery_universeDomain() { .run( ctx -> { BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); - assertThat(options.getUniverseDomain()).isNull(); + assertThat(options.getUniverseDomain()).isEqualTo("myUniverseDomain"); }); } @@ -77,7 +77,7 @@ void testBigQuery_noUniverseDomainSet_useClientDefault() { this.contextRunner.run( ctx -> { BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); - assertThat(options.getUniverseDomain()).isEqualTo("googleapis.com"); + assertThat(options.getUniverseDomain()).isNull(); }); } From e86aa9e9297caf2007e4abd032767c56f63f0a5d Mon Sep 17 00:00:00 2001 From: mpeddada1 Date: Mon, 16 Sep 2024 16:20:16 +0000 Subject: [PATCH 03/12] introduce host configuration for BigqueryOptions --- .../GcpBigQueryAutoConfiguration.java | 14 +++++-- .../bigquery/GcpBigQueryProperties.java | 24 ++++++++--- .../GcpBigQueryAutoConfigurationTests.java | 40 ++++++++++++++++--- 3 files changed, 64 insertions(+), 14 deletions(-) diff --git a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java index ba80f573e7..51bd3a9a8a 100644 --- a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java +++ b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java @@ -58,7 +58,9 @@ public class GcpBigQueryAutoConfiguration { private int threadPoolSize; private String universeDomain; - private String endpoint; + private String jsonWriterEndpoint; + + private String host; GcpBigQueryAutoConfiguration( GcpBigQueryProperties gcpBigQueryProperties, @@ -83,7 +85,8 @@ public class GcpBigQueryAutoConfiguration { this.threadPoolSize = getThreadPoolSize(gcpBigQueryProperties.getThreadPoolSize()); this.universeDomain = gcpBigQueryProperties.getUniverseDomain(); - this.endpoint = gcpBigQueryProperties.getEndpoint(); + this.jsonWriterEndpoint = gcpBigQueryProperties.getJsonWriterEndpoint(); + this.host = gcpBigQueryProperties.getHost(); } /** @@ -111,6 +114,9 @@ public BigQuery bigQuery() throws IOException { if (this.universeDomain != null) { bigQueryOptionsBuilder.setUniverseDomain(this.universeDomain); } + if (this.host != null) { + bigQueryOptionsBuilder.setHost(this.host); + } return bigQueryOptionsBuilder.build().getService(); } @@ -125,8 +131,8 @@ public BigQueryWriteClient bigQueryWriteClient() throws IOException { if (this.universeDomain != null) { bigQueryWriteSettingsBuilder.setUniverseDomain(this.universeDomain); } - if (this.endpoint != null) { - bigQueryWriteSettingsBuilder.setEndpoint(this.endpoint); + if (this.jsonWriterEndpoint != null) { + bigQueryWriteSettingsBuilder.setEndpoint(this.jsonWriterEndpoint); } return BigQueryWriteClient.create(bigQueryWriteSettingsBuilder.build()); } diff --git a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java index dde500bad8..d487f1dffa 100644 --- a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java +++ b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java @@ -46,7 +46,12 @@ public class GcpBigQueryProperties implements CredentialsSupplier { private int threadPoolSize; private String universeDomain; - private String endpoint; + + /** Endpoint that will be used by BigQueryJsonDataWriter. */ + private String jsonWriterEndpoint; + + /** Host (endpoint without the port) that will be used by BigQueryOptions. */ + private String host; public int getJsonWriterBatchSize() { return jsonWriterBatchSize; @@ -92,11 +97,20 @@ public void setUniverseDomain(String universeDomain) { this.universeDomain = universeDomain; } - public String getEndpoint() { - return endpoint; + public String getJsonWriterEndpoint() { + return jsonWriterEndpoint; + } + + public void setJsonWriterEndpoint(String jsonWriterEndpoint) { + this.jsonWriterEndpoint = jsonWriterEndpoint; } - public void setEndpoint(String endpoint) { - this.endpoint = endpoint; + public String getHost() { + return host; } + + public void setHost(String host) { + this.host = host; + } + } diff --git a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java index 0695c586d7..d00dfce537 100644 --- a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java +++ b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java @@ -69,18 +69,44 @@ void testBigQuery_universeDomain() { ctx -> { BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); assertThat(options.getUniverseDomain()).isEqualTo("myUniverseDomain"); + assertThat(options.getHost()).isEqualTo("https://www.googleapis.com"); }); } @Test - void testBigQuery_noUniverseDomainSet_useClientDefault() { + void testBigQuery_noUniverseDomainAndHostSet_useClientDefault() { this.contextRunner.run( ctx -> { BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); assertThat(options.getUniverseDomain()).isNull(); + assertThat(options.getHost()).isEqualTo("https://www.googleapis.com"); }); } + @Test + void testBigQuery_host() { + this.contextRunner + .withPropertyValues("spring.cloud.gcp.bigquery.host=bigquery.example.com") + .run( + ctx -> { + BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); + assertThat(options.getHost()).isEqualTo("bigquery.example.com"); + }); + } + + @Test + void testBigQuery_bothHostAndUniverseDomainSet() { + this.contextRunner + .withPropertyValues("spring.cloud.gcp.bigquery.host=bigquery.example.com") + .withPropertyValues("spring.cloud.gcp.bigquery.universe-domain=myUniverseDomain") + .run( + ctx -> { + BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); + assertThat(options.getHost()).isEqualTo("bigquery.example.com"); + assertThat(options.getUniverseDomain()).isEqualTo("myUniverseDomain"); + }); + } + @Test void testBigQueryWrite_universeDomain() { this.contextRunner @@ -96,11 +122,13 @@ void testBigQueryWrite_universeDomain() { @Test void testBigQueryWrite_endpoint() { this.contextRunner - .withPropertyValues("spring.cloud.gcp.bigquery.endpoint=kms.example.com:123") + .withPropertyValues( + "spring.cloud.gcp.bigquery.jsonWriterEndpoint=bigquerystorage.example.com:123") .run( ctx -> { BigQueryWriteClient client = ctx.getBean(BigQueryWriteClient.class); - assertThat(client.getSettings().getEndpoint()).isEqualTo("kms.example.com:123"); + assertThat(client.getSettings().getEndpoint()) + .isEqualTo("bigquerystorage.example.com:123"); }); } @@ -108,12 +136,14 @@ void testBigQueryWrite_endpoint() { void testBigQueryWrite_bothUniverseDomainAndEndpointSet() { this.contextRunner .withPropertyValues("spring.cloud.gcp.bigquery.universe-domain=myUniverseDomain") - .withPropertyValues("spring.cloud.gcp.bigquery.endpoint=kms.example.com:123") + .withPropertyValues( + "spring.cloud.gcp.bigquery.jsonWriterEndpoint=bigquerystorage.example.com:123") .run( ctx -> { BigQueryWriteClient client = ctx.getBean(BigQueryWriteClient.class); assertThat(client.getSettings().getUniverseDomain()).isEqualTo("myUniverseDomain"); - assertThat(client.getSettings().getEndpoint()).isEqualTo("kms.example.com:123"); + assertThat(client.getSettings().getEndpoint()) + .isEqualTo("bigquerystorage.example.com:123"); }); } From 7e1f734906b9adfca23ca0940e1dd246db17b86d Mon Sep 17 00:00:00 2001 From: mpeddada1 Date: Mon, 16 Sep 2024 16:31:04 +0000 Subject: [PATCH 04/12] verify with getResolvedApiaryHost --- .../bigquery/GcpBigQueryAutoConfigurationTests.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java index d00dfce537..6e9ab3afd1 100644 --- a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java +++ b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java @@ -69,7 +69,8 @@ void testBigQuery_universeDomain() { ctx -> { BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); assertThat(options.getUniverseDomain()).isEqualTo("myUniverseDomain"); - assertThat(options.getHost()).isEqualTo("https://www.googleapis.com"); + assertThat(options.getResolvedApiaryHost("bigquery")) + .isEqualTo("https://bigquery.myUniverseDomain/"); }); } @@ -79,7 +80,8 @@ void testBigQuery_noUniverseDomainAndHostSet_useClientDefault() { ctx -> { BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); assertThat(options.getUniverseDomain()).isNull(); - assertThat(options.getHost()).isEqualTo("https://www.googleapis.com"); + assertThat(options.getResolvedApiaryHost("bigquery")) + .isEqualTo("https://bigquery.googleapis.com/"); }); } @@ -90,7 +92,8 @@ void testBigQuery_host() { .run( ctx -> { BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); - assertThat(options.getHost()).isEqualTo("bigquery.example.com"); + assertThat(options.getResolvedApiaryHost("bigquery")) + .isEqualTo("bigquery.example.com"); }); } @@ -102,7 +105,8 @@ void testBigQuery_bothHostAndUniverseDomainSet() { .run( ctx -> { BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); - assertThat(options.getHost()).isEqualTo("bigquery.example.com"); + assertThat(options.getResolvedApiaryHost("bigquery")) + .isEqualTo("bigquery.example.com"); assertThat(options.getUniverseDomain()).isEqualTo("myUniverseDomain"); }); } From 9419b7bf4c1d7432b9c286934484d4252438a905 Mon Sep 17 00:00:00 2001 From: mpeddada1 Date: Mon, 16 Sep 2024 16:39:43 +0000 Subject: [PATCH 05/12] add documentation for properties --- .../autoconfigure/bigquery/GcpBigQueryProperties.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java index d487f1dffa..7fc14b8ba2 100644 --- a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java +++ b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java @@ -47,10 +47,15 @@ public class GcpBigQueryProperties implements CredentialsSupplier { private String universeDomain; - /** Endpoint that will be used by BigQueryJsonDataWriter. */ + /** + * Endpoint (formatted as `${service}.${universeDomain}:${port}`) that will be used by + * BigQueryJsonDataWriter. + */ private String jsonWriterEndpoint; - /** Host (endpoint without the port) that will be used by BigQueryOptions. */ + /** + * Host (formatted as `https://{service}.{universeDomain}`) that will be used by BigQueryOptions. + */ private String host; public int getJsonWriterBatchSize() { From 8364de19272b486586e99647776312bb6f50ff55 Mon Sep 17 00:00:00 2001 From: mpeddada1 Date: Mon, 23 Sep 2024 17:56:48 +0000 Subject: [PATCH 06/12] add global properties for endpoint and universeDomaine; resolve to host --- .../GcpBigQueryAutoConfiguration.java | 33 +++++++++++++------ .../bigquery/GcpBigQueryProperties.java | 26 ++++----------- .../GcpBigQueryAutoConfigurationTests.java | 12 +++---- 3 files changed, 35 insertions(+), 36 deletions(-) diff --git a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java index 51bd3a9a8a..2b4d36c122 100644 --- a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java +++ b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java @@ -26,7 +26,10 @@ import com.google.cloud.spring.core.DefaultCredentialsProvider; import com.google.cloud.spring.core.GcpProjectIdProvider; import com.google.cloud.spring.core.UserAgentHeaderProvider; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.HashMap; import java.util.Map; import org.springframework.beans.factory.annotation.Qualifier; @@ -58,9 +61,7 @@ public class GcpBigQueryAutoConfiguration { private int threadPoolSize; private String universeDomain; - private String jsonWriterEndpoint; - - private String host; + private String endpoint; GcpBigQueryAutoConfiguration( GcpBigQueryProperties gcpBigQueryProperties, @@ -85,8 +86,8 @@ public class GcpBigQueryAutoConfiguration { this.threadPoolSize = getThreadPoolSize(gcpBigQueryProperties.getThreadPoolSize()); this.universeDomain = gcpBigQueryProperties.getUniverseDomain(); - this.jsonWriterEndpoint = gcpBigQueryProperties.getJsonWriterEndpoint(); - this.host = gcpBigQueryProperties.getHost(); + + this.endpoint = gcpBigQueryProperties.getEndpoint(); } /** @@ -105,7 +106,7 @@ private int getThreadPoolSize(int threadPoolSize) { @Bean @ConditionalOnMissingBean - public BigQuery bigQuery() throws IOException { + public BigQuery bigQuery() throws IOException, URISyntaxException { BigQueryOptions.Builder bigQueryOptionsBuilder = BigQueryOptions.newBuilder() .setProjectId(this.projectId) @@ -114,8 +115,8 @@ public BigQuery bigQuery() throws IOException { if (this.universeDomain != null) { bigQueryOptionsBuilder.setUniverseDomain(this.universeDomain); } - if (this.host != null) { - bigQueryOptionsBuilder.setHost(this.host); + if (this.endpoint != null) { + bigQueryOptionsBuilder.setHost(resolveToHost(this.endpoint)); } return bigQueryOptionsBuilder.build().getService(); } @@ -131,8 +132,8 @@ public BigQueryWriteClient bigQueryWriteClient() throws IOException { if (this.universeDomain != null) { bigQueryWriteSettingsBuilder.setUniverseDomain(this.universeDomain); } - if (this.jsonWriterEndpoint != null) { - bigQueryWriteSettingsBuilder.setEndpoint(this.jsonWriterEndpoint); + if (this.endpoint != null) { + bigQueryWriteSettingsBuilder.setEndpoint(this.endpoint); } return BigQueryWriteClient.create(bigQueryWriteSettingsBuilder.build()); } @@ -160,4 +161,16 @@ public BigQueryTemplate bigQueryTemplate( return new BigQueryTemplate( bigQuery, bigQueryWriteClient, bqInitSettings, bigQueryThreadPoolTaskScheduler); } + + private String resolveToHost(String endpoint) throws URISyntaxException { + if (!endpoint.contains("://")) { + endpoint = "https://" + endpoint; + } + URI uri = new URI(endpoint); + + // Construct the new URL with https and no port + String newUrl = new URI("https", uri.getUserInfo(), uri.getHost(), -1, + uri.getPath(), uri.getQuery(), uri.getFragment()).toString(); + return newUrl; + } } diff --git a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java index 7fc14b8ba2..f555c4ae2c 100644 --- a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java +++ b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryProperties.java @@ -48,15 +48,9 @@ public class GcpBigQueryProperties implements CredentialsSupplier { private String universeDomain; /** - * Endpoint (formatted as `${service}.${universeDomain}:${port}`) that will be used by - * BigQueryJsonDataWriter. + * Endpoint (formatted as `{service}.{universeDomain}:${port}`) */ - private String jsonWriterEndpoint; - - /** - * Host (formatted as `https://{service}.{universeDomain}`) that will be used by BigQueryOptions. - */ - private String host; + private String endpoint; public int getJsonWriterBatchSize() { return jsonWriterBatchSize; @@ -102,20 +96,12 @@ public void setUniverseDomain(String universeDomain) { this.universeDomain = universeDomain; } - public String getJsonWriterEndpoint() { - return jsonWriterEndpoint; - } - - public void setJsonWriterEndpoint(String jsonWriterEndpoint) { - this.jsonWriterEndpoint = jsonWriterEndpoint; - } - - public String getHost() { - return host; + public String getEndpoint() { + return endpoint; } - public void setHost(String host) { - this.host = host; + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; } } diff --git a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java index 6e9ab3afd1..f2aa16b534 100644 --- a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java +++ b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java @@ -88,25 +88,25 @@ void testBigQuery_noUniverseDomainAndHostSet_useClientDefault() { @Test void testBigQuery_host() { this.contextRunner - .withPropertyValues("spring.cloud.gcp.bigquery.host=bigquery.example.com") + .withPropertyValues("spring.cloud.gcp.bigquery.endpoint=bigquery.example.com:443") .run( ctx -> { BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); assertThat(options.getResolvedApiaryHost("bigquery")) - .isEqualTo("bigquery.example.com"); + .isEqualTo("https://bigquery.example.com"); }); } @Test void testBigQuery_bothHostAndUniverseDomainSet() { this.contextRunner - .withPropertyValues("spring.cloud.gcp.bigquery.host=bigquery.example.com") + .withPropertyValues("spring.cloud.gcp.bigquery.endpoint=bigquery.example.com:123") .withPropertyValues("spring.cloud.gcp.bigquery.universe-domain=myUniverseDomain") .run( ctx -> { BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); assertThat(options.getResolvedApiaryHost("bigquery")) - .isEqualTo("bigquery.example.com"); + .isEqualTo("https://bigquery.example.com"); assertThat(options.getUniverseDomain()).isEqualTo("myUniverseDomain"); }); } @@ -127,7 +127,7 @@ void testBigQueryWrite_universeDomain() { void testBigQueryWrite_endpoint() { this.contextRunner .withPropertyValues( - "spring.cloud.gcp.bigquery.jsonWriterEndpoint=bigquerystorage.example.com:123") + "spring.cloud.gcp.bigquery.endpoint=bigquerystorage.example.com:123") .run( ctx -> { BigQueryWriteClient client = ctx.getBean(BigQueryWriteClient.class); @@ -141,7 +141,7 @@ void testBigQueryWrite_bothUniverseDomainAndEndpointSet() { this.contextRunner .withPropertyValues("spring.cloud.gcp.bigquery.universe-domain=myUniverseDomain") .withPropertyValues( - "spring.cloud.gcp.bigquery.jsonWriterEndpoint=bigquerystorage.example.com:123") + "spring.cloud.gcp.bigquery.endpoint=bigquerystorage.example.com:123") .run( ctx -> { BigQueryWriteClient client = ctx.getBean(BigQueryWriteClient.class); From a8b8ad267cf72bf8f0500dae17d03fe24c99e8ab Mon Sep 17 00:00:00 2001 From: mpeddada1 Date: Mon, 23 Sep 2024 18:04:50 +0000 Subject: [PATCH 07/12] fix checkstyle --- .../autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java | 1 - 1 file changed, 1 deletion(-) diff --git a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java index 2b4d36c122..e298af078b 100644 --- a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java +++ b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java @@ -26,7 +26,6 @@ import com.google.cloud.spring.core.DefaultCredentialsProvider; import com.google.cloud.spring.core.GcpProjectIdProvider; import com.google.cloud.spring.core.UserAgentHeaderProvider; -import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; From 8e7b4bf77b3af63b2b94812f8d1c29a707229f17 Mon Sep 17 00:00:00 2001 From: mpeddada1 Date: Mon, 23 Sep 2024 18:08:07 +0000 Subject: [PATCH 08/12] fix comment --- .../autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java index e298af078b..e303aaa45c 100644 --- a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java +++ b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java @@ -167,7 +167,7 @@ private String resolveToHost(String endpoint) throws URISyntaxException { } URI uri = new URI(endpoint); - // Construct the new URL with https and no port + // Construct the new URL with https:// and no port String newUrl = new URI("https", uri.getUserInfo(), uri.getHost(), -1, uri.getPath(), uri.getQuery(), uri.getFragment()).toString(); return newUrl; From 83f0ffff970cd57f34426969a47b58167b40e13a Mon Sep 17 00:00:00 2001 From: mpeddada1 Date: Mon, 23 Sep 2024 18:27:04 +0000 Subject: [PATCH 09/12] make test names consistent --- .../bigquery/GcpBigQueryAutoConfigurationTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java index f2aa16b534..0656ab2e6b 100644 --- a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java +++ b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java @@ -75,7 +75,7 @@ void testBigQuery_universeDomain() { } @Test - void testBigQuery_noUniverseDomainAndHostSet_useClientDefault() { + void testBigQuery_noUniverseDomainAndEndpointSet_useClientDefault() { this.contextRunner.run( ctx -> { BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); @@ -86,7 +86,7 @@ void testBigQuery_noUniverseDomainAndHostSet_useClientDefault() { } @Test - void testBigQuery_host() { + void testBigQuery_endpoint() { this.contextRunner .withPropertyValues("spring.cloud.gcp.bigquery.endpoint=bigquery.example.com:443") .run( @@ -98,7 +98,7 @@ void testBigQuery_host() { } @Test - void testBigQuery_bothHostAndUniverseDomainSet() { + void testBigQuery_bothEndpointAndUniverseDomainSet() { this.contextRunner .withPropertyValues("spring.cloud.gcp.bigquery.endpoint=bigquery.example.com:123") .withPropertyValues("spring.cloud.gcp.bigquery.universe-domain=myUniverseDomain") From c3ebe7c91b6e3c486fadc1b53745df9e2e96bde1 Mon Sep 17 00:00:00 2001 From: mpeddada1 Date: Wed, 25 Sep 2024 14:58:54 +0000 Subject: [PATCH 10/12] simplify reformatting logic --- .../bigquery/GcpBigQueryAutoConfiguration.java | 12 ++++-------- .../bigquery/GcpBigQueryAutoConfigurationTests.java | 4 ++-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java index e303aaa45c..f8ed7fdfad 100644 --- a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java +++ b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java @@ -162,14 +162,10 @@ public BigQueryTemplate bigQueryTemplate( } private String resolveToHost(String endpoint) throws URISyntaxException { - if (!endpoint.contains("://")) { - endpoint = "https://" + endpoint; + int portIndex = endpoint.indexOf(":"); + if (portIndex != -1){ + return "https://" + endpoint.substring(0,portIndex) + "/"; } - URI uri = new URI(endpoint); - - // Construct the new URL with https:// and no port - String newUrl = new URI("https", uri.getUserInfo(), uri.getHost(), -1, - uri.getPath(), uri.getQuery(), uri.getFragment()).toString(); - return newUrl; + return "https://" + endpoint + "/"; } } diff --git a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java index 0656ab2e6b..110b7b1fa5 100644 --- a/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java +++ b/spring-cloud-gcp-autoconfigure/src/test/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfigurationTests.java @@ -93,7 +93,7 @@ void testBigQuery_endpoint() { ctx -> { BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); assertThat(options.getResolvedApiaryHost("bigquery")) - .isEqualTo("https://bigquery.example.com"); + .isEqualTo("https://bigquery.example.com/"); }); } @@ -106,7 +106,7 @@ void testBigQuery_bothEndpointAndUniverseDomainSet() { ctx -> { BigQueryOptions options = ctx.getBean(BigQuery.class).getOptions(); assertThat(options.getResolvedApiaryHost("bigquery")) - .isEqualTo("https://bigquery.example.com"); + .isEqualTo("https://bigquery.example.com/"); assertThat(options.getUniverseDomain()).isEqualTo("myUniverseDomain"); }); } From 4b34ff318e539f5edffbdaf8ae84c9e33b6627ed Mon Sep 17 00:00:00 2001 From: mpeddada1 Date: Wed, 25 Sep 2024 15:01:45 +0000 Subject: [PATCH 11/12] reformat --- .../autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java index f8ed7fdfad..2bab6fb587 100644 --- a/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java +++ b/spring-cloud-gcp-autoconfigure/src/main/java/com/google/cloud/spring/autoconfigure/bigquery/GcpBigQueryAutoConfiguration.java @@ -27,7 +27,6 @@ import com.google.cloud.spring.core.GcpProjectIdProvider; import com.google.cloud.spring.core.UserAgentHeaderProvider; import java.io.IOException; -import java.net.URI; import java.net.URISyntaxException; import java.util.HashMap; import java.util.Map; @@ -163,8 +162,8 @@ public BigQueryTemplate bigQueryTemplate( private String resolveToHost(String endpoint) throws URISyntaxException { int portIndex = endpoint.indexOf(":"); - if (portIndex != -1){ - return "https://" + endpoint.substring(0,portIndex) + "/"; + if (portIndex != -1) { + return "https://" + endpoint.substring(0, portIndex) + "/"; } return "https://" + endpoint + "/"; } From 2309417002ec71e61a17698c0fa8f90613ccbc9e Mon Sep 17 00:00:00 2001 From: mpeddada1 Date: Wed, 25 Sep 2024 15:45:26 +0000 Subject: [PATCH 12/12] add documentation --- docs/src/main/asciidoc/bigquery.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/src/main/asciidoc/bigquery.adoc b/docs/src/main/asciidoc/bigquery.adoc index 180f90716a..2596f39443 100644 --- a/docs/src/main/asciidoc/bigquery.adoc +++ b/docs/src/main/asciidoc/bigquery.adoc @@ -38,6 +38,8 @@ The following application properties may be configured with Spring Framework on | `spring.cloud.gcp.bigquery.credentials.location` | Credentials file location for authenticating with the Google Cloud BigQuery APIs, if different from the ones in the <> | No | Inferred from https://cloud.google.com/docs/authentication/production[Application Default Credentials], typically set by https://cloud.google.com/sdk/gcloud/reference/auth/application-default[`gcloud`]. | `spring.cloud.gcp.bigquery.jsonWriterBatchSize` | Batch size which will be used by `BigQueryJsonDataWriter` while using https://cloud.google.com/bigquery/docs/write-api[BigQuery Storage Write API]. Note too large or too low values might impact performance. | No | 1000 | `spring.cloud.gcp.bigquery.threadPoolSize` | The size of thread pool of `ThreadPoolTaskScheduler` which is used by `BigQueryTemplate` | No | 4 +| `spring.cloud.gcp.bigquery.universe-domain` | Universe domain of the Bigquery service. The universe domain is a part of the endpoint which is formatted as ${service}.${universeDomain}:${port} | Relies on client library’s default universe domain which is googleapis.com +| `spring.cloud.gcp.bigquery.endpoint` | Endpoint of the Bigquery service. Follows the ${service}.${universeDomain}:${port} format for the BigqueryWriteClient otherwise reformats it to `https://${service}.${universeDomain}/` when setting it to Bigquery client. |=========================================================================== ==== BigQuery Client Object