fix: add runtime hints for storage (#2001)

* fix: add runtime hints for sample and add sample test to ci; refactor tests to use unique resources

---------

Co-authored-by: mpeddada1 <[email protected]>
Co-authored-by: Mridula <[email protected]>

Author: zhumin8

pom.xml

@@ -353,6 +353,7 @@
 	<profile>
 		<id>spring-native</id>
 		<modules>
+			<module>spring-cloud-gcp-storage</module>
 			<module>spring-cloud-gcp-vision</module>
 		</modules>
 
@@ -379,6 +380,7 @@
 						<classesDirectory>${project.build.outputDirectory}</classesDirectory>
 						<systemPropertyVariables>
 							<!--integration tests are not invoked unless the relevant system property is set to true here. -->
+              <it.storage>true</it.storage>
 							<it.vision>true</it.vision>
 						</systemPropertyVariables>
 					</configuration>

spring-cloud-gcp-samples/pom.xml

@@ -91,6 +91,8 @@
 			<id>native-sample-config</id>
 			<modules>
 				<module>spring-cloud-gcp-logging-sample</module>
+				<module>spring-cloud-gcp-storage-resource-sample</module>
+				<module>spring-cloud-gcp-integration-storage-sample</module>
 				<module>spring-cloud-gcp-trace-sample</module>
 				<module>spring-cloud-gcp-vision-api-sample</module>
 			</modules>
@@ -135,8 +137,14 @@
 							</includes>
 							<systemPropertyVariables>
 								<it.logging>true</it.logging>
+								<it.storage>true</it.storage>
+                <it.vision>true</it.vision>
 								<it.trace>true</it.trace>
 								<it.vision>true</it.vision>
+                <gcs-resource-test-bucket>gcp-storage-resource-bucket-sample</gcs-resource-test-bucket>
+								<gcs-read-bucket>gcp-storage-bucket-sample-input</gcs-read-bucket>
+								<gcs-write-bucket>gcp-storage-bucket-sample-output</gcs-write-bucket>
+								<gcs-local-directory>/tmp/gcp_integration_tests/integration_storage_sample</gcs-local-directory>
 							</systemPropertyVariables>
 						</configuration>
 					</plugin>

spring-cloud-gcp-samples/spring-cloud-gcp-integration-storage-sample/src/main/java/com/example/GcsSpringIntegrationApplication.java

@@ -44,21 +44,23 @@
 @SpringBootApplication
 public class GcsSpringIntegrationApplication {
 
+  private static final Log LOGGER = LogFactory.getLog(GcsSpringIntegrationApplication.class);
+
   @Value("${gcs-read-bucket}")
   private String gcsReadBucket;
 
   @Value("${gcs-write-bucket}")
   private String gcsWriteBucket;
 
-  @Value("${gcs-local-directory}")
-  private String localDirectory;
-
-  private static final Log LOGGER = LogFactory.getLog(GcsSpringIntegrationApplication.class);
-
   public static void main(String[] args) {
     SpringApplication.run(GcsSpringIntegrationApplication.class, args);
   }
 
+  @Bean("localDirectoryName")
+  public String localDirectory(@Value("${gcs-local-directory}") String localDirectory) {
+    return localDirectory;
+  }
+
   /**
    * A file synchronizer that knows how to connect to the remote file system (GCS) and scan it for
    * new files and then download the files.
@@ -83,10 +85,11 @@ public GcsInboundFileSynchronizer gcsInboundFileSynchronizer(Storage gcs) {
    */
   @Bean
   @InboundChannelAdapter(channel = "new-file-channel", poller = @Poller(fixedDelay = "5000"))
-  public MessageSource<File> synchronizerAdapter(GcsInboundFileSynchronizer synchronizer) {
+  public MessageSource<File> synchronizerAdapter(
+      GcsInboundFileSynchronizer synchronizer, String localDirectoryName) {
     GcsInboundFileSynchronizingMessageSource syncAdapter =
         new GcsInboundFileSynchronizingMessageSource(synchronizer);
-    syncAdapter.setLocalDirectory(Paths.get(this.localDirectory).toFile());
+    syncAdapter.setLocalDirectory(Paths.get(localDirectoryName).toFile());
 
     return syncAdapter;
   }

spring-cloud-gcp-samples/spring-cloud-gcp-integration-storage-sample/src/test/java/com/example/GcsSpringIntegrationTestConfiguration.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.example;
+
+import java.util.UUID;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.context.annotation.Bean;
+
+@TestConfiguration
+public class GcsSpringIntegrationTestConfiguration {
+
+  private String uniqueDirectory;
+
+  public GcsSpringIntegrationTestConfiguration(
+      @Value("${gcs-local-directory}") String localDirectory) {
+    uniqueDirectory = String.format("%s-%s", localDirectory, UUID.randomUUID());
+  }
+
+  @Bean("localDirectoryName")
+  public String uniqueDirectory() {
+    return uniqueDirectory;
+  }
+}

spring-cloud-gcp-samples/spring-cloud-gcp-integration-storage-sample/src/test/java/com/example/GcsSpringIntegrationTests.java

@@ -18,7 +18,6 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-import com.google.api.gax.paging.Page;
 import com.google.cloud.storage.Blob;
 import com.google.cloud.storage.BlobId;
 import com.google.cloud.storage.BlobInfo;
@@ -30,15 +29,19 @@
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.UUID;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.awaitility.Awaitility;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.context.annotation.PropertySource;
@@ -54,44 +57,43 @@
 @EnabledIfSystemProperty(named = "it.storage", matches = "true")
 @ExtendWith(SpringExtension.class)
 @PropertySource("classpath:application.properties")
-@SpringBootTest(classes = {GcsSpringIntegrationApplication.class})
+@SpringBootTest(
+    classes = {GcsSpringIntegrationApplication.class, GcsSpringIntegrationTestConfiguration.class},
+    properties = {"spring.main.allow-bean-definition-overriding=true"})
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
 class GcsSpringIntegrationTests {
 
-  private static final String TEST_FILE_NAME = "test_file";
+  private static final String TEST_FILE = String.format("test_file_%s", UUID.randomUUID());
+  private static final Log LOGGER = LogFactory.getLog(GcsSpringIntegrationTests.class);
 
   @Autowired private Storage storage;
 
+  @Autowired
+  @Qualifier("localDirectoryName")
+  private String uniqueDirectory;
+
   @Value("${gcs-read-bucket}")
   private String cloudInputBucket;
 
   @Value("${gcs-write-bucket}")
   private String cloudOutputBucket;
 
-  @Value("${gcs-local-directory}")
-  private String outputFolder;
-
-  @BeforeEach
-  void setupTestEnvironment() {
-    cleanupCloudStorage();
-  }
-
-  @AfterEach
+  @AfterAll
   void teardownTestEnvironment() throws IOException {
     cleanupCloudStorage();
-    cleanupLocalDirectory();
+    cleanupLocalDirectory(Paths.get(uniqueDirectory));
   }
 
   @Test
   void testFilePropagatedToLocalDirectory() {
-    BlobId blobId = BlobId.of(this.cloudInputBucket, TEST_FILE_NAME);
+    BlobId blobId = BlobId.of(this.cloudInputBucket, TEST_FILE);
     BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
     this.storage.create(blobInfo, "Hello World!".getBytes(StandardCharsets.UTF_8));
-
     Awaitility.await()
-        .atMost(15, TimeUnit.SECONDS)
+        .atMost(30, TimeUnit.SECONDS)
         .untilAsserted(
             () -> {
-              Path outputFile = Paths.get(this.outputFolder + "/" + TEST_FILE_NAME);
+              Path outputFile = Paths.get(uniqueDirectory + "/" + TEST_FILE);
               assertThat(Files.exists(outputFile)).isTrue();
               assertThat(Files.isRegularFile(outputFile)).isTrue();
 
@@ -104,23 +106,39 @@ void testFilePropagatedToLocalDirectory() {
                   .iterateAll()
                   .forEach(b -> blobNamesInOutputBucket.add(b.getName()));
 
-              assertThat(blobNamesInOutputBucket).contains(TEST_FILE_NAME);
+              assertThat(blobNamesInOutputBucket).contains(TEST_FILE);
             });
   }
 
   void cleanupCloudStorage() {
-    Page<Blob> blobs = this.storage.list(this.cloudInputBucket);
-    for (Blob blob : blobs.iterateAll()) {
-      blob.delete();
+    BlobId inputBucketBlobId = BlobId.of(cloudInputBucket, TEST_FILE);
+    Blob inputBucketBlob = storage.get(inputBucketBlobId);
+    if (inputBucketBlob != null) {
+      inputBucketBlob.delete();
+    }
+
+    BlobId outputBucketBlobId = BlobId.of(cloudOutputBucket, TEST_FILE);
+    Blob outputBucketBlob = storage.get(outputBucketBlobId);
+    if (outputBucketBlob != null) {
+      outputBucketBlob.delete();
     }
   }
 
-  void cleanupLocalDirectory() throws IOException {
-    Path localDirectory = Paths.get(this.outputFolder);
-    List<Path> files = Files.list(localDirectory).collect(Collectors.toList());
-    for (Path file : files) {
-      Files.delete(file);
+  void cleanupLocalDirectory(Path testDirectory) throws IOException {
+    if (Files.exists(testDirectory)) {
+      if (Files.isDirectory(testDirectory)) {
+        try (Stream<Path> files = Files.list(testDirectory)) {
+          files.forEach(
+              path -> {
+                try {
+                  Files.delete(path);
+                } catch (IOException ioe) {
+                  LOGGER.info("Error deleting test file.", ioe);
+                }
+              });
+        }
+      }
+      Files.delete(testDirectory);
     }
-    Files.deleteIfExists(Paths.get(this.outputFolder));
   }
 }

spring-cloud-gcp-samples/spring-cloud-gcp-storage-resource-sample/src/main/java/com/example/GcsApplication.java

@@ -19,6 +19,7 @@
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 
+
 /**
  * An example Spring Boot application that reads and writes files stored in Google Cloud Storage
  * (GCS) using the Spring Resource abstraction and the gs: protocol prefix.

spring-cloud-gcp-samples/spring-cloud-gcp-storage-resource-sample/src/main/java/com/example/WebController.java

@@ -16,16 +16,20 @@
 
 package com.example;
 
+import com.google.cloud.spring.storage.GoogleStorageResource;
+import com.google.cloud.storage.Storage;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.charset.Charset;
+import java.util.Optional;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.WritableResource;
 import org.springframework.util.StreamUtils;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 /**
@@ -35,19 +39,46 @@
 @RestController
 public class WebController {
 
+  @Value("${gcs-resource-test-bucket}")
+  private String bucketName;
+
   @Value("gs://${gcs-resource-test-bucket}/my-file.txt")
   private Resource gcsFile;
 
-  @GetMapping("/")
-  public String readGcsFile() throws IOException {
-    return StreamUtils.copyToString(this.gcsFile.getInputStream(), Charset.defaultCharset()) + "\n";
+  private Storage storage;
+
+  private WebController(Storage storage) {
+    this.storage = storage;
+  }
+
+  @GetMapping(value = "/")
+  public String readGcsFile(@RequestParam("filename") Optional<String> filename)
+      throws IOException {
+    return StreamUtils.copyToString(
+            filename.isPresent()
+                ? fetchResource(filename.get()).getInputStream()
+                : this.gcsFile.getInputStream(),
+            Charset.defaultCharset())
+        + "\n";
   }
 
-  @PostMapping("/")
-  public String writeGcs(@RequestBody String data) throws IOException {
-    try (OutputStream os = ((WritableResource) this.gcsFile).getOutputStream()) {
+  @PostMapping(value = "/")
+  public String writeGcs(
+      @RequestBody String data, @RequestParam("filename") Optional<String> filename)
+      throws IOException {
+    return updateResource(
+        filename.map(this::fetchResource).orElse((GoogleStorageResource) this.gcsFile), data);
+  }
+
+  private String updateResource(Resource resource, String data) throws IOException {
+    try (OutputStream os = ((WritableResource) resource).getOutputStream()) {
       os.write(data.getBytes());
     }
     return "file was updated\n";
   }
+
+  private GoogleStorageResource fetchResource(String filename) {
+    return new GoogleStorageResource(
+        this.storage, String.format("gs://%s/%s", this.bucketName, filename));
+  }
 }