samples(storage transfer): add sample and test case for storage transfer services. (#4)

Author: mahendra-google

storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs

@@ -0,0 +1,70 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License").
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at 
+//
+// https://www.apache.org/licenses/LICENSE-2.0 
+//
+// Unless required by applicable law or agreed to in writing, software 
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and 
+// limitations under the License.
+
+using Google.Cloud.Storage.V1;
+using Google.Cloud.StorageTransfer.V1;
+using System;
+using Xunit.Abstractions;
+using Xunit;
+using System.Text;
+using System.IO;
+
+
+namespace StorageTransfer.Samples.Tests;
+[Collection(nameof(StorageFixture))]
+public class DownloadToPosixTest : IDisposable
+{
+    private readonly StorageFixture _fixture;
+    private string _transferJobName;
+    private readonly ITestOutputHelper _outputHelper;
+    public DownloadToPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper)
+    {
+        _fixture = fixture;
+        _outputHelper = outputHelper;
+    }
+
+    [Fact]
+    public void DownloadToPosix()
+    {
+        DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(_outputHelper);
+        var storage = StorageClient.Create();
+        byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg");
+        MemoryStream stream = new MemoryStream(byteArray);
+        storage.UploadObject(_fixture.BucketNameSource,"DownloadToPosixTestFile", "application/octet-stream", stream);
+        var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.RootDirectory);
+        Assert.Contains("transferJobs/", transferJob.Name);
+        _transferJobName = transferJob.Name;
+    }
+
+    public void Dispose()
+    {
+        try
+        {
+            _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest()
+            {
+                ProjectId = _fixture.ProjectId,
+                JobName = _transferJobName,
+                TransferJob = new TransferJob()
+                {
+                    Name = _transferJobName,
+                    Status = TransferJob.Types.Status.Deleted
+                }
+            });
+        }
+        catch (Exception)
+        {
+            // Do nothing, we delete on a best effort basis.
+        }
+    }
+}

storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs

@@ -30,6 +30,8 @@ public class StorageFixture : IDisposable, ICollectionFixture<StorageFixture>
         public string BucketNameSink { get; } = Guid.NewGuid().ToString();
         public string JobName { get; }
         public string SourceAgentPoolName { get; }
+        public string SinkAgentPoolName { get; }
+        public string GcsSourcePath { get;}
         public string RootDirectory { get; } = "/tmp/uploads";
         public StorageClient Storage { get; } = StorageClient.Create();
         public string ManifestObjectName { get; } = "manifest.csv";
@@ -41,7 +43,9 @@ public StorageFixture()
             Random random = new Random();
             JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " ";
             ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID");
-            SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet";
+            SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/source_test_dotnet";
+            SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/sink_test_dotnet";
+            GcsSourcePath = "foo/bar/";
             if (string.IsNullOrWhiteSpace(ProjectId))
             {
                 throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id.");

storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs

@@ -0,0 +1,79 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License").
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at 
+//
+// https://www.apache.org/licenses/LICENSE-2.0 
+//
+// Unless required by applicable law or agreed to in writing, software 
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and 
+// limitations under the License.
+// [START storagetransfer_download_to_posix]
+using Google.Cloud.StorageTransfer.V1;
+using Xunit.Abstractions;
+
+
+namespace StorageTransfer.Samples
+{
+    public class DownloadToPosixSample
+    {
+        /*Create a transfer from a GCS bucket to a POSIX file system.*/
+        private readonly ITestOutputHelper _output;
+        public DownloadToPosixSample(ITestOutputHelper output)
+        {
+            _output = output;
+        }
+        public TransferJob DownloadToPosix(
+            // Your Google Cloud Project ID
+            string projectId = "my-project-id",
+            // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent
+            string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default",
+            // Your GCS source bucket name
+            string gcsSourceBucket = "my-gcs-source-bucket",
+            // An optional path on the Google Cloud Storage bucket to download from
+            string gcsSourcePath = "foo/bar/",
+            // The root directory path on the source filesystem
+            string rootDirectory = "/tmp/uploads")
+        {
+            // # A useful description for your transfer job
+            string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system";
+
+            TransferJob transferJob = new TransferJob
+            {
+                ProjectId = projectId,
+                Description = jobDescription,
+                TransferSpec = new TransferSpec
+                {
+                    GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath},
+                    SinkAgentPoolName = sinkAgentPoolName,
+                    PosixDataSink = new PosixFilesystem { RootDirectory = rootDirectory }
+                },
+                Status = TransferJob.Types.Status.Enabled,
+            };
+
+
+            // Create a Transfer Service client
+            StorageTransferServiceClient client = StorageTransferServiceClient.Create();
+
+            // Create a Transfer job
+            TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob });
+
+            client.RunTransferJob(new RunTransferJobRequest
+            {
+                JobName = response.Name,
+                ProjectId = projectId
+            });
+
+            _output.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}");
+            return response;
+
+
+        }
+    }
+}
+//[END storagetransfer_download_to_posix]
+
+