samples(storage transfer): add samples and test cases for storage transfer services

storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs

@@ -0,0 +1,37 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License").
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at 
+//
+// https://www.apache.org/licenses/LICENSE-2.0 
+//
+// Unless required by applicable law or agreed to in writing, software 
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and 
+// limitations under the License.
+using Xunit;
+
+namespace StorageTransfer.Samples.Tests;
+
+[Collection(nameof(StorageFixture))]
+public class CheckLatestTransferOperationTest
+{
+
+    private readonly StorageFixture _fixture;
+    private string _jobName;
+    public CheckLatestTransferOperationTest(StorageFixture fixture)
+    {
+        _fixture = fixture;
+    }
+
+    [Fact]
+    public void CheckLatestTransferOperation()
+    {
+        CheckLatestTransferOperationSample checkLatestTransferOperationSample = new CheckLatestTransferOperationSample();
+        var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId, _fixture.JobName);
+        Assert.Contains("transferJobs/", transferJob.Name);
+        _jobName = transferJob.Name;
+    }
+}

storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs

@@ -0,0 +1,63 @@
+/**
+ * Copyright 2024 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Google.Cloud.StorageTransfer.V1;
+using System;
+using Xunit;
+
+namespace StorageTransfer.Samples.Tests
+{
+    [Collection(nameof(StorageFixture))]
+    public class CreateEventDrivenGcsTransferTest : IDisposable
+    {
+        private readonly StorageFixture _fixture;
+        private string _transferJobName;
+        public CreateEventDrivenGcsTransferTest(StorageFixture fixture)
+        {
+            _fixture = fixture;
+        }
+
+        [Fact]
+        public void CreateEventDrivenGcsTransfer()
+        {
+            CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample();
+            var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.PubSubId);
+            Assert.Contains("transferJobs/", transferJob.Name);
+            _transferJobName = transferJob.Name;
+        }
+
+        public void Dispose()
+        {
+            try
+            {
+                _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest()
+                {
+                    ProjectId = _fixture.ProjectId,
+                    JobName = _transferJobName,
+                    TransferJob = new TransferJob()
+                    {
+                        Name = _transferJobName,
+                        Status = TransferJob.Types.Status.Deleted
+                    }
+                });
+            }
+            catch (Exception)
+            {
+                // Do nothing, we delete on a best effort basis.
+            }
+        }
+    }
+}

storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs

@@ -0,0 +1,72 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License").
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at 
+//
+// https://www.apache.org/licenses/LICENSE-2.0 
+//
+// Unless required by applicable law or agreed to in writing, software 
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and 
+// limitations under the License.
+
+using Google.Cloud.Storage.V1;
+using Google.Cloud.StorageTransfer.V1;
+using System;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace StorageTransfer.Samples.Tests;
+[Collection(nameof(StorageFixture))]
+public class DownloadToPosixTest : IDisposable
+{
+    private readonly StorageFixture _fixture;
+    private string _transferJobName;
+    public DownloadToPosixTest(StorageFixture fixture)
+    {
+        _fixture = fixture;
+    }
+
+    [Fact]
+    public void DownloadToPosix()
+    {
+        DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample();
+        Directory.CreateDirectory(_fixture.TempDirectory);
+        var storage = StorageClient.Create();
+        byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg");
+        MemoryStream stream = new MemoryStream(byteArray);
+        string fileName = $"{_fixture.GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt";
+        storage.UploadObject(_fixture.BucketNameSource, fileName, "application/octet-stream", stream);
+        var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNameSource, _fixture.GcsSourcePath, _fixture.TempDirectory);
+        Assert.Contains("transferJobs/", transferJob.Name);
+        Assert.True(Directory.Exists(_fixture.TempDirectory));
+        _transferJobName = transferJob.Name;
+    }
+
+    public void Dispose()
+    {
+        try
+        {
+            _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest()
+            {
+                ProjectId = _fixture.ProjectId,
+                JobName = _transferJobName,
+                TransferJob = new TransferJob()
+                {
+                    Name = _transferJobName,
+                    Status = TransferJob.Types.Status.Deleted
+                }
+            });
+            Directory.Delete(_fixture.TempDirectory, true);
+        }
+        catch (Exception)
+        {
+            // Do nothing, we delete on a best effort basis.
+        }
+    }
+}

storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs

@@ -1,5 +1,5 @@
-/**
- * Copyright 2021 Google Inc.
+/**
+ * Copyright 2024 Google Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-using System;
 using Google.Cloud.StorageTransfer.V1;
+using System;
 using Xunit;
 
 namespace StorageTransfer.Samples.Tests
@@ -25,7 +25,6 @@ public class QuickstartTest : IDisposable
     {
         private readonly StorageFixture _fixture;
         private string _transferJobName;
-
         public QuickstartTest(StorageFixture fixture)
         {
             _fixture = fixture;

storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs

@@ -1,5 +1,5 @@
-/**
- * Copyright 2021 Google Inc.
+/**
+ * Copyright 2024 Google Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,12 +14,16 @@
  * limitations under the License.
  */
 
-using System;
-using System.Collections.Generic;
 using Google.Apis.Storage.v1.Data;
+using Google.Cloud.PubSub.V1;
 using Google.Cloud.Storage.V1;
 using Google.Cloud.StorageTransfer.V1;
+using Grpc.Core;
+using System;
+using System.Collections.Generic;
+using System.IO;
 using Xunit;
+
 namespace StorageTransfer.Samples.Tests
 {
     [CollectionDefinition(nameof(StorageFixture))]
@@ -28,19 +32,84 @@ public class StorageFixture : IDisposable, ICollectionFixture<StorageFixture>
         public string ProjectId { get; }
         public string BucketNameSource { get; } = Guid.NewGuid().ToString();
         public string BucketNameSink { get; } = Guid.NewGuid().ToString();
+        public string JobName { get; }
+        public string SourceAgentPoolName { get; }
+        public string SinkAgentPoolName { get; }
+        public string GcsSourcePath { get; }
+        public string RootDirectory { get; } = System.IO.Path.GetTempPath();
+        public string DestinationDirectory { get; } = System.IO.Path.GetTempPath();
+        public string TempDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+        public string TempDestinationDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
         public StorageClient Storage { get; } = StorageClient.Create();
+        public string ManifestObjectName { get; } = "manifest.csv";
+        public string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString();
+        public string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString();
+        public string PubSubId { get; }
         public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create();
 
+        public SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create();
+
+        public PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create();
+
         public StorageFixture()
         {
             ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID");
+            SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default";
+            SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default";
+            PubSubId = "projects/" + ProjectId + "/subscriptions/" + SubscriptionId + "";
+            GcsSourcePath = "foo/bar/";
             if (string.IsNullOrWhiteSpace(ProjectId))
             {
                 throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id.");
             }
 
             CreateBucketAndGrantStsPermissions(BucketNameSink);
             CreateBucketAndGrantStsPermissions(BucketNameSource);
+            // Initialize request argument(s)
+            TransferJob transferJob = new TransferJob
+            {
+                ProjectId = ProjectId,
+                TransferSpec = new TransferSpec
+                {
+                    GcsDataSink = new GcsData { BucketName = BucketNameSource },
+                    GcsDataSource = new GcsData { BucketName = BucketNameSink }
+                },
+                Status = TransferJob.Types.Status.Enabled
+            };
+            CreateTransferJobRequest request = new CreateTransferJobRequest
+            {
+                TransferJob = transferJob
+            };
+            // Make the request
+            TransferJob response = Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob });
+            JobName = response.Name;
+            string email = Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest()
+            {
+                ProjectId = ProjectId
+            }).AccountEmail;
+            string memberServiceAccount = "serviceAccount:" + email;
+            // Create subscription name
+            SubscriptionName subscriptionName = new SubscriptionName(ProjectId, SubscriptionId);
+            // Create topic name
+            TopicName topicName = new TopicName(ProjectId, TopicId);
+            // Create topic
+            PublisherClient.CreateTopic(topicName);
+            // Create subscription.
+            SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500);
+            var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy();
+            policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount);
+            PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest
+            {
+                ResourceAsResourceName = topicName,
+                Policy = policyIamPolicyTopic
+            });
+            var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy();
+            policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount);
+            PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest
+            {
+                ResourceAsResourceName = subscriptionName,
+                Policy = policyIamPolicySubscriber
+            });
         }
 
         private void CreateBucketAndGrantStsPermissions(string bucketName)
@@ -82,11 +151,13 @@ private void CreateBucketAndGrantStsPermissions(string bucketName)
                 Members = new List<string> { member }
             };
 
+
             policy.Bindings.Add(objectViewerBinding);
             policy.Bindings.Add(bucketReaderBinding);
             policy.Bindings.Add(bucketWriterBinding);
 
             Storage.SetBucketIamPolicy(bucketName, policy);
+
         }
 
         public void Dispose()
@@ -97,16 +168,47 @@ public void Dispose()
             }
             catch (Exception)
             {
-                // Do nothing, we delete on a best effort basis.
+                // If bucket is not empty, we delete on a best effort basis.
+                foreach (var storageObject in Storage.ListObjects(BucketNameSink, ""))
+                {
+                    Storage.DeleteObject(BucketNameSink, storageObject.Name);
+                }
+                Storage.DeleteBucket(BucketNameSink);
             }
             try
             {
                 Storage.DeleteBucket(BucketNameSource);
             }
             catch (Exception)
             {
-                // Do nothing, we delete on a best effort basis.
+                // If bucket is not empty, we delete on a best effort basis.
+                foreach (var storageObject in Storage.ListObjects(BucketNameSource, ""))
+                {
+                    Storage.DeleteObject(BucketNameSource, storageObject.Name);
+                }
+                Storage.DeleteBucket(BucketNameSource);
+            }
+
+            try
+            {
+                TopicName topicName = TopicName.FromProjectTopic(ProjectId, TopicId);
+                PublisherClient.DeleteTopic(topicName);
             }
+            catch (RpcException ex)
+            {
+                throw new Exception($"Exception occur while deleting Topic {TopicId} Exception: {ex}");
+            }
+
+            try
+            {
+                SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(ProjectId, SubscriptionId);
+                SubscriberClient.DeleteSubscription(subscriptionName);
+            }
+            catch (RpcException ex)
+            {
+                throw new Exception($"Exception occur while deleting subscription {SubscriptionId} Exception: {ex}");
+            }
+
         }
     }
 }

storagetransfer/api/StorageTransfer.Samples.Tests/StorageTransfer.Samples.Tests.csproj

@@ -16,6 +16,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
     <PackageReference Include="xunit" Version="2.9.2" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
+    <PackageReference Include="Google.Cloud.PubSub.V1" Version="3.18.0" />
   </ItemGroup>
 
 </Project>