samples(storage transfer): transfer from POSIX file system to GCS bucket using manifest file

storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs

@@ -29,14 +29,18 @@ public class StorageFixture : IDisposable, ICollectionFixture<StorageFixture>
         public string BucketNameSource { get; } = Guid.NewGuid().ToString();
         public string BucketNameSink { get; } = Guid.NewGuid().ToString();
         public string JobName { get; }
+        public string SourceAgentPoolName { get; }
+        public string RootDirectory { get; } = "/tmp/uploads";
         public StorageClient Storage { get; } = StorageClient.Create();
+        public string ManifestObjectName { get; } = "manifest.csv";
         public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create();
 
         public StorageFixture()
         {
             // Instantiate random number generator 
             Random random = new Random();
             JobName =  "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " ";
+            SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet";
 
             ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID");
             if (string.IsNullOrWhiteSpace(ProjectId))
@@ -102,15 +106,27 @@ public void Dispose()
             }
             catch (Exception)
             {
-                // Do nothing, we delete on a best effort basis.
+                // If bucket is not empty, we delete on a best effort basis.
+                foreach (var storageObject in Storage.ListObjects(BucketNameSink, ""))
+                {
+                    Storage.DeleteObject(BucketNameSink, storageObject.Name);
+
+                }
+                Storage.DeleteBucket(BucketNameSink);
             }
             try
             {
                 Storage.DeleteBucket(BucketNameSource);
             }
             catch (Exception)
             {
-                // Do nothing, we delete on a best effort basis.
+                // If bucket is not empty, we delete on a best effort basis.
+                foreach (var storageObject in Storage.ListObjects(BucketNameSource, ""))
+                {
+                    Storage.DeleteObject(BucketNameSource, storageObject.Name);
+
+                }
+                Storage.DeleteBucket(BucketNameSource);
             }
         }
     }

storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs

@@ -0,0 +1,72 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License").
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at 
+//
+// https://www.apache.org/licenses/LICENSE-2.0 
+//
+// Unless required by applicable law or agreed to in writing, software 
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and 
+// limitations under the License.
+
+using Google.Cloud.Storage.V1;
+using Google.Cloud.StorageTransfer.V1;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+using Xunit;
+using System.IO;
+
+namespace StorageTransfer.Samples.Tests;
+[Collection(nameof(StorageFixture))]
+public class TransferUsingManifestTest : IDisposable
+{
+    private readonly StorageFixture _fixture;
+    private string _transferJobName;
+    private readonly ITestOutputHelper _outputHelper;
+    public TransferUsingManifestTest(StorageFixture fixture, ITestOutputHelper outputHelper)
+    {
+        _fixture = fixture;
+        _outputHelper = outputHelper;
+    }
+
+    [Fact]
+    public void TransferUsingManifest()
+    {
+        TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(_outputHelper);
+        var storage = StorageClient.Create();
+        byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg");
+        MemoryStream stream = new MemoryStream(byteArray);
+        storage.UploadObject(_fixture.BucketNameSource, _fixture.ManifestObjectName, "application/octet-stream", stream);
+        var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.ManifestObjectName);
+        Assert.Contains("transferJobs/", transferJob.Name);
+        _transferJobName = transferJob.Name;
+    }
+
+    public void Dispose()
+    {
+        try
+        {
+            _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest()
+            {
+                ProjectId = _fixture.ProjectId,
+                JobName = _transferJobName,
+                TransferJob = new TransferJob()
+                {
+                    Name = _transferJobName,
+                    Status = TransferJob.Types.Status.Deleted
+                }
+            });
+        }
+        catch (Exception)
+        {
+            // Do nothing, we delete on a best effort basis.
+        }
+    }
+}

storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs

@@ -0,0 +1,89 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License").
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at 
+//
+// https://www.apache.org/licenses/LICENSE-2.0 
+//
+// Unless required by applicable law or agreed to in writing, software 
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and 
+// limitations under the License.
+// [START storagetransfer_manifest_request]
+using Google.Cloud.StorageTransfer.V1;
+using Google.Protobuf.WellKnownTypes;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using Xunit.Abstractions;
+
+
+namespace StorageTransfer.Samples
+{
+    public class TransferUsingManifestSample
+    {
+        /*Create a transfer from a POSIX file system to a GCS bucket using
+        a manifest file*/
+        private readonly ITestOutputHelper _output;
+        public TransferUsingManifestSample(ITestOutputHelper output)
+        {
+            _output = output;
+        }
+        public TransferJob TransferUsingManifest(
+            // Your Google Cloud Project ID
+            string projectId = "my-project-id",
+            // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent
+            string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default",
+            // The root directory path on the source filesystem
+            string rootDirectory = "/tmp/uploads",
+            // The GCS bucket which has your manifest file
+            string manifestBucket = "my-source-bucket",
+            // The GCS bucket to transfer data to
+            string sinkBucket = "my-sink-bucket",
+            // The name of the manifest file in manifestBucket that specifies which objects to transfer
+            string manifestObjectName = "path/to/manifest.csv")
+        {
+            string manifestLocation = "gs://" + manifestBucket + "/" + manifestObjectName;
+
+            // # A useful description for your transfer job
+            string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file";
+
+            TransferJob transferJob = new TransferJob
+            {
+                ProjectId = projectId,
+                Description = jobDescription,
+                TransferSpec = new TransferSpec
+                {
+                    GcsDataSink = new GcsData { BucketName = sinkBucket },
+                    GcsDataSource = new GcsData { BucketName = manifestBucket },
+                    SourceAgentPoolName = sourceAgentPoolName,
+                    PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory },
+                    TransferManifest = new TransferManifest { Location = manifestLocation }
+                },
+                Status = TransferJob.Types.Status.Enabled,
+            };
+
+
+            // Create a Transfer Service client
+            StorageTransferServiceClient client = StorageTransferServiceClient.Create();
+
+            // Create a Transfer job
+            TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob });
+
+            client.RunTransferJob(new RunTransferJobRequest
+            {
+                JobName = response.Name,
+                ProjectId = projectId
+            });
+
+            _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}");
+            return response;
+
+
+        }
+    }
+}
+// [END storagetransfer_manifest_request]