From b5d9b10f9cbd0ad35d8e78a91b08e5a8ff1c9fc1 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Tue, 29 Oct 2024 06:14:42 -0700 Subject: [PATCH 1/2] storage transfer from POSIX file system to GCS bucket using manifest file --- .../StorageFixture.cs | 20 ++++- .../TransferUsingManifestTest.cs | 72 +++++++++++++++ .../TransferUsingManifestSample.cs | 89 +++++++++++++++++++ 3 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 08e7c9ce72f..9ba481fa175 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -29,7 +29,10 @@ public class StorageFixture : IDisposable, ICollectionFixture public string BucketNameSource { get; } = Guid.NewGuid().ToString(); public string BucketNameSink { get; } = Guid.NewGuid().ToString(); public string JobName { get; } + public string SourceAgentPoolName { get; } + public string RootDirectory { get; } = "/tmp/uploads"; public StorageClient Storage { get; } = StorageClient.Create(); + public string ManifestObjectName { get; } = "manifest.csv"; public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); public StorageFixture() @@ -37,6 +40,7 @@ public StorageFixture() // Instantiate random number generator Random random = new Random(); JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; + SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet"; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); if (string.IsNullOrWhiteSpace(ProjectId)) @@ -102,7 +106,13 @@ public void Dispose() } catch (Exception) { - // Do nothing, we delete on a best effort basis. + // If bucket is not empty, we delete on a best effort basis. + foreach (var storageObject in Storage.ListObjects(BucketNameSink, "")) + { + Storage.DeleteObject(BucketNameSink, storageObject.Name); + + } + Storage.DeleteBucket(BucketNameSink); } try { @@ -110,7 +120,13 @@ public void Dispose() } catch (Exception) { - // Do nothing, we delete on a best effort basis. + // If bucket is not empty, we delete on a best effort basis. + foreach (var storageObject in Storage.ListObjects(BucketNameSource, "")) + { + Storage.DeleteObject(BucketNameSource, storageObject.Name); + + } + Storage.DeleteBucket(BucketNameSource); } } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs new file mode 100644 index 00000000000..5cd2f4f150e --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -0,0 +1,72 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit; +using System.IO; + +namespace StorageTransfer.Samples.Tests; +[Collection(nameof(StorageFixture))] +public class TransferUsingManifestTest : IDisposable +{ + private readonly StorageFixture _fixture; + private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; + public TransferUsingManifestTest(StorageFixture fixture, ITestOutputHelper outputHelper) + { + _fixture = fixture; + _outputHelper = outputHelper; + } + + [Fact] + public void TransferUsingManifest() + { + TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(_outputHelper); + var storage = StorageClient.Create(); + byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + storage.UploadObject(_fixture.BucketNameSource, _fixture.ManifestObjectName, "application/octet-stream", stream); + var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.ManifestObjectName); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs new file mode 100644 index 00000000000..2f7b2102953 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -0,0 +1,89 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_manifest_request] +using Google.Cloud.StorageTransfer.V1; +using Google.Protobuf.WellKnownTypes; +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Xunit.Abstractions; + + +namespace StorageTransfer.Samples +{ + public class TransferUsingManifestSample + { + /*Create a transfer from a POSIX file system to a GCS bucket using + a manifest file*/ + private readonly ITestOutputHelper _output; + public TransferUsingManifestSample(ITestOutputHelper output) + { + _output = output; + } + public TransferJob TransferUsingManifest( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent + string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads", + // The GCS bucket which has your manifest file + string manifestBucket = "my-source-bucket", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket", + // The name of the manifest file in manifestBucket that specifies which objects to transfer + string manifestObjectName = "path/to/manifest.csv") + { + string manifestLocation = "gs://" + manifestBucket + "/" + manifestObjectName; + + // # A useful description for your transfer job + string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file"; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = manifestBucket }, + SourceAgentPoolName = sourceAgentPoolName, + PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, + TransferManifest = new TransferManifest { Location = manifestLocation } + }, + Status = TransferJob.Types.Status.Enabled, + }; + + + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); + + _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}"); + return response; + + + } + } +} +// [END storagetransfer_manifest_request] From 3c9d243bc0e425939be7b325e1d24e369c454829 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 30 Oct 2024 04:23:27 -0700 Subject: [PATCH 2/2] minor changes --- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 9 +++------ .../TransferUsingManifestTest.cs | 3 --- .../TransferUsingManifestSample.cs | 5 ----- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 9ba481fa175..feca9a4068a 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google Inc. + * Copyright 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,10 +39,9 @@ public StorageFixture() { // Instantiate random number generator Random random = new Random(); - JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; - SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet"; - + JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); + SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet"; if (string.IsNullOrWhiteSpace(ProjectId)) { throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id."); @@ -110,7 +109,6 @@ public void Dispose() foreach (var storageObject in Storage.ListObjects(BucketNameSink, "")) { Storage.DeleteObject(BucketNameSink, storageObject.Name); - } Storage.DeleteBucket(BucketNameSink); } @@ -124,7 +122,6 @@ public void Dispose() foreach (var storageObject in Storage.ListObjects(BucketNameSource, "")) { Storage.DeleteObject(BucketNameSource, storageObject.Name); - } Storage.DeleteBucket(BucketNameSource); } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 5cd2f4f150e..c19f33a4fd4 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -15,10 +15,7 @@ using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using System; -using System.Collections.Generic; -using System.Linq; using System.Text; -using System.Threading.Tasks; using Xunit.Abstractions; using Xunit; using System.IO; diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 2f7b2102953..5625a17177e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -13,11 +13,6 @@ // limitations under the License. // [START storagetransfer_manifest_request] using Google.Cloud.StorageTransfer.V1; -using Google.Protobuf.WellKnownTypes; -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; using Xunit.Abstractions;