From 139190a621408f413c9fcf01096d7320a23afaff Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 4 Nov 2024 11:10:29 +0000 Subject: [PATCH] added sample and test case for download to posix storage transfer service. --- .../DownloadToPosixTest.cs | 70 ++++++++++++++++ .../StorageFixture.cs | 6 +- .../DownloadToPosixSample.cs | 79 +++++++++++++++++++ 3 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs new file mode 100644 index 00000000000..f5aedff1291 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -0,0 +1,70 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; +using System; +using Xunit.Abstractions; +using Xunit; +using System.Text; +using System.IO; + + +namespace StorageTransfer.Samples.Tests; +[Collection(nameof(StorageFixture))] +public class DownloadToPosixTest : IDisposable +{ + private readonly StorageFixture _fixture; + private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; + public DownloadToPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper) + { + _fixture = fixture; + _outputHelper = outputHelper; + } + + [Fact] + public void DownloadToPosix() + { + DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(_outputHelper); + var storage = StorageClient.Create(); + byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + storage.UploadObject(_fixture.BucketNameSource,"DownloadToPosixTestFile", "application/octet-stream", stream); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.RootDirectory); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index feca9a4068a..4e19c17d07f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -30,6 +30,8 @@ public class StorageFixture : IDisposable, ICollectionFixture public string BucketNameSink { get; } = Guid.NewGuid().ToString(); public string JobName { get; } public string SourceAgentPoolName { get; } + public string SinkAgentPoolName { get; } + public string GcsSourcePath { get;} public string RootDirectory { get; } = "/tmp/uploads"; public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; @@ -41,7 +43,9 @@ public StorageFixture() Random random = new Random(); JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); - SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet"; + SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/source_test_dotnet"; + SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/sink_test_dotnet"; + GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id."); diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs new file mode 100644 index 00000000000..3a7e114d51a --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -0,0 +1,79 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_download_to_posix] +using Google.Cloud.StorageTransfer.V1; +using Xunit.Abstractions; + + +namespace StorageTransfer.Samples +{ + public class DownloadToPosixSample + { + /*Create a transfer from a GCS bucket to a POSIX file system.*/ + private readonly ITestOutputHelper _output; + public DownloadToPosixSample(ITestOutputHelper output) + { + _output = output; + } + public TransferJob DownloadToPosix( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent + string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // Your GCS source bucket name + string gcsSourceBucket = "my-gcs-source-bucket", + // An optional path on the Google Cloud Storage bucket to download from + string gcsSourcePath = "foo/bar/", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads") + { + // # A useful description for your transfer job + string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath}, + SinkAgentPoolName = sinkAgentPoolName, + PosixDataSink = new PosixFilesystem { RootDirectory = rootDirectory } + }, + Status = TransferJob.Types.Status.Enabled, + }; + + + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); + + _output.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); + return response; + + + } + } +} +//[END storagetransfer_download_to_posix] + +