From bb66a633f26680d8916342851b25e17b3bd7b9b7 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Tue, 5 Nov 2024 11:58:18 +0000 Subject: [PATCH 1/2] code changes for creation of temp folder in posix file system --- .../StorageTransfer.Samples.Tests/DownloadToPosixTest.cs | 7 +++++-- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 8 +++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index f5aedff1291..9edf6dbf59b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -38,11 +38,13 @@ public DownloadToPosixTest(StorageFixture fixture, ITestOutputHelper outputHelpe public void DownloadToPosix() { DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(_outputHelper); + Directory.CreateDirectory(_fixture.TempDirectory); var storage = StorageClient.Create(); byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); - storage.UploadObject(_fixture.BucketNameSource,"DownloadToPosixTestFile", "application/octet-stream", stream); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.RootDirectory); + string fileName = $"{_fixture.GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; + storage.UploadObject(_fixture.BucketNameSource,fileName, "application/octet-stream", stream); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.TempDirectory); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -61,6 +63,7 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + Directory.Delete(_fixture.TempDirectory, true); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 4e19c17d07f..51019127d8c 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; +using System.IO; using Google.Apis.Storage.v1.Data; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; @@ -32,7 +33,8 @@ public class StorageFixture : IDisposable, ICollectionFixture public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } public string GcsSourcePath { get;} - public string RootDirectory { get; } = "/tmp/uploads"; + public string RootDirectory { get; } = System.IO.Path.GetTempPath(); + public string TempDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); @@ -43,8 +45,8 @@ public StorageFixture() Random random = new Random(); JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); - SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/source_test_dotnet"; - SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/sink_test_dotnet"; + SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; + SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { From 526b4452d1aa49cca75c7b9569a104116e1f698e Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 6 Nov 2024 07:02:48 +0000 Subject: [PATCH 2/2] couple of test checks added in download to posix --- .../StorageTransfer.Samples.Tests/DownloadToPosixTest.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 9edf6dbf59b..cb1abf8769e 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -19,6 +19,8 @@ using Xunit; using System.Text; using System.IO; +using System.Linq; +using System.Threading.Tasks; namespace StorageTransfer.Samples.Tests; @@ -43,9 +45,13 @@ public void DownloadToPosix() byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); string fileName = $"{_fixture.GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; + string filePath = $"{_fixture.TempDirectory}/{fileName.Split('/').Last()}"; storage.UploadObject(_fixture.BucketNameSource,fileName, "application/octet-stream", stream); var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.TempDirectory); Assert.Contains("transferJobs/", transferJob.Name); + Assert.True( Directory.Exists(_fixture.TempDirectory)); + System.Threading.Thread.Sleep(TimeSpan.FromSeconds(45)); + Assert.True( File.Exists(filePath)); _transferJobName = transferJob.Name; }