From ba3c128633f7fb9f4686af3893820a60d3050ffe Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 9 Oct 2024 01:50:09 -0700 Subject: [PATCH 01/49] feat(Storage Transfer) : Addition of Sample and Test Case for storagetransfer_transfer_to_nearline. --- .../TransferToNearlineTest.cs | 67 ++++++++++++++++++ .../TransferToNearlineSample.cs | 68 +++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs new file mode 100644 index 00000000000..bf688b6eb7f --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -0,0 +1,67 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; +using System; +using Xunit; + +namespace StorageTransfer.Samples.Tests; + +[Collection(nameof(StorageFixture))] +public class TransferToNearlineTest : IDisposable +{ + private readonly StorageFixture _fixture; + private string _transferJobName; + public TransferToNearlineTest(StorageFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public void TestTransferToNearline() + { + TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(); + var storage = StorageClient.Create(); + var bucket = storage.GetBucket(_fixture.BucketNameSink); + string storageClass = StorageClasses.Nearline; + bucket.StorageClass = storageClass; + bucket = storage.UpdateBucket(bucket); + var transferJob = transferToNearlineSample.TransferToNearline(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs new file mode 100644 index 00000000000..c62ec5eb74d --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -0,0 +1,68 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_transfer_to_nearline] + +using Google.Cloud.StorageTransfer.V1; +using Google.Protobuf.WellKnownTypes; +using System; + + +namespace StorageTransfer.Samples +{ + + public class TransferToNearlineSample + { + /*Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more + than 30 days old to a Nearline GCS bucket.*/ + public TransferJob TransferToNearline( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The GCS bucket to transfer objects from + string sourceBucket = "my-source-bucket", + // The GCS Nearline bucket to transfer old objects to + string sinkBucket = "my-sink-bucket") + { + // A description of this job + string jobDescription = $"Transfers old objects from standard bucket ({sourceBucket}) that haven't been modified in the last 30 days to a Nearline bucket ({sinkBucket})"; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = sourceBucket }, + ObjectConditions = new ObjectConditions { MinTimeElapsedSinceLastModification = Duration.FromTimeSpan(TimeSpan.FromSeconds(2592000)) }, + TransferOptions = new TransferOptions { DeleteObjectsFromSourceAfterTransfer = true }, + }, + Status = TransferJob.Types.Status.Enabled, + Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } + }; + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); + + Console.WriteLine($"Created transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with name {response.Name}"); + return response; + } + } +} +// [END storagetransfer_transfer_to_nearline] From 36706a44a3919436d4c8515d19afc9436f167998 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 9 Oct 2024 02:17:01 -0700 Subject: [PATCH 02/49] feat(Storage Transfer) : Addition of Schedule End Date in Sample for storagetransfer_transfer_to_nearline. --- .../api/StorageTransfer.Samples/TransferToNearlineSample.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index c62ec5eb74d..7b50d3e015e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -48,7 +48,7 @@ public TransferJob TransferToNearline( TransferOptions = new TransferOptions { DeleteObjectsFromSourceAfterTransfer = true }, }, Status = TransferJob.Types.Status.Enabled, - Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } + Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)), ScheduleEndDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } }; // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); @@ -60,7 +60,7 @@ public TransferJob TransferToNearline( ProjectId = projectId }); - Console.WriteLine($"Created transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with name {response.Name}"); + Console.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with name {response.Name}"); return response; } } From afcba298ce660c521a296791ed6f1c28098f10a6 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 9 Oct 2024 02:17:01 -0700 Subject: [PATCH 03/49] samples(storage transfer) : Addition of schedule end date in sample for storagetransfer_transfer_to_nearline. --- .../api/StorageTransfer.Samples/TransferToNearlineSample.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index c62ec5eb74d..7b50d3e015e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -48,7 +48,7 @@ public TransferJob TransferToNearline( TransferOptions = new TransferOptions { DeleteObjectsFromSourceAfterTransfer = true }, }, Status = TransferJob.Types.Status.Enabled, - Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } + Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)), ScheduleEndDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } }; // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); @@ -60,7 +60,7 @@ public TransferJob TransferToNearline( ProjectId = projectId }); - Console.WriteLine($"Created transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with name {response.Name}"); + Console.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with name {response.Name}"); return response; } } From 3efcf764d5d7521dd8f60c335be6346bc889637d Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 10 Oct 2024 02:55:33 -0700 Subject: [PATCH 04/49] chore(deps): add dependency xunit.abstractions version 2.0.3 to see output message in sample output and related small changes. --- .../QuickstartTest.cs | 11 +++++++---- .../QuickstartSample.cs | 18 +++++++++++------- .../StorageTransfer.Samples.csproj | 1 + 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 30cac132e63..5b96b7519c6 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -1,5 +1,5 @@ -/** - * Copyright 2021 Google Inc. +/** + * Copyright 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ using System; using Google.Cloud.StorageTransfer.V1; using Xunit; +using Xunit.Abstractions; namespace StorageTransfer.Samples.Tests { @@ -25,16 +26,18 @@ public class QuickstartTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; - public QuickstartTest(StorageFixture fixture) + public QuickstartTest(StorageFixture fixture , ITestOutputHelper outputHelper) { _fixture = fixture; + _outputHelper = outputHelper; } [Fact] public void TestQuickstart() { - QuickstartSample quickstartSample = new QuickstartSample(); + QuickstartSample quickstartSample = new QuickstartSample(_outputHelper); var transferJob = quickstartSample.Quickstart(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index cc84bfe0620..68e60844d0d 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -1,5 +1,5 @@ -/** - * Copyright 2021 Google Inc. +/** + * Copyright 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,19 @@ * limitations under the License. */ - // [START storagetransfer_quickstart] -using System; using Google.Cloud.StorageTransfer.V1; +using Xunit.Abstractions; namespace StorageTransfer.Samples { public class QuickstartSample { + private readonly ITestOutputHelper _output; + public QuickstartSample(ITestOutputHelper output) + { + _output = output; + } public TransferJob Quickstart( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -36,8 +40,8 @@ public TransferJob Quickstart( ProjectId = projectId, TransferSpec = new TransferSpec { - GcsDataSink = new GcsData { BucketName = sourceBucket }, - GcsDataSource = new GcsData { BucketName = sinkBucket } + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = sourceBucket } }, Status = TransferJob.Types.Status.Enabled }; @@ -50,7 +54,7 @@ public TransferJob Quickstart( ProjectId = projectId }); - Console.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); + _output.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); return response; } diff --git a/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj b/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj index cb35bdd48fa..2b68373281f 100644 --- a/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj +++ b/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj @@ -6,5 +6,6 @@ + From 35e18a36bb880ab70a9205a7e732df3059c98cc3 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 10 Oct 2024 04:37:44 -0700 Subject: [PATCH 05/49] code changes in transfer to nearline related to capture message in standard output of test log summary. --- .../TransferToNearlineTest.cs | 7 +++++-- .../StorageTransfer.Samples/TransferToNearlineSample.cs | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index bf688b6eb7f..c3d1856f192 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -17,6 +17,7 @@ using Google.Cloud.StorageTransfer.V1; using System; using Xunit; +using Xunit.Abstractions; namespace StorageTransfer.Samples.Tests; @@ -25,15 +26,17 @@ public class TransferToNearlineTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - public TransferToNearlineTest(StorageFixture fixture) + private readonly ITestOutputHelper _outputHelper; + public TransferToNearlineTest(StorageFixture fixture, ITestOutputHelper outputHelper) { _fixture = fixture; + _outputHelper = outputHelper; } [Fact] public void TestTransferToNearline() { - TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(); + TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(_outputHelper); var storage = StorageClient.Create(); var bucket = storage.GetBucket(_fixture.BucketNameSink); string storageClass = StorageClasses.Nearline; diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 7b50d3e015e..2d7238eea76 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -16,6 +16,7 @@ using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; using System; +using Xunit.Abstractions; namespace StorageTransfer.Samples @@ -25,6 +26,11 @@ public class TransferToNearlineSample { /*Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more than 30 days old to a Nearline GCS bucket.*/ + private readonly ITestOutputHelper _output; + public TransferToNearlineSample(ITestOutputHelper output) + { + _output = output; + } public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -60,7 +66,7 @@ public TransferJob TransferToNearline( ProjectId = projectId }); - Console.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with name {response.Name}"); + _output.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with the name {response.Name}"); return response; } } From 0acb7fdb5d1a113ec95fbb0d20fc50126e6a953a Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 10 Oct 2024 07:14:02 -0700 Subject: [PATCH 06/49] feat(storage transfer) : addition of sample and test case to check latest transfer operation. --- .../CheckLatestTransferOperationTest.cs | 40 +++++++++++ .../StorageFixture.cs | 7 +- .../CheckLatestTransferOperationSample.cs | 70 +++++++++++++++++++ 3 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs new file mode 100644 index 00000000000..4790fc473b4 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -0,0 +1,40 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +using Xunit; +using Xunit.Abstractions; + +namespace StorageTransfer.Samples.Tests; + +[Collection(nameof(StorageFixture))] +public class CheckLatestTransferOperationTest +{ + + private readonly StorageFixture _fixture; + private string _jobName; + private readonly ITestOutputHelper _outputHelper; + public CheckLatestTransferOperationTest(StorageFixture fixture, ITestOutputHelper outputHelper) + { + _outputHelper = outputHelper; + _fixture = fixture; + } + + [Fact] + public void CheckLatestTransferOperation() + { + CheckLatestTransferOperationSample checkLatestTransferOperationSample = new CheckLatestTransferOperationSample(_outputHelper); + var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId,_fixture.JobName); + Assert.Contains("transferJobs/", transferJob.Name); + _jobName = transferJob.Name; + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 2938516a605..08e7c9ce72f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -1,4 +1,4 @@ -/** +/** * Copyright 2021 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,11 +28,16 @@ public class StorageFixture : IDisposable, ICollectionFixture public string ProjectId { get; } public string BucketNameSource { get; } = Guid.NewGuid().ToString(); public string BucketNameSink { get; } = Guid.NewGuid().ToString(); + public string JobName { get; } public StorageClient Storage { get; } = StorageClient.Create(); public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); public StorageFixture() { + // Instantiate random number generator + Random random = new Random(); + JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; + ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); if (string.IsNullOrWhiteSpace(ProjectId)) { diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs new file mode 100644 index 00000000000..c8f447e3b91 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -0,0 +1,70 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_get_latest_transfer_operation] + +using Google.Cloud.StorageTransfer.V1; +using System; +using Xunit.Abstractions; + +namespace StorageTransfer.Samples +{ + public class CheckLatestTransferOperationSample + { + private readonly ITestOutputHelper _output; + public CheckLatestTransferOperationSample(ITestOutputHelper output) + { + _output = output; + + } + //Checks the latest transfer operation for a given transfer job. + public TransferJob CheckLatestTransferOperation( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The name of the job to check + string jobName = "transferJobs/1234567890") + { + if(string.IsNullOrEmpty(jobName)) + { + throw new Exception("JobName can not be null or empty"); + } + // Create a Transfer Service client + StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.Create(); + + GetTransferJobRequest getTransferJobRequest = new GetTransferJobRequest { ProjectId = projectId, JobName = jobName }; + try + { + // Get Transfer job + TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); + // Get Latest operation name from tranfer job + string latestOperationName = transferJob.LatestOperationName; + + + if (!string.IsNullOrEmpty(latestOperationName)) + { + _output.WriteLine("The latest operation for transfer job " +jobName+ " is: " +latestOperationName+ ""); + } + else + { + _output.WriteLine("Transfer job "+ jobName +" hasn't run yet, try again once after job started running."); + } + return transferJob; + } + catch (Exception) + { + throw new Exception("Failed to get transfer job "+ jobName + ""); + } + } + } +} +// [END storagetransfer_get_latest_transfer_operation] From b5d9b10f9cbd0ad35d8e78a91b08e5a8ff1c9fc1 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Tue, 29 Oct 2024 06:14:42 -0700 Subject: [PATCH 07/49] storage transfer from POSIX file system to GCS bucket using manifest file --- .../StorageFixture.cs | 20 ++++- .../TransferUsingManifestTest.cs | 72 +++++++++++++++ .../TransferUsingManifestSample.cs | 89 +++++++++++++++++++ 3 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 08e7c9ce72f..9ba481fa175 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -29,7 +29,10 @@ public class StorageFixture : IDisposable, ICollectionFixture public string BucketNameSource { get; } = Guid.NewGuid().ToString(); public string BucketNameSink { get; } = Guid.NewGuid().ToString(); public string JobName { get; } + public string SourceAgentPoolName { get; } + public string RootDirectory { get; } = "/tmp/uploads"; public StorageClient Storage { get; } = StorageClient.Create(); + public string ManifestObjectName { get; } = "manifest.csv"; public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); public StorageFixture() @@ -37,6 +40,7 @@ public StorageFixture() // Instantiate random number generator Random random = new Random(); JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; + SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet"; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); if (string.IsNullOrWhiteSpace(ProjectId)) @@ -102,7 +106,13 @@ public void Dispose() } catch (Exception) { - // Do nothing, we delete on a best effort basis. + // If bucket is not empty, we delete on a best effort basis. + foreach (var storageObject in Storage.ListObjects(BucketNameSink, "")) + { + Storage.DeleteObject(BucketNameSink, storageObject.Name); + + } + Storage.DeleteBucket(BucketNameSink); } try { @@ -110,7 +120,13 @@ public void Dispose() } catch (Exception) { - // Do nothing, we delete on a best effort basis. + // If bucket is not empty, we delete on a best effort basis. + foreach (var storageObject in Storage.ListObjects(BucketNameSource, "")) + { + Storage.DeleteObject(BucketNameSource, storageObject.Name); + + } + Storage.DeleteBucket(BucketNameSource); } } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs new file mode 100644 index 00000000000..5cd2f4f150e --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -0,0 +1,72 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit; +using System.IO; + +namespace StorageTransfer.Samples.Tests; +[Collection(nameof(StorageFixture))] +public class TransferUsingManifestTest : IDisposable +{ + private readonly StorageFixture _fixture; + private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; + public TransferUsingManifestTest(StorageFixture fixture, ITestOutputHelper outputHelper) + { + _fixture = fixture; + _outputHelper = outputHelper; + } + + [Fact] + public void TransferUsingManifest() + { + TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(_outputHelper); + var storage = StorageClient.Create(); + byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + storage.UploadObject(_fixture.BucketNameSource, _fixture.ManifestObjectName, "application/octet-stream", stream); + var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.ManifestObjectName); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs new file mode 100644 index 00000000000..2f7b2102953 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -0,0 +1,89 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_manifest_request] +using Google.Cloud.StorageTransfer.V1; +using Google.Protobuf.WellKnownTypes; +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Xunit.Abstractions; + + +namespace StorageTransfer.Samples +{ + public class TransferUsingManifestSample + { + /*Create a transfer from a POSIX file system to a GCS bucket using + a manifest file*/ + private readonly ITestOutputHelper _output; + public TransferUsingManifestSample(ITestOutputHelper output) + { + _output = output; + } + public TransferJob TransferUsingManifest( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent + string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads", + // The GCS bucket which has your manifest file + string manifestBucket = "my-source-bucket", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket", + // The name of the manifest file in manifestBucket that specifies which objects to transfer + string manifestObjectName = "path/to/manifest.csv") + { + string manifestLocation = "gs://" + manifestBucket + "/" + manifestObjectName; + + // # A useful description for your transfer job + string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file"; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = manifestBucket }, + SourceAgentPoolName = sourceAgentPoolName, + PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, + TransferManifest = new TransferManifest { Location = manifestLocation } + }, + Status = TransferJob.Types.Status.Enabled, + }; + + + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); + + _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}"); + return response; + + + } + } +} +// [END storagetransfer_manifest_request] From 3c9d243bc0e425939be7b325e1d24e369c454829 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 30 Oct 2024 04:23:27 -0700 Subject: [PATCH 08/49] minor changes --- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 9 +++------ .../TransferUsingManifestTest.cs | 3 --- .../TransferUsingManifestSample.cs | 5 ----- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 9ba481fa175..feca9a4068a 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google Inc. + * Copyright 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,10 +39,9 @@ public StorageFixture() { // Instantiate random number generator Random random = new Random(); - JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; - SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet"; - + JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); + SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet"; if (string.IsNullOrWhiteSpace(ProjectId)) { throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id."); @@ -110,7 +109,6 @@ public void Dispose() foreach (var storageObject in Storage.ListObjects(BucketNameSink, "")) { Storage.DeleteObject(BucketNameSink, storageObject.Name); - } Storage.DeleteBucket(BucketNameSink); } @@ -124,7 +122,6 @@ public void Dispose() foreach (var storageObject in Storage.ListObjects(BucketNameSource, "")) { Storage.DeleteObject(BucketNameSource, storageObject.Name); - } Storage.DeleteBucket(BucketNameSource); } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 5cd2f4f150e..c19f33a4fd4 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -15,10 +15,7 @@ using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using System; -using System.Collections.Generic; -using System.Linq; using System.Text; -using System.Threading.Tasks; using Xunit.Abstractions; using Xunit; using System.IO; diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 2f7b2102953..5625a17177e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -13,11 +13,6 @@ // limitations under the License. // [START storagetransfer_manifest_request] using Google.Cloud.StorageTransfer.V1; -using Google.Protobuf.WellKnownTypes; -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; using Xunit.Abstractions; From 375f1e011f7447be278345dbdf78d7e842dee696 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 31 Oct 2024 09:28:27 +0000 Subject: [PATCH 09/49] addition of sample and test case for transfer from posix --- .../TransferFromPosixTest.cs | 65 ++++++++++++++++ .../TransferFromPosixSample.cs | 78 +++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs new file mode 100644 index 00000000000..6f512b11fc2 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -0,0 +1,65 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; +using System; +using Xunit.Abstractions; +using Xunit; + + +namespace StorageTransfer.Samples.Tests; +[Collection(nameof(StorageFixture))] +public class TransferFromPosixTest : IDisposable +{ + private readonly StorageFixture _fixture; + private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; + public TransferFromPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper) + { + _fixture = fixture; + _outputHelper = outputHelper; + } + + [Fact] + public void TransferFromPosix() + { + TransferFromPosixSample transferFromPosixSample = new TransferFromPosixSample(_outputHelper); + var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory,_fixture.BucketNameSink); + var storage = StorageClient.Create(); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs new file mode 100644 index 00000000000..6bf91095722 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -0,0 +1,78 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_transfer_from_posix] +using Google.Cloud.StorageTransfer.V1; +using Xunit.Abstractions; + + +namespace StorageTransfer.Samples +{ + public class TransferFromPosixSample + { + /*Create a transfer from a POSIX file system to a GCS sink bucket*/ + private readonly ITestOutputHelper _output; + public TransferFromPosixSample(ITestOutputHelper output) + { + _output = output; + } + public TransferJob TransferFromPosix( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent + string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket") + { + + // # A useful description for your transfer job + string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket})"; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + GcsDataSink = new GcsData { BucketName = sinkBucket }, + SourceAgentPoolName = sourceAgentPoolName, + PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory } + }, + Status = TransferJob.Types.Status.Enabled, + }; + + + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); + + _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} with the name {response.Name}"); + return response; + + + } + } +} +// [END storagetransfer_transfer_from_posix] + + From 139190a621408f413c9fcf01096d7320a23afaff Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 4 Nov 2024 11:10:29 +0000 Subject: [PATCH 10/49] added sample and test case for download to posix storage transfer service. --- .../DownloadToPosixTest.cs | 70 ++++++++++++++++ .../StorageFixture.cs | 6 +- .../DownloadToPosixSample.cs | 79 +++++++++++++++++++ 3 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs new file mode 100644 index 00000000000..f5aedff1291 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -0,0 +1,70 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; +using System; +using Xunit.Abstractions; +using Xunit; +using System.Text; +using System.IO; + + +namespace StorageTransfer.Samples.Tests; +[Collection(nameof(StorageFixture))] +public class DownloadToPosixTest : IDisposable +{ + private readonly StorageFixture _fixture; + private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; + public DownloadToPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper) + { + _fixture = fixture; + _outputHelper = outputHelper; + } + + [Fact] + public void DownloadToPosix() + { + DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(_outputHelper); + var storage = StorageClient.Create(); + byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + storage.UploadObject(_fixture.BucketNameSource,"DownloadToPosixTestFile", "application/octet-stream", stream); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.RootDirectory); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index feca9a4068a..4e19c17d07f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -30,6 +30,8 @@ public class StorageFixture : IDisposable, ICollectionFixture public string BucketNameSink { get; } = Guid.NewGuid().ToString(); public string JobName { get; } public string SourceAgentPoolName { get; } + public string SinkAgentPoolName { get; } + public string GcsSourcePath { get;} public string RootDirectory { get; } = "/tmp/uploads"; public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; @@ -41,7 +43,9 @@ public StorageFixture() Random random = new Random(); JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); - SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet"; + SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/source_test_dotnet"; + SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/sink_test_dotnet"; + GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id."); diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs new file mode 100644 index 00000000000..3a7e114d51a --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -0,0 +1,79 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_download_to_posix] +using Google.Cloud.StorageTransfer.V1; +using Xunit.Abstractions; + + +namespace StorageTransfer.Samples +{ + public class DownloadToPosixSample + { + /*Create a transfer from a GCS bucket to a POSIX file system.*/ + private readonly ITestOutputHelper _output; + public DownloadToPosixSample(ITestOutputHelper output) + { + _output = output; + } + public TransferJob DownloadToPosix( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent + string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // Your GCS source bucket name + string gcsSourceBucket = "my-gcs-source-bucket", + // An optional path on the Google Cloud Storage bucket to download from + string gcsSourcePath = "foo/bar/", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads") + { + // # A useful description for your transfer job + string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath}, + SinkAgentPoolName = sinkAgentPoolName, + PosixDataSink = new PosixFilesystem { RootDirectory = rootDirectory } + }, + Status = TransferJob.Types.Status.Enabled, + }; + + + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); + + _output.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); + return response; + + + } + } +} +//[END storagetransfer_download_to_posix] + + From bb66a633f26680d8916342851b25e17b3bd7b9b7 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Tue, 5 Nov 2024 11:58:18 +0000 Subject: [PATCH 11/49] code changes for creation of temp folder in posix file system --- .../StorageTransfer.Samples.Tests/DownloadToPosixTest.cs | 7 +++++-- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 8 +++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index f5aedff1291..9edf6dbf59b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -38,11 +38,13 @@ public DownloadToPosixTest(StorageFixture fixture, ITestOutputHelper outputHelpe public void DownloadToPosix() { DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(_outputHelper); + Directory.CreateDirectory(_fixture.TempDirectory); var storage = StorageClient.Create(); byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); - storage.UploadObject(_fixture.BucketNameSource,"DownloadToPosixTestFile", "application/octet-stream", stream); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.RootDirectory); + string fileName = $"{_fixture.GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; + storage.UploadObject(_fixture.BucketNameSource,fileName, "application/octet-stream", stream); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.TempDirectory); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -61,6 +63,7 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + Directory.Delete(_fixture.TempDirectory, true); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 4e19c17d07f..51019127d8c 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; +using System.IO; using Google.Apis.Storage.v1.Data; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; @@ -32,7 +33,8 @@ public class StorageFixture : IDisposable, ICollectionFixture public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } public string GcsSourcePath { get;} - public string RootDirectory { get; } = "/tmp/uploads"; + public string RootDirectory { get; } = System.IO.Path.GetTempPath(); + public string TempDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); @@ -43,8 +45,8 @@ public StorageFixture() Random random = new Random(); JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); - SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/source_test_dotnet"; - SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/sink_test_dotnet"; + SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; + SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { From 526b4452d1aa49cca75c7b9569a104116e1f698e Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 6 Nov 2024 07:02:48 +0000 Subject: [PATCH 12/49] couple of test checks added in download to posix --- .../StorageTransfer.Samples.Tests/DownloadToPosixTest.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 9edf6dbf59b..cb1abf8769e 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -19,6 +19,8 @@ using Xunit; using System.Text; using System.IO; +using System.Linq; +using System.Threading.Tasks; namespace StorageTransfer.Samples.Tests; @@ -43,9 +45,13 @@ public void DownloadToPosix() byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); string fileName = $"{_fixture.GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; + string filePath = $"{_fixture.TempDirectory}/{fileName.Split('/').Last()}"; storage.UploadObject(_fixture.BucketNameSource,fileName, "application/octet-stream", stream); var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.TempDirectory); Assert.Contains("transferJobs/", transferJob.Name); + Assert.True( Directory.Exists(_fixture.TempDirectory)); + System.Threading.Thread.Sleep(TimeSpan.FromSeconds(45)); + Assert.True( File.Exists(filePath)); _transferJobName = transferJob.Name; } From edc89e412bbd35cc9cebf9cdb7f1141e956be953 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 6 Nov 2024 13:58:32 +0000 Subject: [PATCH 13/49] added sample and test case for posix to posix storage transfer --- .../StorageFixture.cs | 2 + .../TransferBetweenPosixTest.cs | 88 +++++++++++++++++++ .../TransferBetweenPosixSample.cs | 82 +++++++++++++++++ 3 files changed, 172 insertions(+) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 51019127d8c..22b43f7ea8e 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -34,7 +34,9 @@ public class StorageFixture : IDisposable, ICollectionFixture public string SinkAgentPoolName { get; } public string GcsSourcePath { get;} public string RootDirectory { get; } = System.IO.Path.GetTempPath(); + public string DestinationDirectory { get; } = System.IO.Path.GetTempPath(); public string TempDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + public string TempDestinationDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs new file mode 100644 index 00000000000..e8fd90bc63b --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -0,0 +1,88 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; +using System; +using Xunit.Abstractions; +using Xunit; +using System.IO; +using System.Linq; + + +namespace StorageTransfer.Samples.Tests; +[Collection(nameof(StorageFixture))] +public class TransferBetweenPosixTest : IDisposable +{ + private readonly StorageFixture _fixture; + private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; + public TransferBetweenPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper) + { + _fixture = fixture; + _outputHelper = outputHelper; + } + + [Fact] + public void TransferBetweenPosix() + { + TransferBetweenPosixSample transferBetweenPosixSample = new TransferBetweenPosixSample(_outputHelper); + Directory.CreateDirectory(_fixture.TempDirectory); + Directory.CreateDirectory(_fixture.TempDestinationDirectory); + string sourceDir =_fixture.RootDirectory; + string[] txtList = Directory.GetFiles(sourceDir, "*.txt"); + // Copy one txt file. + foreach (string f in txtList) + { + // Remove path from the file name. + string fName = f.Split('/').Last(); + // Copy only one file from source directory to temp source directory. Will overwrite if the destination file already exists. + File.Copy(Path.Combine(sourceDir, fName), Path.Combine(_fixture.TempDirectory, fName), true); + break; + } + var storage = StorageClient.Create(); + var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId,_fixture.SourceAgentPoolName,_fixture.SinkAgentPoolName,_fixture.TempDirectory,_fixture.TempDestinationDirectory,_fixture.BucketNameSource); + Assert.Contains("transferJobs/", transferJob.Name); + Assert.True(Directory.Exists(_fixture.TempDirectory)); + Assert.True(Directory.Exists(_fixture.TempDestinationDirectory)); + Assert.True(File.Exists(txtList[0])); + System.Threading.Thread.Sleep(TimeSpan.FromSeconds(45)); + string destinationFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0,_fixture.RootDirectory.Length - 1),_fixture.TempDestinationDirectory); + Assert.True(File.Exists(destinationFilePath)); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + Directory.Delete(_fixture.TempDirectory, true); + Directory.Delete(_fixture.TempDestinationDirectory, true); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs new file mode 100644 index 00000000000..3cbc42dd7a0 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -0,0 +1,82 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_transfer_posix_to_posix] +using Google.Cloud.StorageTransfer.V1; +using Xunit.Abstractions; + + +namespace StorageTransfer.Samples +{ + public class TransferBetweenPosixSample + { + /*Creates a transfer between POSIX file systems.*/ + private readonly ITestOutputHelper _output; + public TransferBetweenPosixSample(ITestOutputHelper output) + { + _output = output; + } + public TransferJob TransferBetweenPosix( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent + string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent + string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads", + // The root directory path on the sink filesystem + string destinationDirectory = "/directory/to/transfer/sink", + // The name of GCS bucket for intermediate storage + string intermediate_bucket = "my-intermediate-bucket") + { + // # A useful description for your transfer job + string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + SourceAgentPoolName = sourceAgentPoolName, + SinkAgentPoolName = sinkAgentPoolName, + PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, + PosixDataSink = new PosixFilesystem { RootDirectory = destinationDirectory }, + GcsIntermediateDataLocation = new GcsData { BucketName = intermediate_bucket } + }, + Status = TransferJob.Types.Status.Enabled, + }; + + + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); + + _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {destinationDirectory} with the name {response.Name}"); + return response; + + } + } +} +//[END storagetransfer_transfer_posix_to_posix] + + From 7d100d4b8e5d433a8660caebdc7b5ab3e379b079 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 7 Nov 2024 11:26:48 +0000 Subject: [PATCH 14/49] one test check added in posix to posix storage transfer --- .../StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index e8fd90bc63b..43a2f86d888 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -54,13 +54,15 @@ public void TransferBetweenPosix() var storage = StorageClient.Create(); var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId,_fixture.SourceAgentPoolName,_fixture.SinkAgentPoolName,_fixture.TempDirectory,_fixture.TempDestinationDirectory,_fixture.BucketNameSource); Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; Assert.True(Directory.Exists(_fixture.TempDirectory)); Assert.True(Directory.Exists(_fixture.TempDestinationDirectory)); Assert.True(File.Exists(txtList[0])); + string sourceFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0,_fixture.RootDirectory.Length - 1),_fixture.TempDirectory); + Assert.True(File.Exists(sourceFilePath)); System.Threading.Thread.Sleep(TimeSpan.FromSeconds(45)); string destinationFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0,_fixture.RootDirectory.Length - 1),_fixture.TempDestinationDirectory); Assert.True(File.Exists(destinationFilePath)); - _transferJobName = transferJob.Name; } public void Dispose() From 4b8848067694d7634fe486a654d43c51adf64999 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 11 Nov 2024 10:35:52 +0000 Subject: [PATCH 15/49] add sample and test case for event driven transfer and few storage fixture changes --- .../CreateEventDrivenGcsTransferTest.cs | 67 +++++++++++++++ .../StorageFixture.cs | 85 +++++++++++++++++-- .../StorageTransfer.Samples.Tests.csproj | 1 + .../CreateEventDrivenGcsTransferSample.cs | 65 ++++++++++++++ 4 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs new file mode 100644 index 00000000000..a301d040476 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -0,0 +1,67 @@ +/** + * Copyright 2024 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using Google.Cloud.StorageTransfer.V1; +using Xunit; +using Xunit.Abstractions; + +namespace StorageTransfer.Samples.Tests +{ + [Collection(nameof(StorageFixture))] + public class CreateEventDrivenGcsTransferTest : IDisposable + { + private readonly StorageFixture _fixture; + private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; + + public CreateEventDrivenGcsTransferTest(StorageFixture fixture , ITestOutputHelper outputHelper) + { + _fixture = fixture; + _outputHelper = outputHelper; + } + + [Fact] + public void CreateEventDrivenGcsTransfer() + { + CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(_outputHelper); + var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.PubSubId); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 22b43f7ea8e..d1cb7fe2972 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -21,34 +21,41 @@ using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; +using Google.Cloud.PubSub.V1; +using Grpc.Core; namespace StorageTransfer.Samples.Tests { [CollectionDefinition(nameof(StorageFixture))] public class StorageFixture : IDisposable, ICollectionFixture - { + { public string ProjectId { get; } public string BucketNameSource { get; } = Guid.NewGuid().ToString(); public string BucketNameSink { get; } = Guid.NewGuid().ToString(); public string JobName { get; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } - public string GcsSourcePath { get;} + public string GcsSourcePath { get; } public string RootDirectory { get; } = System.IO.Path.GetTempPath(); public string DestinationDirectory { get; } = System.IO.Path.GetTempPath(); public string TempDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public string TempDestinationDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; + public string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); + public string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); + public string PubSubId { get; } public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); + + public SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); + + public PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); public StorageFixture() { - // Instantiate random number generator - Random random = new Random(); - JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; + PubSubId = "projects/" + ProjectId + "/subscriptions/" + SubscriptionId + ""; GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { @@ -57,6 +64,49 @@ public StorageFixture() CreateBucketAndGrantStsPermissions(BucketNameSink); CreateBucketAndGrantStsPermissions(BucketNameSource); + // Initialize request argument(s) + TransferJob transferJob = new TransferJob{ + ProjectId = ProjectId, + TransferSpec = new TransferSpec + { + GcsDataSink = new GcsData { BucketName = BucketNameSource }, + GcsDataSource = new GcsData { BucketName = BucketNameSink } + }, + Status = TransferJob.Types.Status.Enabled}; + CreateTransferJobRequest request = new CreateTransferJobRequest + { + TransferJob = transferJob + }; + // Make the request + TransferJob response = Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + JobName = response.Name; + string email = Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest() + { + ProjectId = ProjectId + }).AccountEmail; + string memberServiceAccount = "serviceAccount:" + email; + // Create subscription name + SubscriptionName subscriptionName = new SubscriptionName(ProjectId, SubscriptionId); + // Create topic name + TopicName topicName = new TopicName(ProjectId, TopicId); + // Create topic + PublisherClient.CreateTopic(topicName); + // Create subscription. + SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500); + var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy(); + policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount); + PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest + { + ResourceAsResourceName = topicName, + Policy = policyIamPolicyTopic + }); + var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy(); + policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount); + PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest + { + ResourceAsResourceName = subscriptionName, + Policy = policyIamPolicySubscriber + }); } private void CreateBucketAndGrantStsPermissions(string bucketName) @@ -74,7 +124,7 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) string objectViewer = "roles/storage.objectViewer"; string bucketReader = "roles/storage.legacyBucketReader"; string bucketWriter = "roles/storage.legacyBucketWriter"; - + var policy = Storage.GetBucketIamPolicy(bucketName, new GetBucketIamPolicyOptions { RequestedPolicyVersion = 3 @@ -97,12 +147,14 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) Role = bucketWriter, Members = new List { member } }; + policy.Bindings.Add(objectViewerBinding); policy.Bindings.Add(bucketReaderBinding); policy.Bindings.Add(bucketWriterBinding); Storage.SetBucketIamPolicy(bucketName, policy); + } public void Dispose() @@ -133,6 +185,27 @@ public void Dispose() } Storage.DeleteBucket(BucketNameSource); } + + try + { + TopicName topicName = TopicName.FromProjectTopic(ProjectId,TopicId); + PublisherClient.DeleteTopic(topicName); + } + catch (RpcException ex) + { + throw new Exception ($"Exception occur while deleting Topic {TopicId} Exception: {ex}"); + } + + try + { + SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(ProjectId, SubscriptionId); + SubscriberClient.DeleteSubscription(subscriptionName); + } + catch (RpcException ex) + { + throw new Exception($"Exception occur while deleting subscription {SubscriptionId} Exception: {ex}"); + } + } } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageTransfer.Samples.Tests.csproj b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageTransfer.Samples.Tests.csproj index 4991e2fa1a4..38402a526aa 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageTransfer.Samples.Tests.csproj +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageTransfer.Samples.Tests.csproj @@ -16,6 +16,7 @@ + \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs new file mode 100644 index 00000000000..d865d532468 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -0,0 +1,65 @@ +/** + * Copyright 2024 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// [START storagetransfer_create_event_driven_gcs_transfer] +using Google.Cloud.StorageTransfer.V1; +using Xunit.Abstractions; + +namespace StorageTransfer.Samples +{ + public class CreateEventDrivenGcsTransferSample + { + private readonly ITestOutputHelper _output; + public CreateEventDrivenGcsTransferSample(ITestOutputHelper output) + { + _output = output; + } + public TransferJob CreateEventDrivenGcsTransfer( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The GCS bucket to transfer data from + string sourceBucket = "my-source-bucket", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket", + // The subscription ID to a Pubsub queue to track + string pubSubId = "projects/PROJECT_NAME/subscriptions/SUBSCRIPTION_ID") + { + // # A useful description for your transfer job + string jobDescription = $"Event driven gcs data transfer from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} "; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = sourceBucket }, + + }, + Status = TransferJob.Types.Status.Enabled, + EventStream = new EventStream { Name = pubSubId } + }; + + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + _output.WriteLine($"Created an event driven transfer job from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} with name {response.Name}"); + + return response; + } + } +} +// [END storagetransfer_create_event_driven_gcs_transfer] + From effd0c2ed5a67be5ed46d64fc71de255970b7d71 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Tue, 12 Nov 2024 03:59:58 -0800 Subject: [PATCH 16/49] samples(storage transfer): code changes for printing messages to console , rewording , removing some unnecessary symbols, few test checks removed related to posix file transfer. --- .../CheckLatestTransferOperationTest.cs | 9 ++-- .../CreateEventDrivenGcsTransferTest.cs | 10 ++--- .../DownloadToPosixTest.cs | 21 ++++------ .../QuickstartTest.cs | 10 ++--- .../StorageFixture.cs | 41 ++++++++++--------- .../TransferBetweenPosixTest.cs | 19 +++------ .../TransferFromPosixTest.cs | 10 ++--- .../TransferToNearlineTest.cs | 8 +--- .../TransferUsingManifestTest.cs | 9 ++-- .../CheckLatestTransferOperationSample.cs | 31 ++++++-------- .../CreateEventDrivenGcsTransferSample.cs | 14 +++---- .../DownloadToPosixSample.cs | 12 ++---- .../QuickstartSample.cs | 9 +--- .../TransferBetweenPosixSample.cs | 12 ++---- .../TransferFromPosixSample.cs | 14 ++----- .../TransferToNearlineSample.cs | 9 +--- .../TransferUsingManifestSample.cs | 12 ++---- 17 files changed, 85 insertions(+), 165 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index 4790fc473b4..39d3dfce59e 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. using Xunit; -using Xunit.Abstractions; namespace StorageTransfer.Samples.Tests; @@ -22,18 +21,16 @@ public class CheckLatestTransferOperationTest private readonly StorageFixture _fixture; private string _jobName; - private readonly ITestOutputHelper _outputHelper; - public CheckLatestTransferOperationTest(StorageFixture fixture, ITestOutputHelper outputHelper) + public CheckLatestTransferOperationTest(StorageFixture fixture) { - _outputHelper = outputHelper; _fixture = fixture; } [Fact] public void CheckLatestTransferOperation() { - CheckLatestTransferOperationSample checkLatestTransferOperationSample = new CheckLatestTransferOperationSample(_outputHelper); - var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId,_fixture.JobName); + CheckLatestTransferOperationSample checkLatestTransferOperationSample = new CheckLatestTransferOperationSample(); + var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId, _fixture.JobName); Assert.Contains("transferJobs/", transferJob.Name); _jobName = transferJob.Name; } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index a301d040476..af042168a98 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -14,10 +14,9 @@ * limitations under the License. */ -using System; using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; -using Xunit.Abstractions; namespace StorageTransfer.Samples.Tests { @@ -26,18 +25,15 @@ public class CreateEventDrivenGcsTransferTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - - public CreateEventDrivenGcsTransferTest(StorageFixture fixture , ITestOutputHelper outputHelper) + public CreateEventDrivenGcsTransferTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void CreateEventDrivenGcsTransfer() { - CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(_outputHelper); + CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.PubSubId); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index cb1abf8769e..95a7ef904cf 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -15,13 +15,11 @@ using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using System; -using Xunit.Abstractions; -using Xunit; -using System.Text; using System.IO; using System.Linq; +using System.Text; using System.Threading.Tasks; - +using Xunit; namespace StorageTransfer.Samples.Tests; [Collection(nameof(StorageFixture))] @@ -29,29 +27,24 @@ public class DownloadToPosixTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - public DownloadToPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper) + public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void DownloadToPosix() { - DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(_outputHelper); + DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(); Directory.CreateDirectory(_fixture.TempDirectory); var storage = StorageClient.Create(); byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); string fileName = $"{_fixture.GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; - string filePath = $"{_fixture.TempDirectory}/{fileName.Split('/').Last()}"; - storage.UploadObject(_fixture.BucketNameSource,fileName, "application/octet-stream", stream); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.TempDirectory); + storage.UploadObject(_fixture.BucketNameSource, fileName, "application/octet-stream", stream); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNameSource, _fixture.GcsSourcePath, _fixture.TempDirectory); Assert.Contains("transferJobs/", transferJob.Name); - Assert.True( Directory.Exists(_fixture.TempDirectory)); - System.Threading.Thread.Sleep(TimeSpan.FromSeconds(45)); - Assert.True( File.Exists(filePath)); + Assert.True(Directory.Exists(_fixture.TempDirectory)); _transferJobName = transferJob.Name; } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 5b96b7519c6..e826771b11b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -14,10 +14,9 @@ * limitations under the License. */ -using System; using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; -using Xunit.Abstractions; namespace StorageTransfer.Samples.Tests { @@ -26,18 +25,15 @@ public class QuickstartTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - - public QuickstartTest(StorageFixture fixture , ITestOutputHelper outputHelper) + public QuickstartTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void TestQuickstart() { - QuickstartSample quickstartSample = new QuickstartSample(_outputHelper); + QuickstartSample quickstartSample = new QuickstartSample(); var transferJob = quickstartSample.Quickstart(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index d1cb7fe2972..e9853e71828 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -14,20 +14,21 @@ * limitations under the License. */ -using System; -using System.Collections.Generic; -using System.IO; using Google.Apis.Storage.v1.Data; +using Google.Cloud.PubSub.V1; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; -using Xunit; -using Google.Cloud.PubSub.V1; using Grpc.Core; +using System; +using System.Collections.Generic; +using System.IO; +using Xunit; + namespace StorageTransfer.Samples.Tests { [CollectionDefinition(nameof(StorageFixture))] public class StorageFixture : IDisposable, ICollectionFixture - { + { public string ProjectId { get; } public string BucketNameSource { get; } = Guid.NewGuid().ToString(); public string BucketNameSink { get; } = Guid.NewGuid().ToString(); @@ -45,9 +46,9 @@ public class StorageFixture : IDisposable, ICollectionFixture public string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); public string PubSubId { get; } public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); - + public SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); - + public PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); public StorageFixture() @@ -65,14 +66,16 @@ public StorageFixture() CreateBucketAndGrantStsPermissions(BucketNameSink); CreateBucketAndGrantStsPermissions(BucketNameSource); // Initialize request argument(s) - TransferJob transferJob = new TransferJob{ - ProjectId = ProjectId, - TransferSpec = new TransferSpec + TransferJob transferJob = new TransferJob + { + ProjectId = ProjectId, + TransferSpec = new TransferSpec { GcsDataSink = new GcsData { BucketName = BucketNameSource }, GcsDataSource = new GcsData { BucketName = BucketNameSink } }, - Status = TransferJob.Types.Status.Enabled}; + Status = TransferJob.Types.Status.Enabled + }; CreateTransferJobRequest request = new CreateTransferJobRequest { TransferJob = transferJob @@ -124,7 +127,7 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) string objectViewer = "roles/storage.objectViewer"; string bucketReader = "roles/storage.legacyBucketReader"; string bucketWriter = "roles/storage.legacyBucketWriter"; - + var policy = Storage.GetBucketIamPolicy(bucketName, new GetBucketIamPolicyOptions { RequestedPolicyVersion = 3 @@ -147,14 +150,14 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) Role = bucketWriter, Members = new List { member } }; - + policy.Bindings.Add(objectViewerBinding); policy.Bindings.Add(bucketReaderBinding); policy.Bindings.Add(bucketWriterBinding); Storage.SetBucketIamPolicy(bucketName, policy); - + } public void Dispose() @@ -185,15 +188,15 @@ public void Dispose() } Storage.DeleteBucket(BucketNameSource); } - + try { - TopicName topicName = TopicName.FromProjectTopic(ProjectId,TopicId); + TopicName topicName = TopicName.FromProjectTopic(ProjectId, TopicId); PublisherClient.DeleteTopic(topicName); } catch (RpcException ex) { - throw new Exception ($"Exception occur while deleting Topic {TopicId} Exception: {ex}"); + throw new Exception($"Exception occur while deleting Topic {TopicId} Exception: {ex}"); } try @@ -205,7 +208,7 @@ public void Dispose() { throw new Exception($"Exception occur while deleting subscription {SubscriptionId} Exception: {ex}"); } - + } } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 43a2f86d888..52fba525b7f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -15,11 +15,9 @@ using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using System; -using Xunit.Abstractions; -using Xunit; using System.IO; using System.Linq; - +using Xunit; namespace StorageTransfer.Samples.Tests; [Collection(nameof(StorageFixture))] @@ -27,20 +25,18 @@ public class TransferBetweenPosixTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - public TransferBetweenPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper) + public TransferBetweenPosixTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void TransferBetweenPosix() { - TransferBetweenPosixSample transferBetweenPosixSample = new TransferBetweenPosixSample(_outputHelper); + TransferBetweenPosixSample transferBetweenPosixSample = new TransferBetweenPosixSample(); Directory.CreateDirectory(_fixture.TempDirectory); Directory.CreateDirectory(_fixture.TempDestinationDirectory); - string sourceDir =_fixture.RootDirectory; + string sourceDir = _fixture.RootDirectory; string[] txtList = Directory.GetFiles(sourceDir, "*.txt"); // Copy one txt file. foreach (string f in txtList) @@ -52,17 +48,14 @@ public void TransferBetweenPosix() break; } var storage = StorageClient.Create(); - var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId,_fixture.SourceAgentPoolName,_fixture.SinkAgentPoolName,_fixture.TempDirectory,_fixture.TempDestinationDirectory,_fixture.BucketNameSource); + var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _fixture.TempDirectory, _fixture.TempDestinationDirectory, _fixture.BucketNameSource); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; Assert.True(Directory.Exists(_fixture.TempDirectory)); Assert.True(Directory.Exists(_fixture.TempDestinationDirectory)); Assert.True(File.Exists(txtList[0])); - string sourceFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0,_fixture.RootDirectory.Length - 1),_fixture.TempDirectory); + string sourceFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0, _fixture.RootDirectory.Length - 1), _fixture.TempDirectory); Assert.True(File.Exists(sourceFilePath)); - System.Threading.Thread.Sleep(TimeSpan.FromSeconds(45)); - string destinationFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0,_fixture.RootDirectory.Length - 1),_fixture.TempDestinationDirectory); - Assert.True(File.Exists(destinationFilePath)); } public void Dispose() diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 6f512b11fc2..d6ab8163804 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -15,28 +15,24 @@ using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using System; -using Xunit.Abstractions; using Xunit; - namespace StorageTransfer.Samples.Tests; [Collection(nameof(StorageFixture))] public class TransferFromPosixTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - public TransferFromPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper) + public TransferFromPosixTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void TransferFromPosix() { - TransferFromPosixSample transferFromPosixSample = new TransferFromPosixSample(_outputHelper); - var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory,_fixture.BucketNameSink); + TransferFromPosixSample transferFromPosixSample = new TransferFromPosixSample(); + var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameSink); var storage = StorageClient.Create(); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index c3d1856f192..b03cdb68c12 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -17,26 +17,22 @@ using Google.Cloud.StorageTransfer.V1; using System; using Xunit; -using Xunit.Abstractions; namespace StorageTransfer.Samples.Tests; - [Collection(nameof(StorageFixture))] public class TransferToNearlineTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - public TransferToNearlineTest(StorageFixture fixture, ITestOutputHelper outputHelper) + public TransferToNearlineTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void TestTransferToNearline() { - TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(_outputHelper); + TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(); var storage = StorageClient.Create(); var bucket = storage.GetBucket(_fixture.BucketNameSink); string storageClass = StorageClasses.Nearline; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index c19f33a4fd4..66fde006ad2 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -15,10 +15,9 @@ using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using System; +using System.IO; using System.Text; -using Xunit.Abstractions; using Xunit; -using System.IO; namespace StorageTransfer.Samples.Tests; [Collection(nameof(StorageFixture))] @@ -26,17 +25,15 @@ public class TransferUsingManifestTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - public TransferUsingManifestTest(StorageFixture fixture, ITestOutputHelper outputHelper) + public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void TransferUsingManifest() { - TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(_outputHelper); + TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(); var storage = StorageClient.Create(); byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index c8f447e3b91..9a7e7a369c7 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -15,26 +15,19 @@ using Google.Cloud.StorageTransfer.V1; using System; -using Xunit.Abstractions; namespace StorageTransfer.Samples { public class CheckLatestTransferOperationSample { - private readonly ITestOutputHelper _output; - public CheckLatestTransferOperationSample(ITestOutputHelper output) + //Checks the latest transfer operation for a given transfer job. + public TransferJob CheckLatestTransferOperation( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The name of the job to check + string jobName = "transferJobs/1234567890") { - _output = output; - - } - //Checks the latest transfer operation for a given transfer job. - public TransferJob CheckLatestTransferOperation( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The name of the job to check - string jobName = "transferJobs/1234567890") - { - if(string.IsNullOrEmpty(jobName)) + if (string.IsNullOrEmpty(jobName)) { throw new Exception("JobName can not be null or empty"); } @@ -46,23 +39,23 @@ public TransferJob CheckLatestTransferOperation( { // Get Transfer job TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); - // Get Latest operation name from tranfer job + // Get Latest operation name from transfer job string latestOperationName = transferJob.LatestOperationName; - + if (!string.IsNullOrEmpty(latestOperationName)) { - _output.WriteLine("The latest operation for transfer job " +jobName+ " is: " +latestOperationName+ ""); + Console.WriteLine("The latest operation for transfer job " + jobName + " is: " + latestOperationName + ""); } else { - _output.WriteLine("Transfer job "+ jobName +" hasn't run yet, try again once after job started running."); + Console.WriteLine("Transfer job " + jobName + " hasn't run yet, try again after the job has started running."); } return transferJob; } catch (Exception) { - throw new Exception("Failed to get transfer job "+ jobName + ""); + throw new Exception("Failed to get transfer job " + jobName + ""); } } } diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index d865d532468..30e80f3281b 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -15,17 +15,13 @@ */ // [START storagetransfer_create_event_driven_gcs_transfer] using Google.Cloud.StorageTransfer.V1; -using Xunit.Abstractions; +using System; namespace StorageTransfer.Samples { public class CreateEventDrivenGcsTransferSample { - private readonly ITestOutputHelper _output; - public CreateEventDrivenGcsTransferSample(ITestOutputHelper output) - { - _output = output; - } + public TransferJob CreateEventDrivenGcsTransfer( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -36,7 +32,7 @@ public TransferJob CreateEventDrivenGcsTransfer( // The subscription ID to a Pubsub queue to track string pubSubId = "projects/PROJECT_NAME/subscriptions/SUBSCRIPTION_ID") { - // # A useful description for your transfer job + // A useful description for your transfer job string jobDescription = $"Event driven gcs data transfer from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} "; TransferJob transferJob = new TransferJob @@ -47,7 +43,7 @@ public TransferJob CreateEventDrivenGcsTransfer( { GcsDataSink = new GcsData { BucketName = sinkBucket }, GcsDataSource = new GcsData { BucketName = sourceBucket }, - + }, Status = TransferJob.Types.Status.Enabled, EventStream = new EventStream { Name = pubSubId } @@ -55,7 +51,7 @@ public TransferJob CreateEventDrivenGcsTransfer( StorageTransferServiceClient client = StorageTransferServiceClient.Create(); TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - _output.WriteLine($"Created an event driven transfer job from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} with name {response.Name}"); + Console.WriteLine($"Created an event driven transfer job from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} with name {response.Name}"); return response; } diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 3a7e114d51a..4161ebf8304 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -13,19 +13,13 @@ // limitations under the License. // [START storagetransfer_download_to_posix] using Google.Cloud.StorageTransfer.V1; -using Xunit.Abstractions; - +using System; namespace StorageTransfer.Samples { public class DownloadToPosixSample { /*Create a transfer from a GCS bucket to a POSIX file system.*/ - private readonly ITestOutputHelper _output; - public DownloadToPosixSample(ITestOutputHelper output) - { - _output = output; - } public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -47,7 +41,7 @@ public TransferJob DownloadToPosix( Description = jobDescription, TransferSpec = new TransferSpec { - GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath}, + GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath }, SinkAgentPoolName = sinkAgentPoolName, PosixDataSink = new PosixFilesystem { RootDirectory = rootDirectory } }, @@ -67,7 +61,7 @@ public TransferJob DownloadToPosix( ProjectId = projectId }); - _output.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); + Console.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); return response; diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index 68e60844d0d..b69fe4b2c28 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -16,17 +16,12 @@ // [START storagetransfer_quickstart] using Google.Cloud.StorageTransfer.V1; -using Xunit.Abstractions; +using System; namespace StorageTransfer.Samples { public class QuickstartSample { - private readonly ITestOutputHelper _output; - public QuickstartSample(ITestOutputHelper output) - { - _output = output; - } public TransferJob Quickstart( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -54,7 +49,7 @@ public TransferJob Quickstart( ProjectId = projectId }); - _output.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); + Console.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); return response; } diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 3cbc42dd7a0..7fa63a9b666 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -13,19 +13,13 @@ // limitations under the License. // [START storagetransfer_transfer_posix_to_posix] using Google.Cloud.StorageTransfer.V1; -using Xunit.Abstractions; - +using System; namespace StorageTransfer.Samples { public class TransferBetweenPosixSample { /*Creates a transfer between POSIX file systems.*/ - private readonly ITestOutputHelper _output; - public TransferBetweenPosixSample(ITestOutputHelper output) - { - _output = output; - } public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -40,7 +34,7 @@ public TransferJob TransferBetweenPosix( // The name of GCS bucket for intermediate storage string intermediate_bucket = "my-intermediate-bucket") { - // # A useful description for your transfer job + // A useful description for your transfer job string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; TransferJob transferJob = new TransferJob @@ -71,7 +65,7 @@ public TransferJob TransferBetweenPosix( ProjectId = projectId }); - _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {destinationDirectory} with the name {response.Name}"); + Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {destinationDirectory} with the name {response.Name}"); return response; } diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 6bf91095722..7cc1e600028 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -13,19 +13,13 @@ // limitations under the License. // [START storagetransfer_transfer_from_posix] using Google.Cloud.StorageTransfer.V1; -using Xunit.Abstractions; - +using System; namespace StorageTransfer.Samples { public class TransferFromPosixSample { /*Create a transfer from a POSIX file system to a GCS sink bucket*/ - private readonly ITestOutputHelper _output; - public TransferFromPosixSample(ITestOutputHelper output) - { - _output = output; - } public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -36,8 +30,8 @@ public TransferJob TransferFromPosix( // The GCS bucket to transfer data to string sinkBucket = "my-sink-bucket") { - - // # A useful description for your transfer job + + // A useful description for your transfer job string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket})"; TransferJob transferJob = new TransferJob @@ -66,7 +60,7 @@ public TransferJob TransferFromPosix( ProjectId = projectId }); - _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} with the name {response.Name}"); + Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} with the name {response.Name}"); return response; diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 2d7238eea76..83c4dcc0858 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -16,8 +16,6 @@ using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; using System; -using Xunit.Abstractions; - namespace StorageTransfer.Samples { @@ -26,11 +24,6 @@ public class TransferToNearlineSample { /*Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more than 30 days old to a Nearline GCS bucket.*/ - private readonly ITestOutputHelper _output; - public TransferToNearlineSample(ITestOutputHelper output) - { - _output = output; - } public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -66,7 +59,7 @@ public TransferJob TransferToNearline( ProjectId = projectId }); - _output.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with the name {response.Name}"); + Console.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with the name {response.Name}"); return response; } } diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 5625a17177e..4f4d26168fd 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -13,8 +13,7 @@ // limitations under the License. // [START storagetransfer_manifest_request] using Google.Cloud.StorageTransfer.V1; -using Xunit.Abstractions; - +using System; namespace StorageTransfer.Samples { @@ -22,11 +21,6 @@ public class TransferUsingManifestSample { /*Create a transfer from a POSIX file system to a GCS bucket using a manifest file*/ - private readonly ITestOutputHelper _output; - public TransferUsingManifestSample(ITestOutputHelper output) - { - _output = output; - } public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -43,7 +37,7 @@ public TransferJob TransferUsingManifest( { string manifestLocation = "gs://" + manifestBucket + "/" + manifestObjectName; - // # A useful description for your transfer job + // A useful description for your transfer job string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file"; TransferJob transferJob = new TransferJob @@ -74,7 +68,7 @@ public TransferJob TransferUsingManifest( ProjectId = projectId }); - _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}"); + Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}"); return response; From f5d8252eb0b6984f5681cc296a3817c8a91824f7 Mon Sep 17 00:00:00 2001 From: Mahendra Date: Thu, 19 Dec 2024 14:49:12 +0530 Subject: [PATCH 17/49] Update storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs Co-authored-by: Amanda Tarafa Mas <14878252+amanda-tarafa@users.noreply.github.com> --- .../api/StorageTransfer.Samples/TransferUsingManifestSample.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 4f4d26168fd..35fa1c5641b 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + // [START storagetransfer_manifest_request] using Google.Cloud.StorageTransfer.V1; using System; From be9c4bb6bb44ef03481d4040469051b26c58b0dd Mon Sep 17 00:00:00 2001 From: Mahendra Date: Thu, 19 Dec 2024 14:53:55 +0530 Subject: [PATCH 18/49] Update storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs Co-authored-by: Amanda Tarafa Mas <14878252+amanda-tarafa@users.noreply.github.com> --- .../StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 66fde006ad2..391274b245b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -20,6 +20,7 @@ using Xunit; namespace StorageTransfer.Samples.Tests; + [Collection(nameof(StorageFixture))] public class TransferUsingManifestTest : IDisposable { From be748187bf23c8af8aae45bd7e8865a2c442f23c Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 19 Dec 2024 01:58:32 -0800 Subject: [PATCH 19/49] code refactoring changes --- .../CreateEventDrivenGcsTransferTest.cs | 2 +- .../DownloadToPosixTest.cs | 4 +-- .../QuickstartTest.cs | 5 ++- .../StorageFixture.cs | 33 ++++++------------- .../TransferBetweenPosixTest.cs | 4 +-- .../TransferFromPosixTest.cs | 2 +- .../TransferToNearlineTest.cs | 3 +- .../TransferUsingManifestTest.cs | 4 +-- .../CheckLatestTransferOperationSample.cs | 2 +- .../CreateEventDrivenGcsTransferSample.cs | 3 +- .../DownloadToPosixSample.cs | 3 +- .../QuickstartSample.cs | 7 ++-- .../StorageTransfer.Samples.csproj | 1 - .../TransferBetweenPosixSample.cs | 3 +- .../TransferFromPosixSample.cs | 3 +- .../TransferToNearlineSample.cs | 2 +- .../TransferUsingManifestSample.cs | 5 +-- 17 files changed, 38 insertions(+), 48 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index af042168a98..7e3f2cb1724 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -14,8 +14,8 @@ * limitations under the License. */ -using Google.Cloud.StorageTransfer.V1; using System; +using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 95a7ef904cf..0cbab538ccf 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Google.Cloud.Storage.V1; -using Google.Cloud.StorageTransfer.V1; using System; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index e826771b11b..b72b1900c5f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google Inc. + * Copyright 2021 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -using Google.Cloud.StorageTransfer.V1; using System; +using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index e9853e71828..e2e06260b91 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -150,53 +150,40 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) Role = bucketWriter, Members = new List { member } }; - - policy.Bindings.Add(objectViewerBinding); policy.Bindings.Add(bucketReaderBinding); policy.Bindings.Add(bucketWriterBinding); - Storage.SetBucketIamPolicy(bucketName, policy); - } public void Dispose() { try { - Storage.DeleteBucket(BucketNameSink); + Storage.DeleteBucket(BucketNameSink, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) { - // If bucket is not empty, we delete on a best effort basis. - foreach (var storageObject in Storage.ListObjects(BucketNameSink, "")) - { - Storage.DeleteObject(BucketNameSink, storageObject.Name); - } - Storage.DeleteBucket(BucketNameSink); + // Do nothing, we delete on a best effort basis. + } try { - Storage.DeleteBucket(BucketNameSource); + Storage.DeleteBucket(BucketNameSource, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) { - // If bucket is not empty, we delete on a best effort basis. - foreach (var storageObject in Storage.ListObjects(BucketNameSource, "")) - { - Storage.DeleteObject(BucketNameSource, storageObject.Name); - } - Storage.DeleteBucket(BucketNameSource); - } + // Do nothing, we delete on a best effort basis. + } try { TopicName topicName = TopicName.FromProjectTopic(ProjectId, TopicId); PublisherClient.DeleteTopic(topicName); } - catch (RpcException ex) + catch (RpcException) { - throw new Exception($"Exception occur while deleting Topic {TopicId} Exception: {ex}"); + // Do nothing, we delete on a best effort basis. } try @@ -204,9 +191,9 @@ public void Dispose() SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(ProjectId, SubscriptionId); SubscriberClient.DeleteSubscription(subscriptionName); } - catch (RpcException ex) + catch (RpcException) { - throw new Exception($"Exception occur while deleting subscription {SubscriptionId} Exception: {ex}"); + // Do nothing, we delete on a best effort basis. } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 52fba525b7f..02d941cabc1 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Google.Cloud.Storage.V1; -using Google.Cloud.StorageTransfer.V1; using System; using System.IO; using System.Linq; +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index d6ab8163804..f9f0d3e68c9 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +using System; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; -using System; using Xunit; namespace StorageTransfer.Samples.Tests; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index b03cdb68c12..6fe8e41abcb 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -12,10 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. - +using System; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; -using System; using Xunit; namespace StorageTransfer.Samples.Tests; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 66fde006ad2..96910225ac7 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Google.Cloud.Storage.V1; -using Google.Cloud.StorageTransfer.V1; using System; using System.IO; using System.Text; +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 9a7e7a369c7..e6c7614d227 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -13,8 +13,8 @@ // limitations under the License. // [START storagetransfer_get_latest_transfer_operation] -using Google.Cloud.StorageTransfer.V1; using System; +using Google.Cloud.StorageTransfer.V1; namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 30e80f3281b..717d3ed6207 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -14,8 +14,9 @@ * limitations under the License. */ // [START storagetransfer_create_event_driven_gcs_transfer] -using Google.Cloud.StorageTransfer.V1; + using System; +using Google.Cloud.StorageTransfer.V1; namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 4161ebf8304..719c3553ce6 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. // [START storagetransfer_download_to_posix] -using Google.Cloud.StorageTransfer.V1; + using System; +using Google.Cloud.StorageTransfer.V1; namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index b69fe4b2c28..3adcc527053 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google Inc. + * Copyright 2021 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,10 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - // [START storagetransfer_quickstart] -using Google.Cloud.StorageTransfer.V1; + using System; +using Google.Cloud.StorageTransfer.V1; + namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj b/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj index 2b68373281f..cb35bdd48fa 100644 --- a/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj +++ b/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj @@ -6,6 +6,5 @@ - diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 7fa63a9b666..6dcf7d9ea3c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. // [START storagetransfer_transfer_posix_to_posix] -using Google.Cloud.StorageTransfer.V1; + using System; +using Google.Cloud.StorageTransfer.V1; namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 7cc1e600028..69af8cad20a 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. // [START storagetransfer_transfer_from_posix] -using Google.Cloud.StorageTransfer.V1; + using System; +using Google.Cloud.StorageTransfer.V1; namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 83c4dcc0858..3f9ad9b655c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -13,9 +13,9 @@ // limitations under the License. // [START storagetransfer_transfer_to_nearline] +using System; using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; -using System; namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 4f4d26168fd..5ef812cb7ef 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. // [START storagetransfer_manifest_request] -using Google.Cloud.StorageTransfer.V1; + using System; +using Google.Cloud.StorageTransfer.V1; namespace StorageTransfer.Samples { @@ -35,7 +36,7 @@ public TransferJob TransferUsingManifest( // The name of the manifest file in manifestBucket that specifies which objects to transfer string manifestObjectName = "path/to/manifest.csv") { - string manifestLocation = "gs://" + manifestBucket + "/" + manifestObjectName; + string manifestLocation = $"gs://{manifestBucket}/{manifestObjectName}"; // A useful description for your transfer job string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file"; From 4b5d2e555bae5e091681cf9e8196a91dc7f9bafa Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 19 Dec 2024 04:27:48 -0800 Subject: [PATCH 20/49] code changes --- .../DownloadToPosixTest.cs | 7 +--- .../StorageFixture.cs | 35 +++++++++++++++---- .../TransferBetweenPosixTest.cs | 22 ++++++------ .../TransferUsingManifestTest.cs | 8 +---- .../CheckLatestTransferOperationSample.cs | 4 +-- .../CreateEventDrivenGcsTransferSample.cs | 4 +-- .../DownloadToPosixSample.cs | 8 ++--- .../QuickstartSample.cs | 5 +-- .../TransferBetweenPosixSample.cs | 6 ++-- .../TransferFromPosixSample.cs | 6 ++-- .../TransferToNearlineSample.cs | 9 ++--- .../TransferUsingManifestSample.cs | 8 +---- 12 files changed, 55 insertions(+), 67 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 0cbab538ccf..aa60ca5deef 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -37,12 +37,7 @@ public void DownloadToPosix() { DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(); Directory.CreateDirectory(_fixture.TempDirectory); - var storage = StorageClient.Create(); - byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); - MemoryStream stream = new MemoryStream(byteArray); - string fileName = $"{_fixture.GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; - storage.UploadObject(_fixture.BucketNameSource, fileName, "application/octet-stream", stream); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNameSource, _fixture.GcsSourcePath, _fixture.TempDirectory); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNamePosixSource, _fixture.GcsSourcePath, _fixture.TempDirectory); Assert.Contains("transferJobs/", transferJob.Name); Assert.True(Directory.Exists(_fixture.TempDirectory)); _transferJobName = transferJob.Name; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index e2e06260b91..e0e084fb9cf 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -14,14 +14,14 @@ * limitations under the License. */ +using System; +using System.Collections.Generic; +using System.IO; using Google.Apis.Storage.v1.Data; using Google.Cloud.PubSub.V1; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Grpc.Core; -using System; -using System.Collections.Generic; -using System.IO; using Xunit; namespace StorageTransfer.Samples.Tests @@ -32,6 +32,8 @@ public class StorageFixture : IDisposable, ICollectionFixture public string ProjectId { get; } public string BucketNameSource { get; } = Guid.NewGuid().ToString(); public string BucketNameSink { get; } = Guid.NewGuid().ToString(); + public string BucketNameManifestSource { get; } = Guid.NewGuid().ToString(); + public string BucketNamePosixSource { get; } = Guid.NewGuid().ToString(); public string JobName { get; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } @@ -54,9 +56,9 @@ public class StorageFixture : IDisposable, ICollectionFixture public StorageFixture() { ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); - SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; - SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; - PubSubId = "projects/" + ProjectId + "/subscriptions/" + SubscriptionId + ""; + SourceAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; + SinkAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; + PubSubId = $"projects/{ProjectId}/subscriptions/{SubscriptionId}"; GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { @@ -65,6 +67,10 @@ public StorageFixture() CreateBucketAndGrantStsPermissions(BucketNameSink); CreateBucketAndGrantStsPermissions(BucketNameSource); + CreateBucketAndGrantStsPermissions(BucketNameManifestSource); + CreateBucketAndGrantStsPermissions(BucketNamePosixSource); + UploadObjectToManifestBucket(BucketNameManifestSource); + UploadObjectToPosixBucket(BucketNamePosixSource); // Initialize request argument(s) TransferJob transferJob = new TransferJob { @@ -156,6 +162,23 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) Storage.SetBucketIamPolicy(bucketName, policy); } + private void UploadObjectToManifestBucket(string bucketName) + { + var storage = StorageClient.Create(); + byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + storage.UploadObject(bucketName,ManifestObjectName, "application/octet-stream", stream); + } + + private void UploadObjectToPosixBucket(string bucketName) + { + var storage = StorageClient.Create(); + byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + string fileName = $"{GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; + storage.UploadObject(bucketName, fileName, "application/octet-stream", stream); + } + public void Dispose() { try diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 02d941cabc1..7c22c2bbc32 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -15,6 +15,7 @@ using System; using System.IO; using System.Linq; +using System.Xml.Linq; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; @@ -37,25 +38,22 @@ public void TransferBetweenPosix() Directory.CreateDirectory(_fixture.TempDirectory); Directory.CreateDirectory(_fixture.TempDestinationDirectory); string sourceDir = _fixture.RootDirectory; - string[] txtList = Directory.GetFiles(sourceDir, "*.txt"); - // Copy one txt file. - foreach (string f in txtList) + string fileName = Path.Combine(_fixture.TempDirectory, "test.txt"); + // Check if file already exists. If yes, delete it. + if (File.Exists(fileName)) { - // Remove path from the file name. - string fName = f.Split('/').Last(); - // Copy only one file from source directory to temp source directory. Will overwrite if the destination file already exists. - File.Copy(Path.Combine(sourceDir, fName), Path.Combine(_fixture.TempDirectory, fName), true); - break; + File.Delete(fileName); + } + using (StreamWriter sw = new StreamWriter(fileName)) + { + sw.WriteLine("test message"); } - var storage = StorageClient.Create(); var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _fixture.TempDirectory, _fixture.TempDestinationDirectory, _fixture.BucketNameSource); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; Assert.True(Directory.Exists(_fixture.TempDirectory)); Assert.True(Directory.Exists(_fixture.TempDestinationDirectory)); - Assert.True(File.Exists(txtList[0])); - string sourceFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0, _fixture.RootDirectory.Length - 1), _fixture.TempDirectory); - Assert.True(File.Exists(sourceFilePath)); + Assert.True(File.Exists(fileName)); } public void Dispose() diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 3181c95eed7..7149781b88b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -13,8 +13,6 @@ // limitations under the License. using System; -using System.IO; -using System.Text; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; @@ -35,11 +33,7 @@ public TransferUsingManifestTest(StorageFixture fixture) public void TransferUsingManifest() { TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(); - var storage = StorageClient.Create(); - byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); - MemoryStream stream = new MemoryStream(byteArray); - storage.UploadObject(_fixture.BucketNameSource, _fixture.ManifestObjectName, "application/octet-stream", stream); - var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.ManifestObjectName); + var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameManifestSource, _fixture.BucketNameSink, _fixture.ManifestObjectName); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index e6c7614d227..58b3258563e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -16,8 +16,6 @@ using System; using Google.Cloud.StorageTransfer.V1; -namespace StorageTransfer.Samples -{ public class CheckLatestTransferOperationSample { //Checks the latest transfer operation for a given transfer job. @@ -59,5 +57,5 @@ public TransferJob CheckLatestTransferOperation( } } } -} + // [END storagetransfer_get_latest_transfer_operation] diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 717d3ed6207..a7363b9be93 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -18,8 +18,6 @@ using System; using Google.Cloud.StorageTransfer.V1; -namespace StorageTransfer.Samples -{ public class CreateEventDrivenGcsTransferSample { @@ -57,6 +55,6 @@ public TransferJob CreateEventDrivenGcsTransfer( return response; } } -} + // [END storagetransfer_create_event_driven_gcs_transfer] diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 719c3553ce6..e1c1feedfb2 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -16,11 +16,9 @@ using System; using Google.Cloud.StorageTransfer.V1; -namespace StorageTransfer.Samples -{ public class DownloadToPosixSample { - /*Create a transfer from a GCS bucket to a POSIX file system.*/ + // Create a transfer from a GCS bucket to a POSIX file system. public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -33,7 +31,7 @@ public TransferJob DownloadToPosix( // The root directory path on the source filesystem string rootDirectory = "/tmp/uploads") { - // # A useful description for your transfer job + // A useful description for your transfer job string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; TransferJob transferJob = new TransferJob @@ -68,7 +66,7 @@ public TransferJob DownloadToPosix( } } -} + //[END storagetransfer_download_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index 3adcc527053..bbc1003ef6c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -18,9 +18,6 @@ using System; using Google.Cloud.StorageTransfer.V1; - -namespace StorageTransfer.Samples -{ public class QuickstartSample { public TransferJob Quickstart( @@ -55,5 +52,5 @@ public TransferJob Quickstart( return response; } } -} + // [END storagetransfer_quickstart] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 6dcf7d9ea3c..b48a20d1045 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -16,11 +16,9 @@ using System; using Google.Cloud.StorageTransfer.V1; -namespace StorageTransfer.Samples -{ public class TransferBetweenPosixSample { - /*Creates a transfer between POSIX file systems.*/ + // Creates a transfer between POSIX file systems. public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -71,7 +69,7 @@ public TransferJob TransferBetweenPosix( } } -} + //[END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 69af8cad20a..7fc30f98650 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -16,11 +16,9 @@ using System; using Google.Cloud.StorageTransfer.V1; -namespace StorageTransfer.Samples -{ public class TransferFromPosixSample { - /*Create a transfer from a POSIX file system to a GCS sink bucket*/ + // Create a transfer from a POSIX file system to a GCS sink bucket public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -67,7 +65,7 @@ public TransferJob TransferFromPosix( } } -} + // [END storagetransfer_transfer_from_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 3f9ad9b655c..d946b04f4cd 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -17,13 +17,10 @@ using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; -namespace StorageTransfer.Samples -{ - public class TransferToNearlineSample { - /*Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more - than 30 days old to a Nearline GCS bucket.*/ + //Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more + // than 30 days old to a Nearline GCS bucket. public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -63,5 +60,5 @@ public TransferJob TransferToNearline( return response; } } -} + // [END storagetransfer_transfer_to_nearline] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 41db94e96f0..32b9ab884e0 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -17,12 +17,9 @@ using System; using Google.Cloud.StorageTransfer.V1; -namespace StorageTransfer.Samples -{ public class TransferUsingManifestSample { - /*Create a transfer from a POSIX file system to a GCS bucket using - a manifest file*/ + //Create a transfer from a POSIX file system to a GCS bucket using a manifest file*/ public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -72,9 +69,6 @@ public TransferJob TransferUsingManifest( Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}"); return response; - - } } -} // [END storagetransfer_manifest_request] From f57ea408cb20c77ebd4c6507126044748005ea0c Mon Sep 17 00:00:00 2001 From: Mahendra Date: Thu, 30 Jan 2025 15:42:38 +0530 Subject: [PATCH 21/49] Copyright year reverted for old files --- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index e0e084fb9cf..087d111872d 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google Inc. + * Copyright 2021 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From a2192fc33480cc7fa1163a67331566e6884b6deb Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 30 Jan 2025 02:57:08 -0800 Subject: [PATCH 22/49] Blank line removed --- .../api/StorageTransfer.Samples/TransferUsingManifestSample.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 32b9ab884e0..e279e8bcf6c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -54,7 +54,6 @@ public TransferJob TransferUsingManifest( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); From 2d068d29d03b074f52dc852b488180b980e58192 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 30 Jan 2025 05:30:23 -0800 Subject: [PATCH 23/49] changes related to blank spaces --- .../CheckLatestTransferOperationSample.cs | 7 +++---- .../CreateEventDrivenGcsTransferSample.cs | 5 +---- .../api/StorageTransfer.Samples/DownloadToPosixSample.cs | 7 ++----- .../api/StorageTransfer.Samples/QuickstartSample.cs | 3 +-- .../StorageTransfer.Samples/TransferBetweenPosixSample.cs | 6 ++---- .../StorageTransfer.Samples/TransferFromPosixSample.cs | 8 ++------ .../StorageTransfer.Samples/TransferToNearlineSample.cs | 8 +++++--- .../TransferUsingManifestSample.cs | 2 +- 8 files changed, 17 insertions(+), 29 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 58b3258563e..f17b6f604b1 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + // [START storagetransfer_get_latest_transfer_operation] using System; @@ -18,7 +19,7 @@ public class CheckLatestTransferOperationSample { - //Checks the latest transfer operation for a given transfer job. + // Checks the latest transfer operation for a given transfer job public TransferJob CheckLatestTransferOperation( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -29,9 +30,9 @@ public TransferJob CheckLatestTransferOperation( { throw new Exception("JobName can not be null or empty"); } + // Create a Transfer Service client StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.Create(); - GetTransferJobRequest getTransferJobRequest = new GetTransferJobRequest { ProjectId = projectId, JobName = jobName }; try { @@ -40,7 +41,6 @@ public TransferJob CheckLatestTransferOperation( // Get Latest operation name from transfer job string latestOperationName = transferJob.LatestOperationName; - if (!string.IsNullOrEmpty(latestOperationName)) { Console.WriteLine("The latest operation for transfer job " + jobName + " is: " + latestOperationName + ""); @@ -57,5 +57,4 @@ public TransferJob CheckLatestTransferOperation( } } } - // [END storagetransfer_get_latest_transfer_operation] diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index a7363b9be93..de783bdf219 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + // [START storagetransfer_create_event_driven_gcs_transfer] using System; @@ -20,7 +21,6 @@ public class CreateEventDrivenGcsTransferSample { - public TransferJob CreateEventDrivenGcsTransfer( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -42,7 +42,6 @@ public TransferJob CreateEventDrivenGcsTransfer( { GcsDataSink = new GcsData { BucketName = sinkBucket }, GcsDataSource = new GcsData { BucketName = sourceBucket }, - }, Status = TransferJob.Types.Status.Enabled, EventStream = new EventStream { Name = pubSubId } @@ -51,10 +50,8 @@ public TransferJob CreateEventDrivenGcsTransfer( StorageTransferServiceClient client = StorageTransferServiceClient.Create(); TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); Console.WriteLine($"Created an event driven transfer job from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} with name {response.Name}"); - return response; } } - // [END storagetransfer_create_event_driven_gcs_transfer] diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index e1c1feedfb2..5c70ee2b6a9 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + // [START storagetransfer_download_to_posix] using System; @@ -18,7 +19,7 @@ public class DownloadToPosixSample { - // Create a transfer from a GCS bucket to a POSIX file system. + // Create a transfer from a GCS bucket to a POSIX file system public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -47,7 +48,6 @@ public TransferJob DownloadToPosix( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); @@ -62,11 +62,8 @@ public TransferJob DownloadToPosix( Console.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); return response; - - } } - //[END storagetransfer_download_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index bbc1003ef6c..76295086cbf 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + // [START storagetransfer_quickstart] using System; @@ -48,9 +49,7 @@ public TransferJob Quickstart( }); Console.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); - return response; } } - // [END storagetransfer_quickstart] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index b48a20d1045..807e8d998f1 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + // [START storagetransfer_transfer_posix_to_posix] using System; @@ -18,7 +19,7 @@ public class TransferBetweenPosixSample { - // Creates a transfer between POSIX file systems. + // Creates a transfer between POSIX file systems public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -51,7 +52,6 @@ public TransferJob TransferBetweenPosix( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); @@ -66,10 +66,8 @@ public TransferJob TransferBetweenPosix( Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {destinationDirectory} with the name {response.Name}"); return response; - } } - //[END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 7fc30f98650..b46df62cf89 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + // [START storagetransfer_transfer_from_posix] using System; @@ -18,7 +19,7 @@ public class TransferFromPosixSample { - // Create a transfer from a POSIX file system to a GCS sink bucket + // Create a transfer from a POSIX file system to a GCS sink bucket public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -29,7 +30,6 @@ public TransferJob TransferFromPosix( // The GCS bucket to transfer data to string sinkBucket = "my-sink-bucket") { - // A useful description for your transfer job string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket})"; @@ -46,7 +46,6 @@ public TransferJob TransferFromPosix( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); @@ -61,11 +60,8 @@ public TransferJob TransferFromPosix( Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} with the name {response.Name}"); return response; - - } } - // [END storagetransfer_transfer_from_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index d946b04f4cd..5d97f7204a8 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + // [START storagetransfer_transfer_to_nearline] using System; @@ -19,8 +20,8 @@ public class TransferToNearlineSample { - //Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more - // than 30 days old to a Nearline GCS bucket. + // Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more + // than 30 days old to a Nearline GCS bucket public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -46,8 +47,10 @@ public TransferJob TransferToNearline( Status = TransferJob.Types.Status.Enabled, Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)), ScheduleEndDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } }; + // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer job TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); client.RunTransferJob(new RunTransferJobRequest @@ -60,5 +63,4 @@ public TransferJob TransferToNearline( return response; } } - // [END storagetransfer_transfer_to_nearline] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index e279e8bcf6c..73fab93c6fd 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -19,7 +19,7 @@ public class TransferUsingManifestSample { - //Create a transfer from a POSIX file system to a GCS bucket using a manifest file*/ + // Create a transfer from a POSIX file system to a GCS bucket using a manifest file public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", From 2e7aa6229e2d115d2690ac6b73b749af0bac6684 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 30 Jan 2025 23:43:16 -0800 Subject: [PATCH 24/49] Blank spaces reverted in QuickStartTest --- .../api/StorageTransfer.Samples.Tests/QuickstartTest.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index b72b1900c5f..33fa4dd687d 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + using System; using Google.Cloud.StorageTransfer.V1; using Xunit; @@ -24,6 +25,7 @@ public class QuickstartTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + public QuickstartTest(StorageFixture fixture) { _fixture = fixture; From 02a0e140b97ff35962d08a8bebbf84715ca62259 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 3 Feb 2025 01:51:30 -0800 Subject: [PATCH 25/49] resources for event driven transfer are moved from storage fixture to test --- .../CreateEventDrivenGcsTransferTest.cs | 50 ++++++++++++++++- .../StorageFixture.cs | 54 +------------------ 2 files changed, 50 insertions(+), 54 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 7e3f2cb1724..e40c7971474 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -15,6 +15,8 @@ */ using System; +using Google.Cloud.PubSub.V1; +using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; @@ -24,17 +26,58 @@ namespace StorageTransfer.Samples.Tests public class CreateEventDrivenGcsTransferTest : IDisposable { private readonly StorageFixture _fixture; + private readonly StorageTransferServiceClient _sts; + private readonly string _pubSubId; private string _transferJobName; + private string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); + private string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); + private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); + private PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); public CreateEventDrivenGcsTransferTest(StorageFixture fixture) { _fixture = fixture; + _pubSubId = $"projects/{_fixture.ProjectId}/subscriptions/{SubscriptionId}"; + string email = _fixture.Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest() + { + ProjectId = _fixture.ProjectId + }).AccountEmail; + + string memberServiceAccount = "serviceAccount:" + email; + // Create subscription name + SubscriptionName subscriptionName = new SubscriptionName(_fixture.ProjectId, SubscriptionId); + // Create topic name + TopicName topicName = new TopicName(_fixture.ProjectId, TopicId); + // Create topic + PublisherClient.CreateTopic(topicName); + // Create subscription. + SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500); + + var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy(); + + policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount); + + PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest + { + ResourceAsResourceName = topicName, + Policy = policyIamPolicyTopic + }); + + var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy(); + + policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount); + + PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest + { + ResourceAsResourceName = subscriptionName, + Policy = policyIamPolicySubscriber + }); } [Fact] public void CreateEventDrivenGcsTransfer() { CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); - var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.PubSubId); + var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _pubSubId); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -53,6 +96,11 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + + TopicName topicName = TopicName.FromProjectTopic(_fixture.ProjectId, TopicId); + PublisherClient.DeleteTopic(topicName); + SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(_fixture.ProjectId, SubscriptionId); + SubscriberClient.DeleteSubscription(subscriptionName); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 087d111872d..fc5b73438b3 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -44,21 +44,13 @@ public class StorageFixture : IDisposable, ICollectionFixture public string TempDestinationDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; - public string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); - public string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); - public string PubSubId { get; } public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); - public SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); - - public PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); - public StorageFixture() { ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); SourceAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; SinkAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; - PubSubId = $"projects/{ProjectId}/subscriptions/{SubscriptionId}"; GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { @@ -94,28 +86,6 @@ public StorageFixture() ProjectId = ProjectId }).AccountEmail; string memberServiceAccount = "serviceAccount:" + email; - // Create subscription name - SubscriptionName subscriptionName = new SubscriptionName(ProjectId, SubscriptionId); - // Create topic name - TopicName topicName = new TopicName(ProjectId, TopicId); - // Create topic - PublisherClient.CreateTopic(topicName); - // Create subscription. - SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500); - var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy(); - policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount); - PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest - { - ResourceAsResourceName = topicName, - Policy = policyIamPolicyTopic - }); - var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy(); - policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount); - PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest - { - ResourceAsResourceName = subscriptionName, - Policy = policyIamPolicySubscriber - }); } private void CreateBucketAndGrantStsPermissions(string bucketName) @@ -167,7 +137,7 @@ private void UploadObjectToManifestBucket(string bucketName) var storage = StorageClient.Create(); byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); - storage.UploadObject(bucketName,ManifestObjectName, "application/octet-stream", stream); + storage.UploadObject(bucketName, ManifestObjectName, "application/octet-stream", stream); } private void UploadObjectToPosixBucket(string bucketName) @@ -188,37 +158,15 @@ public void Dispose() catch (Exception) { // Do nothing, we delete on a best effort basis. - } try { Storage.DeleteBucket(BucketNameSource, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) - { - // Do nothing, we delete on a best effort basis. - - } - try - { - TopicName topicName = TopicName.FromProjectTopic(ProjectId, TopicId); - PublisherClient.DeleteTopic(topicName); - } - catch (RpcException) { // Do nothing, we delete on a best effort basis. } - - try - { - SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(ProjectId, SubscriptionId); - SubscriberClient.DeleteSubscription(subscriptionName); - } - catch (RpcException) - { - // Do nothing, we delete on a best effort basis. - } - } } } From 5e709004081caa6ba9ced75c3fe28e0cefdb9cf5 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Tue, 4 Feb 2025 03:25:46 -0800 Subject: [PATCH 26/49] movement of temporary folder path from storage fixture to respective tests --- .../CreateEventDrivenGcsTransferTest.cs | 1 - .../DownloadToPosixTest.cs | 10 +++++---- .../StorageFixture.cs | 9 ++------ .../TransferBetweenPosixTest.cs | 21 +++++++++++-------- .../TransferFromPosixTest.cs | 5 +++-- .../TransferUsingManifestTest.cs | 4 +++- 6 files changed, 26 insertions(+), 24 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index e40c7971474..60833043c26 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -26,7 +26,6 @@ namespace StorageTransfer.Samples.Tests public class CreateEventDrivenGcsTransferTest : IDisposable { private readonly StorageFixture _fixture; - private readonly StorageTransferServiceClient _sts; private readonly string _pubSubId; private string _transferJobName; private string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index aa60ca5deef..e23f6f4f1c7 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -27,19 +27,21 @@ public class DownloadToPosixTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly string _tempDirectory; public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; + _tempDirectory = fixture.GenerateTempFolderPath(); } [Fact] public void DownloadToPosix() { DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(); - Directory.CreateDirectory(_fixture.TempDirectory); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNamePosixSource, _fixture.GcsSourcePath, _fixture.TempDirectory); + Directory.CreateDirectory(_tempDirectory); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNamePosixSource, _fixture.GcsSourcePath, _tempDirectory); Assert.Contains("transferJobs/", transferJob.Name); - Assert.True(Directory.Exists(_fixture.TempDirectory)); + Assert.True(Directory.Exists(_tempDirectory)); _transferJobName = transferJob.Name; } @@ -57,7 +59,7 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); - Directory.Delete(_fixture.TempDirectory, true); + Directory.Delete(_tempDirectory, true); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index fc5b73438b3..386cd12120d 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -18,10 +18,8 @@ using System.Collections.Generic; using System.IO; using Google.Apis.Storage.v1.Data; -using Google.Cloud.PubSub.V1; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; -using Grpc.Core; using Xunit; namespace StorageTransfer.Samples.Tests @@ -38,10 +36,6 @@ public class StorageFixture : IDisposable, ICollectionFixture public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } public string GcsSourcePath { get; } - public string RootDirectory { get; } = System.IO.Path.GetTempPath(); - public string DestinationDirectory { get; } = System.IO.Path.GetTempPath(); - public string TempDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); - public string TempDestinationDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); @@ -148,7 +142,8 @@ private void UploadObjectToPosixBucket(string bucketName) string fileName = $"{GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; storage.UploadObject(bucketName, fileName, "application/octet-stream", stream); } - + internal string GetCurrentUserTempFolderPath() => System.IO.Path.GetTempPath(); + internal string GenerateTempFolderPath() => Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public void Dispose() { try diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 7c22c2bbc32..4185a094b4f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -26,19 +26,22 @@ public class TransferBetweenPosixTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly string _tempDirectory; + private readonly string _tempDestinationDirectory; public TransferBetweenPosixTest(StorageFixture fixture) { _fixture = fixture; + _tempDirectory = fixture.GenerateTempFolderPath(); + _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } [Fact] public void TransferBetweenPosix() { TransferBetweenPosixSample transferBetweenPosixSample = new TransferBetweenPosixSample(); - Directory.CreateDirectory(_fixture.TempDirectory); - Directory.CreateDirectory(_fixture.TempDestinationDirectory); - string sourceDir = _fixture.RootDirectory; - string fileName = Path.Combine(_fixture.TempDirectory, "test.txt"); + Directory.CreateDirectory(_tempDirectory); + Directory.CreateDirectory(_tempDestinationDirectory); + string fileName = Path.Combine(_tempDirectory, "test.txt"); // Check if file already exists. If yes, delete it. if (File.Exists(fileName)) { @@ -48,11 +51,11 @@ public void TransferBetweenPosix() { sw.WriteLine("test message"); } - var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _fixture.TempDirectory, _fixture.TempDestinationDirectory, _fixture.BucketNameSource); + var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _tempDirectory, _tempDestinationDirectory, _fixture.BucketNameSource); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; - Assert.True(Directory.Exists(_fixture.TempDirectory)); - Assert.True(Directory.Exists(_fixture.TempDestinationDirectory)); + Assert.True(Directory.Exists(_tempDirectory)); + Assert.True(Directory.Exists(_tempDestinationDirectory)); Assert.True(File.Exists(fileName)); } @@ -70,8 +73,8 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); - Directory.Delete(_fixture.TempDirectory, true); - Directory.Delete(_fixture.TempDestinationDirectory, true); + Directory.Delete(_tempDirectory, true); + Directory.Delete(_tempDestinationDirectory, true); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index f9f0d3e68c9..0cfeab95a4a 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -23,17 +23,18 @@ public class TransferFromPosixTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly string _rootDirectory; public TransferFromPosixTest(StorageFixture fixture) { _fixture = fixture; + _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } [Fact] public void TransferFromPosix() { TransferFromPosixSample transferFromPosixSample = new TransferFromPosixSample(); - var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameSink); - var storage = StorageClient.Create(); + var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _fixture.BucketNameSink); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 7149781b88b..ddd6c5e1662 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -24,16 +24,18 @@ public class TransferUsingManifestTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly string _rootDirectory; public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; + _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } [Fact] public void TransferUsingManifest() { TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(); - var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameManifestSource, _fixture.BucketNameSink, _fixture.ManifestObjectName); + var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _fixture.BucketNameManifestSource, _fixture.BucketNameSink, _fixture.ManifestObjectName); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } From d9421774384122c52bf101fef8b9108b3b5b27a7 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 5 Feb 2025 03:01:39 -0800 Subject: [PATCH 27/49] gcssourcepath , manifestobjectname is moved from fixture to test --- .../DownloadToPosixTest.cs | 4 +++- .../StorageFixture.cs | 14 ++++---------- .../TransferUsingManifestTest.cs | 4 +++- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index e23f6f4f1c7..325905aa0bb 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -28,10 +28,12 @@ public class DownloadToPosixTest : IDisposable private readonly StorageFixture _fixture; private string _transferJobName; private readonly string _tempDirectory; + private readonly string _gcsSourcePath; public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; _tempDirectory = fixture.GenerateTempFolderPath(); + _gcsSourcePath = "foo/bar/"; } [Fact] @@ -39,7 +41,7 @@ public void DownloadToPosix() { DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(); Directory.CreateDirectory(_tempDirectory); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNamePosixSource, _fixture.GcsSourcePath, _tempDirectory); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNamePosixSource, _gcsSourcePath, _tempDirectory); Assert.Contains("transferJobs/", transferJob.Name); Assert.True(Directory.Exists(_tempDirectory)); _transferJobName = transferJob.Name; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 386cd12120d..f1b4780c709 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -35,9 +35,7 @@ public class StorageFixture : IDisposable, ICollectionFixture public string JobName { get; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } - public string GcsSourcePath { get; } public StorageClient Storage { get; } = StorageClient.Create(); - public string ManifestObjectName { get; } = "manifest.csv"; public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); public StorageFixture() @@ -45,7 +43,6 @@ public StorageFixture() ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); SourceAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; SinkAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; - GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id."); @@ -75,11 +72,6 @@ public StorageFixture() // Make the request TransferJob response = Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); JobName = response.Name; - string email = Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest() - { - ProjectId = ProjectId - }).AccountEmail; - string memberServiceAccount = "serviceAccount:" + email; } private void CreateBucketAndGrantStsPermissions(string bucketName) @@ -128,18 +120,20 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) private void UploadObjectToManifestBucket(string bucketName) { + var manifestObjectName = "manifest.csv"; var storage = StorageClient.Create(); byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); - storage.UploadObject(bucketName, ManifestObjectName, "application/octet-stream", stream); + storage.UploadObject(bucketName, manifestObjectName, "application/octet-stream", stream); } private void UploadObjectToPosixBucket(string bucketName) { var storage = StorageClient.Create(); + var gcsSourcePath = "foo/bar/"; byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); - string fileName = $"{GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; + string fileName = $"{gcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; storage.UploadObject(bucketName, fileName, "application/octet-stream", stream); } internal string GetCurrentUserTempFolderPath() => System.IO.Path.GetTempPath(); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index ddd6c5e1662..442b4c17aa4 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -25,17 +25,19 @@ public class TransferUsingManifestTest : IDisposable private readonly StorageFixture _fixture; private string _transferJobName; private readonly string _rootDirectory; + private readonly string _manifestObjectName; public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; _rootDirectory = fixture.GetCurrentUserTempFolderPath(); + _manifestObjectName = "manifest.csv"; } [Fact] public void TransferUsingManifest() { TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(); - var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _fixture.BucketNameManifestSource, _fixture.BucketNameSink, _fixture.ManifestObjectName); + var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _fixture.BucketNameManifestSource, _fixture.BucketNameSink, _manifestObjectName); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } From a59263c04ab6bf09111ae74d0363890e903af4ca Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 6 Feb 2025 03:36:53 -0800 Subject: [PATCH 28/49] UploadObjectToManifestBucket , UploadObjectToPosixBucket , CreateTransferJob moved to respective tests --- .../CheckLatestTransferOperationTest.cs | 27 +++++++++++- .../DownloadToPosixTest.cs | 10 ++++- .../StorageFixture.cs | 41 ------------------- .../TransferUsingManifestTest.cs | 11 ++++- 4 files changed, 45 insertions(+), 44 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index 39d3dfce59e..a5cbd41b2aa 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -11,6 +11,8 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -21,17 +23,40 @@ public class CheckLatestTransferOperationTest private readonly StorageFixture _fixture; private string _jobName; + private readonly string _transferJobName; public CheckLatestTransferOperationTest(StorageFixture fixture) { _fixture = fixture; + _transferJobName = CreateTransferJob(); } [Fact] public void CheckLatestTransferOperation() { CheckLatestTransferOperationSample checkLatestTransferOperationSample = new CheckLatestTransferOperationSample(); - var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId, _fixture.JobName); + var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId, _transferJobName); Assert.Contains("transferJobs/", transferJob.Name); _jobName = transferJob.Name; } + private string CreateTransferJob() + { + // Initialize request argument(s) + TransferJob transferJob = new TransferJob + { + ProjectId = _fixture.ProjectId, + TransferSpec = new TransferSpec + { + GcsDataSink = new GcsData { BucketName = _fixture.BucketNameSource }, + GcsDataSource = new GcsData { BucketName = _fixture.BucketNameSink } + }, + Status = TransferJob.Types.Status.Enabled + }; + CreateTransferJobRequest request = new CreateTransferJobRequest + { + TransferJob = transferJob + }; + // Make the request + TransferJob response = _fixture.Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + return response.Name; + } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 325905aa0bb..b808b2b0c2a 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -33,7 +33,8 @@ public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; _tempDirectory = fixture.GenerateTempFolderPath(); - _gcsSourcePath = "foo/bar/"; + _gcsSourcePath = $"{Guid.NewGuid()}/{Guid.NewGuid()}/"; + UploadObjectToPosixBucket(_fixture.BucketNamePosixSource); } [Fact] @@ -46,6 +47,13 @@ public void DownloadToPosix() Assert.True(Directory.Exists(_tempDirectory)); _transferJobName = transferJob.Name; } + private void UploadObjectToPosixBucket(string bucketName) + { + byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + string fileName = $"{_gcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; + _fixture.Storage.UploadObject(bucketName, fileName, "application/octet-stream", stream); + } public void Dispose() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index f1b4780c709..b942c5b21f6 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -32,7 +32,6 @@ public class StorageFixture : IDisposable, ICollectionFixture public string BucketNameSink { get; } = Guid.NewGuid().ToString(); public string BucketNameManifestSource { get; } = Guid.NewGuid().ToString(); public string BucketNamePosixSource { get; } = Guid.NewGuid().ToString(); - public string JobName { get; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } public StorageClient Storage { get; } = StorageClient.Create(); @@ -52,28 +51,7 @@ public StorageFixture() CreateBucketAndGrantStsPermissions(BucketNameSource); CreateBucketAndGrantStsPermissions(BucketNameManifestSource); CreateBucketAndGrantStsPermissions(BucketNamePosixSource); - UploadObjectToManifestBucket(BucketNameManifestSource); - UploadObjectToPosixBucket(BucketNamePosixSource); - // Initialize request argument(s) - TransferJob transferJob = new TransferJob - { - ProjectId = ProjectId, - TransferSpec = new TransferSpec - { - GcsDataSink = new GcsData { BucketName = BucketNameSource }, - GcsDataSource = new GcsData { BucketName = BucketNameSink } - }, - Status = TransferJob.Types.Status.Enabled - }; - CreateTransferJobRequest request = new CreateTransferJobRequest - { - TransferJob = transferJob - }; - // Make the request - TransferJob response = Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - JobName = response.Name; } - private void CreateBucketAndGrantStsPermissions(string bucketName) { var bucket = Storage.CreateBucket(ProjectId, new Bucket @@ -117,25 +95,6 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) policy.Bindings.Add(bucketWriterBinding); Storage.SetBucketIamPolicy(bucketName, policy); } - - private void UploadObjectToManifestBucket(string bucketName) - { - var manifestObjectName = "manifest.csv"; - var storage = StorageClient.Create(); - byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); - MemoryStream stream = new MemoryStream(byteArray); - storage.UploadObject(bucketName, manifestObjectName, "application/octet-stream", stream); - } - - private void UploadObjectToPosixBucket(string bucketName) - { - var storage = StorageClient.Create(); - var gcsSourcePath = "foo/bar/"; - byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); - MemoryStream stream = new MemoryStream(byteArray); - string fileName = $"{gcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; - storage.UploadObject(bucketName, fileName, "application/octet-stream", stream); - } internal string GetCurrentUserTempFolderPath() => System.IO.Path.GetTempPath(); internal string GenerateTempFolderPath() => Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public void Dispose() diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 442b4c17aa4..b122e99f9c0 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -13,6 +13,7 @@ // limitations under the License. using System; +using System.IO; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; @@ -30,7 +31,8 @@ public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; _rootDirectory = fixture.GetCurrentUserTempFolderPath(); - _manifestObjectName = "manifest.csv"; + _manifestObjectName = $@"{Guid.NewGuid()}.csv"; + UploadObjectToManifestBucket(_fixture.BucketNameManifestSource); } [Fact] @@ -42,6 +44,13 @@ public void TransferUsingManifest() _transferJobName = transferJob.Name; } + private void UploadObjectToManifestBucket(string bucketName) + { + byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + _fixture.Storage.UploadObject(bucketName, _manifestObjectName, "application/octet-stream", stream); + } + public void Dispose() { try From 7e2b4ddab872a46e73e2e0f51518c62a0b994fbe Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 10 Feb 2025 02:59:41 -0800 Subject: [PATCH 29/49] style and structure changes as per canonical sample --- .../CheckLatestTransferOperationSample.cs | 56 ++++++------ .../CreateEventDrivenGcsTransferSample.cs | 58 ++++++------ .../DownloadToPosixSample.cs | 78 ++++++++-------- .../QuickstartSample.cs | 54 ++++++------ .../TransferBetweenPosixSample.cs | 64 +++++++------- .../TransferFromPosixSample.cs | 74 ++++++++-------- .../TransferToNearlineSample.cs | 76 ++++++++-------- .../TransferUsingManifestSample.cs | 88 +++++++++---------- 8 files changed, 274 insertions(+), 274 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index f17b6f604b1..7ef3dab7d55 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -17,44 +17,44 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class CheckLatestTransferOperationSample - { - // Checks the latest transfer operation for a given transfer job - public TransferJob CheckLatestTransferOperation( +public class CheckLatestTransferOperationSample +{ + // Checks the latest transfer operation for a given transfer job + public TransferJob CheckLatestTransferOperation( // Your Google Cloud Project ID string projectId = "my-project-id", // The name of the job to check string jobName = "transferJobs/1234567890") + { + if (string.IsNullOrEmpty(jobName)) { - if (string.IsNullOrEmpty(jobName)) - { - throw new Exception("JobName can not be null or empty"); - } + throw new Exception("JobName can not be null or empty"); + } - // Create a Transfer Service client - StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.Create(); - GetTransferJobRequest getTransferJobRequest = new GetTransferJobRequest { ProjectId = projectId, JobName = jobName }; - try - { - // Get Transfer job - TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); - // Get Latest operation name from transfer job - string latestOperationName = transferJob.LatestOperationName; + // Create a Transfer Service client + StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.Create(); + GetTransferJobRequest getTransferJobRequest = new GetTransferJobRequest { ProjectId = projectId, JobName = jobName }; + try + { + // Get Transfer job + TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); + // Get Latest operation name from transfer job + string latestOperationName = transferJob.LatestOperationName; - if (!string.IsNullOrEmpty(latestOperationName)) - { - Console.WriteLine("The latest operation for transfer job " + jobName + " is: " + latestOperationName + ""); - } - else - { - Console.WriteLine("Transfer job " + jobName + " hasn't run yet, try again after the job has started running."); - } - return transferJob; + if (!string.IsNullOrEmpty(latestOperationName)) + { + Console.WriteLine("The latest operation for transfer job " + jobName + " is: " + latestOperationName + ""); } - catch (Exception) + else { - throw new Exception("Failed to get transfer job " + jobName + ""); + Console.WriteLine("Transfer job " + jobName + " hasn't run yet, try again after the job has started running."); } + return transferJob; + } + catch (Exception) + { + throw new Exception("Failed to get transfer job " + jobName + ""); } } +} // [END storagetransfer_get_latest_transfer_operation] diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index de783bdf219..d57407d18dc 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -19,39 +19,39 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class CreateEventDrivenGcsTransferSample +public class CreateEventDrivenGcsTransferSample +{ + public TransferJob CreateEventDrivenGcsTransfer( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The GCS bucket to transfer data from + string sourceBucket = "my-source-bucket", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket", + // The subscription ID to a Pubsub queue to track + string pubSubId = "projects/PROJECT_NAME/subscriptions/SUBSCRIPTION_ID") { - public TransferJob CreateEventDrivenGcsTransfer( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The GCS bucket to transfer data from - string sourceBucket = "my-source-bucket", - // The GCS bucket to transfer data to - string sinkBucket = "my-sink-bucket", - // The subscription ID to a Pubsub queue to track - string pubSubId = "projects/PROJECT_NAME/subscriptions/SUBSCRIPTION_ID") - { - // A useful description for your transfer job - string jobDescription = $"Event driven gcs data transfer from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} "; + // A useful description for your transfer job + string jobDescription = $"Event driven gcs data transfer from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} "; - TransferJob transferJob = new TransferJob + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec { - ProjectId = projectId, - Description = jobDescription, - TransferSpec = new TransferSpec - { - GcsDataSink = new GcsData { BucketName = sinkBucket }, - GcsDataSource = new GcsData { BucketName = sourceBucket }, - }, - Status = TransferJob.Types.Status.Enabled, - EventStream = new EventStream { Name = pubSubId } - }; + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = sourceBucket }, + }, + Status = TransferJob.Types.Status.Enabled, + EventStream = new EventStream { Name = pubSubId } + }; - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - Console.WriteLine($"Created an event driven transfer job from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} with name {response.Name}"); - return response; - } + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + Console.WriteLine($"Created an event driven transfer job from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} with name {response.Name}"); + return response; } +} // [END storagetransfer_create_event_driven_gcs_transfer] diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 5c70ee2b6a9..5e15ee5b5de 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -17,53 +17,53 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class DownloadToPosixSample +public class DownloadToPosixSample +{ + // Create a transfer from a GCS bucket to a POSIX file system + public TransferJob DownloadToPosix( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent + string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // Your GCS source bucket name + string gcsSourceBucket = "my-gcs-source-bucket", + // An optional path on the Google Cloud Storage bucket to download from + string gcsSourcePath = "foo/bar/", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads") { - // Create a transfer from a GCS bucket to a POSIX file system - public TransferJob DownloadToPosix( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent - string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // Your GCS source bucket name - string gcsSourceBucket = "my-gcs-source-bucket", - // An optional path on the Google Cloud Storage bucket to download from - string gcsSourcePath = "foo/bar/", - // The root directory path on the source filesystem - string rootDirectory = "/tmp/uploads") - { - // A useful description for your transfer job - string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; + // A useful description for your transfer job + string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; - TransferJob transferJob = new TransferJob + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec { - ProjectId = projectId, - Description = jobDescription, - TransferSpec = new TransferSpec - { - GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath }, - SinkAgentPoolName = sinkAgentPoolName, - PosixDataSink = new PosixFilesystem { RootDirectory = rootDirectory } - }, - Status = TransferJob.Types.Status.Enabled, - }; + GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath }, + SinkAgentPoolName = sinkAgentPoolName, + PosixDataSink = new PosixFilesystem { RootDirectory = rootDirectory } + }, + Status = TransferJob.Types.Status.Enabled, + }; - // Create a Transfer Service client - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - // Create a Transfer job - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest - { - JobName = response.Name, - ProjectId = projectId - }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); - Console.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); - return response; - } + Console.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); + return response; } +} //[END storagetransfer_download_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index 76295086cbf..9726dfb777c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -19,37 +19,37 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class QuickstartSample +public class QuickstartSample +{ + public TransferJob Quickstart( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The GCS bucket to transfer data from + string sourceBucket = "my-source-bucket", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket") { - public TransferJob Quickstart( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The GCS bucket to transfer data from - string sourceBucket = "my-source-bucket", - // The GCS bucket to transfer data to - string sinkBucket = "my-sink-bucket") + TransferJob transferJob = new TransferJob { - TransferJob transferJob = new TransferJob + ProjectId = projectId, + TransferSpec = new TransferSpec { - ProjectId = projectId, - TransferSpec = new TransferSpec - { - GcsDataSink = new GcsData { BucketName = sinkBucket }, - GcsDataSource = new GcsData { BucketName = sourceBucket } - }, - Status = TransferJob.Types.Status.Enabled - }; + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = sourceBucket } + }, + Status = TransferJob.Types.Status.Enabled + }; - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest - { - JobName = response.Name, - ProjectId = projectId - }); + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); - Console.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); - return response; - } + Console.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); + return response; } +} // [END storagetransfer_quickstart] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 807e8d998f1..b07b0bdb118 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -17,10 +17,10 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class TransferBetweenPosixSample - { - // Creates a transfer between POSIX file systems - public TransferJob TransferBetweenPosix( +public class TransferBetweenPosixSample +{ + // Creates a transfer between POSIX file systems + public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent @@ -33,41 +33,41 @@ public TransferJob TransferBetweenPosix( string destinationDirectory = "/directory/to/transfer/sink", // The name of GCS bucket for intermediate storage string intermediate_bucket = "my-intermediate-bucket") - { - // A useful description for your transfer job - string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; + { + // A useful description for your transfer job + string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; - TransferJob transferJob = new TransferJob + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec { - ProjectId = projectId, - Description = jobDescription, - TransferSpec = new TransferSpec - { - SourceAgentPoolName = sourceAgentPoolName, - SinkAgentPoolName = sinkAgentPoolName, - PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, - PosixDataSink = new PosixFilesystem { RootDirectory = destinationDirectory }, - GcsIntermediateDataLocation = new GcsData { BucketName = intermediate_bucket } - }, - Status = TransferJob.Types.Status.Enabled, - }; + SourceAgentPoolName = sourceAgentPoolName, + SinkAgentPoolName = sinkAgentPoolName, + PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, + PosixDataSink = new PosixFilesystem { RootDirectory = destinationDirectory }, + GcsIntermediateDataLocation = new GcsData { BucketName = intermediate_bucket } + }, + Status = TransferJob.Types.Status.Enabled, + }; - // Create a Transfer Service client - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - // Create a Transfer job - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest - { - JobName = response.Name, - ProjectId = projectId - }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); - Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {destinationDirectory} with the name {response.Name}"); - return response; - } + Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {destinationDirectory} with the name {response.Name}"); + return response; } +} //[END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index b46df62cf89..60e6f29af2f 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -17,51 +17,51 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class TransferFromPosixSample +public class TransferFromPosixSample +{ + // Create a transfer from a POSIX file system to a GCS sink bucket + public TransferJob TransferFromPosix( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent + string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket") { - // Create a transfer from a POSIX file system to a GCS sink bucket - public TransferJob TransferFromPosix( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent - string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // The root directory path on the source filesystem - string rootDirectory = "/tmp/uploads", - // The GCS bucket to transfer data to - string sinkBucket = "my-sink-bucket") - { - // A useful description for your transfer job - string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket})"; + // A useful description for your transfer job + string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket})"; - TransferJob transferJob = new TransferJob + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec { - ProjectId = projectId, - Description = jobDescription, - TransferSpec = new TransferSpec - { - GcsDataSink = new GcsData { BucketName = sinkBucket }, - SourceAgentPoolName = sourceAgentPoolName, - PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory } - }, - Status = TransferJob.Types.Status.Enabled, - }; + GcsDataSink = new GcsData { BucketName = sinkBucket }, + SourceAgentPoolName = sourceAgentPoolName, + PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory } + }, + Status = TransferJob.Types.Status.Enabled, + }; - // Create a Transfer Service client - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - // Create a Transfer job - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest - { - JobName = response.Name, - ProjectId = projectId - }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); - Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} with the name {response.Name}"); - return response; - } + Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} with the name {response.Name}"); + return response; } +} // [END storagetransfer_transfer_from_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 5d97f7204a8..f9bcfd443cb 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -18,49 +18,49 @@ using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; - public class TransferToNearlineSample +public class TransferToNearlineSample +{ + // Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more + // than 30 days old to a Nearline GCS bucket + public TransferJob TransferToNearline( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The GCS bucket to transfer objects from + string sourceBucket = "my-source-bucket", + // The GCS Nearline bucket to transfer old objects to + string sinkBucket = "my-sink-bucket") { - // Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more - // than 30 days old to a Nearline GCS bucket - public TransferJob TransferToNearline( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The GCS bucket to transfer objects from - string sourceBucket = "my-source-bucket", - // The GCS Nearline bucket to transfer old objects to - string sinkBucket = "my-sink-bucket") - { - // A description of this job - string jobDescription = $"Transfers old objects from standard bucket ({sourceBucket}) that haven't been modified in the last 30 days to a Nearline bucket ({sinkBucket})"; + // A description of this job + string jobDescription = $"Transfers old objects from standard bucket ({sourceBucket}) that haven't been modified in the last 30 days to a Nearline bucket ({sinkBucket})"; - TransferJob transferJob = new TransferJob + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec { - ProjectId = projectId, - Description = jobDescription, - TransferSpec = new TransferSpec - { - GcsDataSink = new GcsData { BucketName = sinkBucket }, - GcsDataSource = new GcsData { BucketName = sourceBucket }, - ObjectConditions = new ObjectConditions { MinTimeElapsedSinceLastModification = Duration.FromTimeSpan(TimeSpan.FromSeconds(2592000)) }, - TransferOptions = new TransferOptions { DeleteObjectsFromSourceAfterTransfer = true }, - }, - Status = TransferJob.Types.Status.Enabled, - Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)), ScheduleEndDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } - }; + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = sourceBucket }, + ObjectConditions = new ObjectConditions { MinTimeElapsedSinceLastModification = Duration.FromTimeSpan(TimeSpan.FromSeconds(2592000)) }, + TransferOptions = new TransferOptions { DeleteObjectsFromSourceAfterTransfer = true }, + }, + Status = TransferJob.Types.Status.Enabled, + Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)), ScheduleEndDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } + }; - // Create a Transfer Service client - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - // Create a Transfer job - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest - { - JobName = response.Name, - ProjectId = projectId - }); + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); - Console.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with the name {response.Name}"); - return response; - } + Console.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with the name {response.Name}"); + return response; } +} // [END storagetransfer_transfer_to_nearline] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 73fab93c6fd..da490aa4b96 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -17,57 +17,57 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class TransferUsingManifestSample +public class TransferUsingManifestSample +{ + // Create a transfer from a POSIX file system to a GCS bucket using a manifest file + public TransferJob TransferUsingManifest( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent + string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads", + // The GCS bucket which has your manifest file + string manifestBucket = "my-source-bucket", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket", + // The name of the manifest file in manifestBucket that specifies which objects to transfer + string manifestObjectName = "path/to/manifest.csv") { - // Create a transfer from a POSIX file system to a GCS bucket using a manifest file - public TransferJob TransferUsingManifest( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent - string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // The root directory path on the source filesystem - string rootDirectory = "/tmp/uploads", - // The GCS bucket which has your manifest file - string manifestBucket = "my-source-bucket", - // The GCS bucket to transfer data to - string sinkBucket = "my-sink-bucket", - // The name of the manifest file in manifestBucket that specifies which objects to transfer - string manifestObjectName = "path/to/manifest.csv") - { - string manifestLocation = $"gs://{manifestBucket}/{manifestObjectName}"; + string manifestLocation = $"gs://{manifestBucket}/{manifestObjectName}"; - // A useful description for your transfer job - string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file"; + // A useful description for your transfer job + string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file"; - TransferJob transferJob = new TransferJob + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec { - ProjectId = projectId, - Description = jobDescription, - TransferSpec = new TransferSpec - { - GcsDataSink = new GcsData { BucketName = sinkBucket }, - GcsDataSource = new GcsData { BucketName = manifestBucket }, - SourceAgentPoolName = sourceAgentPoolName, - PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, - TransferManifest = new TransferManifest { Location = manifestLocation } - }, - Status = TransferJob.Types.Status.Enabled, - }; + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = manifestBucket }, + SourceAgentPoolName = sourceAgentPoolName, + PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, + TransferManifest = new TransferManifest { Location = manifestLocation } + }, + Status = TransferJob.Types.Status.Enabled, + }; - // Create a Transfer Service client - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - // Create a Transfer job - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest - { - JobName = response.Name, - ProjectId = projectId - }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); - Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}"); - return response; - } + Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}"); + return response; } +} // [END storagetransfer_manifest_request] From 332f78d72a13b0b75bb73da4dc2922bc84f266a9 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Tue, 11 Feb 2025 03:24:44 -0800 Subject: [PATCH 30/49] XML documentation is added to describe the sample methods --- .../CheckLatestTransferOperationSample.cs | 4 +++- .../CreateEventDrivenGcsTransferSample.cs | 3 +++ .../api/StorageTransfer.Samples/DownloadToPosixSample.cs | 4 +++- .../StorageTransfer.Samples/TransferBetweenPosixSample.cs | 4 +++- .../api/StorageTransfer.Samples/TransferFromPosixSample.cs | 4 +++- .../api/StorageTransfer.Samples/TransferToNearlineSample.cs | 6 ++++-- .../StorageTransfer.Samples/TransferUsingManifestSample.cs | 4 +++- 7 files changed, 22 insertions(+), 7 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 7ef3dab7d55..d3ab8445472 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -17,9 +17,11 @@ using System; using Google.Cloud.StorageTransfer.V1; +/// +/// Checks the latest transfer operation for a given transfer job. +/// public class CheckLatestTransferOperationSample { - // Checks the latest transfer operation for a given transfer job public TransferJob CheckLatestTransferOperation( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index d57407d18dc..25f0549a8d9 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -19,6 +19,9 @@ using System; using Google.Cloud.StorageTransfer.V1; +/// +/// Create an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. +/// public class CreateEventDrivenGcsTransferSample { public TransferJob CreateEventDrivenGcsTransfer( diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 5e15ee5b5de..0d6aef6867e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -17,9 +17,11 @@ using System; using Google.Cloud.StorageTransfer.V1; +/// +/// Create a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. +/// public class DownloadToPosixSample { - // Create a transfer from a GCS bucket to a POSIX file system public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index b07b0bdb118..cfefc2751bf 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -17,9 +17,11 @@ using System; using Google.Cloud.StorageTransfer.V1; +/// +/// Create a transfer to transfer objects from root directory to the destination directory between POSIX file systems. +/// public class TransferBetweenPosixSample { - // Creates a transfer between POSIX file systems public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 60e6f29af2f..4f484dde1de 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -17,9 +17,11 @@ using System; using Google.Cloud.StorageTransfer.V1; +/// +/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket. +/// public class TransferFromPosixSample { - // Create a transfer from a POSIX file system to a GCS sink bucket public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index f9bcfd443cb..282723133e3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -18,10 +18,12 @@ using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; +/// +/// Creates a one-off transfer job that transfers objects from a standard gcs bucket that are more +/// than 30 days old to a nearline gcs bucket. +/// public class TransferToNearlineSample { - // Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more - // than 30 days old to a Nearline GCS bucket public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index da490aa4b96..27db28527a5 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -17,9 +17,11 @@ using System; using Google.Cloud.StorageTransfer.V1; +/// +/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. +/// public class TransferUsingManifestSample { - // Create a transfer from a POSIX file system to a GCS bucket using a manifest file public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", From 522b48a813f57d63f720dda30f689db420c023c9 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 19 Feb 2025 03:59:19 -0800 Subject: [PATCH 31/49] samples and test linter changes --- .../CheckLatestTransferOperationTest.cs | 2 +- .../CreateEventDrivenGcsTransferTest.cs | 2 +- .../api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs | 2 +- .../api/StorageTransfer.Samples.Tests/QuickstartTest.cs | 2 +- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 2 +- .../StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs | 2 +- .../StorageTransfer.Samples.Tests/TransferFromPosixTest.cs | 2 +- .../StorageTransfer.Samples.Tests/TransferToNearlineTest.cs | 2 +- .../TransferUsingManifestTest.cs | 2 +- .../CheckLatestTransferOperationSample.cs | 2 +- .../CreateEventDrivenGcsTransferSample.cs | 3 +-- .../api/StorageTransfer.Samples/DownloadToPosixSample.cs | 4 +--- .../api/StorageTransfer.Samples/QuickstartSample.cs | 2 +- .../api/StorageTransfer.Samples/TransferBetweenPosixSample.cs | 4 +--- .../api/StorageTransfer.Samples/TransferFromPosixSample.cs | 4 +--- .../api/StorageTransfer.Samples/TransferToNearlineSample.cs | 2 +- .../StorageTransfer.Samples/TransferUsingManifestSample.cs | 2 +- 17 files changed, 17 insertions(+), 24 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index a5cbd41b2aa..0b1c68cb130 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -59,4 +59,4 @@ private string CreateTransferJob() TransferJob response = _fixture.Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); return response.Name; } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 60833043c26..161c7d69063 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -107,4 +107,4 @@ public void Dispose() } } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index b808b2b0c2a..00c71e1d74a 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -76,4 +76,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 33fa4dd687d..3a3687db9ad 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -61,4 +61,4 @@ public void Dispose() } } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index b942c5b21f6..fb767111962 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -117,4 +117,4 @@ public void Dispose() } } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 4185a094b4f..3a387cd6c5a 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -81,4 +81,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 0cfeab95a4a..2bbadffca52 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -59,4 +59,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index 6fe8e41abcb..49d8e64e813 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -62,4 +62,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index b122e99f9c0..6df7c3f1336 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -71,4 +71,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index d3ab8445472..611d4751998 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -59,4 +59,4 @@ public TransferJob CheckLatestTransferOperation( } } } -// [END storagetransfer_get_latest_transfer_operation] +// [END storagetransfer_get_latest_transfer_operation] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 25f0549a8d9..9eac88090f4 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -56,5 +56,4 @@ public TransferJob CreateEventDrivenGcsTransfer( return response; } } -// [END storagetransfer_create_event_driven_gcs_transfer] - +// [END storagetransfer_create_event_driven_gcs_transfer] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 0d6aef6867e..83172e16f99 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -66,6 +66,4 @@ public TransferJob DownloadToPosix( return response; } } -//[END storagetransfer_download_to_posix] - - +//[END storagetransfer_download_to_posix] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index 9726dfb777c..d963a11a4c7 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -52,4 +52,4 @@ public TransferJob Quickstart( return response; } } -// [END storagetransfer_quickstart] +// [END storagetransfer_quickstart] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index cfefc2751bf..202c8fd3b29 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -70,6 +70,4 @@ public TransferJob TransferBetweenPosix( return response; } } -//[END storagetransfer_transfer_posix_to_posix] - - +//[END storagetransfer_transfer_posix_to_posix] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 4f484dde1de..9bfc347ee27 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -64,6 +64,4 @@ public TransferJob TransferFromPosix( return response; } } -// [END storagetransfer_transfer_from_posix] - - +// [END storagetransfer_transfer_from_posix] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 282723133e3..be0f2176778 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -65,4 +65,4 @@ public TransferJob TransferToNearline( return response; } } -// [END storagetransfer_transfer_to_nearline] +// [END storagetransfer_transfer_to_nearline] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 27db28527a5..454836d1b81 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -72,4 +72,4 @@ public TransferJob TransferUsingManifest( return response; } } -// [END storagetransfer_manifest_request] +// [END storagetransfer_manifest_request] \ No newline at end of file From 8642fbf93f75661c81f3bbfba9dda0649925732d Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 20 Feb 2025 05:38:57 -0800 Subject: [PATCH 32/49] Structural and linter changes in tests --- .../CheckLatestTransferOperationTest.cs | 9 +- .../CreateEventDrivenGcsTransferTest.cs | 143 +++++++++--------- .../DownloadToPosixTest.cs | 15 +- .../QuickstartTest.cs | 4 + .../StorageFixture.cs | 17 +-- .../TransferBetweenPosixTest.cs | 6 +- .../TransferFromPosixTest.cs | 4 +- .../TransferToNearlineTest.cs | 5 + .../TransferUsingManifestTest.cs | 10 +- 9 files changed, 119 insertions(+), 94 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index 0b1c68cb130..9462c17b69f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -using Google.Cloud.Storage.V1; + using Google.Cloud.StorageTransfer.V1; using Xunit; @@ -20,13 +20,17 @@ namespace StorageTransfer.Samples.Tests; [Collection(nameof(StorageFixture))] public class CheckLatestTransferOperationTest { - private readonly StorageFixture _fixture; private string _jobName; private readonly string _transferJobName; + public CheckLatestTransferOperationTest(StorageFixture fixture) { _fixture = fixture; + _fixture.BucketNameSource = _fixture.GenerateBucketName(); + _fixture.BucketNameSink = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); _transferJobName = CreateTransferJob(); } @@ -38,6 +42,7 @@ public void CheckLatestTransferOperation() Assert.Contains("transferJobs/", transferJob.Name); _jobName = transferJob.Name; } + private string CreateTransferJob() { // Initialize request argument(s) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 161c7d69063..14bc4f1b267 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -16,95 +16,98 @@ using System; using Google.Cloud.PubSub.V1; -using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; -namespace StorageTransfer.Samples.Tests +namespace StorageTransfer.Samples.Tests; + +[Collection(nameof(StorageFixture))] +public class CreateEventDrivenGcsTransferTest : IDisposable { - [Collection(nameof(StorageFixture))] - public class CreateEventDrivenGcsTransferTest : IDisposable + private readonly StorageFixture _fixture; + private readonly string _pubSubId; + private string _transferJobName; + private string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); + private string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); + private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); + private PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); + + public CreateEventDrivenGcsTransferTest(StorageFixture fixture) { - private readonly StorageFixture _fixture; - private readonly string _pubSubId; - private string _transferJobName; - private string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); - private string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); - private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); - private PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); - public CreateEventDrivenGcsTransferTest(StorageFixture fixture) + _fixture = fixture; + _fixture.BucketNameSource = _fixture.GenerateBucketName(); + _fixture.BucketNameSink = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _pubSubId = $"projects/{_fixture.ProjectId}/subscriptions/{SubscriptionId}"; + string email = _fixture.Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest() { - _fixture = fixture; - _pubSubId = $"projects/{_fixture.ProjectId}/subscriptions/{SubscriptionId}"; - string email = _fixture.Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest() - { - ProjectId = _fixture.ProjectId - }).AccountEmail; + ProjectId = _fixture.ProjectId + }).AccountEmail; - string memberServiceAccount = "serviceAccount:" + email; - // Create subscription name - SubscriptionName subscriptionName = new SubscriptionName(_fixture.ProjectId, SubscriptionId); - // Create topic name - TopicName topicName = new TopicName(_fixture.ProjectId, TopicId); - // Create topic - PublisherClient.CreateTopic(topicName); - // Create subscription. - SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500); + string memberServiceAccount = "serviceAccount:" + email; + // Create subscription name + SubscriptionName subscriptionName = new SubscriptionName(_fixture.ProjectId, SubscriptionId); + // Create topic name + TopicName topicName = new TopicName(_fixture.ProjectId, TopicId); + // Create topic + PublisherClient.CreateTopic(topicName); + // Create subscription. + SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500); - var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy(); + var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy(); - policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount); - - PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest - { - ResourceAsResourceName = topicName, - Policy = policyIamPolicyTopic - }); + policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount); - var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy(); + PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest + { + ResourceAsResourceName = topicName, + Policy = policyIamPolicyTopic + }); - policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount); + var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy(); - PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest - { - ResourceAsResourceName = subscriptionName, - Policy = policyIamPolicySubscriber - }); - } + policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount); - [Fact] - public void CreateEventDrivenGcsTransfer() + PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest { - CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); - var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _pubSubId); - Assert.Contains("transferJobs/", transferJob.Name); - _transferJobName = transferJob.Name; - } + ResourceAsResourceName = subscriptionName, + Policy = policyIamPolicySubscriber + }); + } + + [Fact] + public void CreateEventDrivenGcsTransfer() + { + CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); + var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _pubSubId); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } - public void Dispose() + public void Dispose() + { + try { - try + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() { - _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() { - ProjectId = _fixture.ProjectId, - JobName = _transferJobName, - TransferJob = new TransferJob() - { - Name = _transferJobName, - Status = TransferJob.Types.Status.Deleted - } - }); + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); - TopicName topicName = TopicName.FromProjectTopic(_fixture.ProjectId, TopicId); - PublisherClient.DeleteTopic(topicName); - SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(_fixture.ProjectId, SubscriptionId); - SubscriberClient.DeleteSubscription(subscriptionName); - } - catch (Exception) - { - // Do nothing, we delete on a best effort basis. - } + TopicName topicName = TopicName.FromProjectTopic(_fixture.ProjectId, TopicId); + PublisherClient.DeleteTopic(topicName); + SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(_fixture.ProjectId, SubscriptionId); + SubscriberClient.DeleteSubscription(subscriptionName); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. } } } \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 00c71e1d74a..75c4f77918d 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -14,14 +14,12 @@ using System; using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; + [Collection(nameof(StorageFixture))] public class DownloadToPosixTest : IDisposable { @@ -29,12 +27,15 @@ public class DownloadToPosixTest : IDisposable private string _transferJobName; private readonly string _tempDirectory; private readonly string _gcsSourcePath; + private readonly string _bucketNamePosixSource; public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; - _tempDirectory = fixture.GenerateTempFolderPath(); + _bucketNamePosixSource = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_bucketNamePosixSource); + _tempDirectory = _fixture.GenerateTempFolderPath(); _gcsSourcePath = $"{Guid.NewGuid()}/{Guid.NewGuid()}/"; - UploadObjectToPosixBucket(_fixture.BucketNamePosixSource); + UploadObjectToPosixBucket(_bucketNamePosixSource); } [Fact] @@ -42,11 +43,12 @@ public void DownloadToPosix() { DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(); Directory.CreateDirectory(_tempDirectory); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNamePosixSource, _gcsSourcePath, _tempDirectory); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _bucketNamePosixSource, _gcsSourcePath, _tempDirectory); Assert.Contains("transferJobs/", transferJob.Name); Assert.True(Directory.Exists(_tempDirectory)); _transferJobName = transferJob.Name; } + private void UploadObjectToPosixBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); @@ -70,6 +72,7 @@ public void Dispose() } }); Directory.Delete(_tempDirectory, true); + _fixture.Storage.DeleteBucket(_bucketNamePosixSource, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 3a3687db9ad..f8f67687ef7 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -29,6 +29,10 @@ public class QuickstartTest : IDisposable public QuickstartTest(StorageFixture fixture) { _fixture = fixture; + _fixture.BucketNameSource = _fixture.GenerateBucketName(); + _fixture.BucketNameSink = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } [Fact] diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index fb767111962..43aff5dbbf4 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -28,10 +28,8 @@ namespace StorageTransfer.Samples.Tests public class StorageFixture : IDisposable, ICollectionFixture { public string ProjectId { get; } - public string BucketNameSource { get; } = Guid.NewGuid().ToString(); - public string BucketNameSink { get; } = Guid.NewGuid().ToString(); - public string BucketNameManifestSource { get; } = Guid.NewGuid().ToString(); - public string BucketNamePosixSource { get; } = Guid.NewGuid().ToString(); + public string BucketNameSource { get; set; } + public string BucketNameSink { get; set; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } public StorageClient Storage { get; } = StorageClient.Create(); @@ -46,13 +44,9 @@ public StorageFixture() { throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id."); } - - CreateBucketAndGrantStsPermissions(BucketNameSink); - CreateBucketAndGrantStsPermissions(BucketNameSource); - CreateBucketAndGrantStsPermissions(BucketNameManifestSource); - CreateBucketAndGrantStsPermissions(BucketNamePosixSource); } - private void CreateBucketAndGrantStsPermissions(string bucketName) + + internal void CreateBucketAndGrantStsPermissions(string bucketName) { var bucket = Storage.CreateBucket(ProjectId, new Bucket { @@ -95,8 +89,11 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) policy.Bindings.Add(bucketWriterBinding); Storage.SetBucketIamPolicy(bucketName, policy); } + + internal string GenerateBucketName() => Guid.NewGuid().ToString(); internal string GetCurrentUserTempFolderPath() => System.IO.Path.GetTempPath(); internal string GenerateTempFolderPath() => Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + public void Dispose() { try diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 3a387cd6c5a..be9980c3927 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -14,13 +14,11 @@ using System; using System.IO; -using System.Linq; -using System.Xml.Linq; -using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; + [Collection(nameof(StorageFixture))] public class TransferBetweenPosixTest : IDisposable { @@ -31,6 +29,8 @@ public class TransferBetweenPosixTest : IDisposable public TransferBetweenPosixTest(StorageFixture fixture) { _fixture = fixture; + _fixture.BucketNameSource = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); _tempDirectory = fixture.GenerateTempFolderPath(); _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 2bbadffca52..49e23e5a7dc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -13,11 +13,11 @@ // limitations under the License. using System; -using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; + [Collection(nameof(StorageFixture))] public class TransferFromPosixTest : IDisposable { @@ -27,6 +27,8 @@ public class TransferFromPosixTest : IDisposable public TransferFromPosixTest(StorageFixture fixture) { _fixture = fixture; + _fixture.BucketNameSink = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index 49d8e64e813..dd2617e5000 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -18,6 +18,7 @@ using Xunit; namespace StorageTransfer.Samples.Tests; + [Collection(nameof(StorageFixture))] public class TransferToNearlineTest : IDisposable { @@ -26,6 +27,10 @@ public class TransferToNearlineTest : IDisposable public TransferToNearlineTest(StorageFixture fixture) { _fixture = fixture; + _fixture.BucketNameSource = _fixture.GenerateBucketName(); + _fixture.BucketNameSink = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } [Fact] diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 6df7c3f1336..74a3b3ccb18 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -27,19 +27,24 @@ public class TransferUsingManifestTest : IDisposable private string _transferJobName; private readonly string _rootDirectory; private readonly string _manifestObjectName; + private readonly string _bucketNameManifestSource; public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; + _bucketNameManifestSource = _fixture.GenerateBucketName(); + _fixture.BucketNameSink = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_bucketNameManifestSource); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); _rootDirectory = fixture.GetCurrentUserTempFolderPath(); _manifestObjectName = $@"{Guid.NewGuid()}.csv"; - UploadObjectToManifestBucket(_fixture.BucketNameManifestSource); + UploadObjectToManifestBucket(_bucketNameManifestSource); } [Fact] public void TransferUsingManifest() { TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(); - var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _fixture.BucketNameManifestSource, _fixture.BucketNameSink, _manifestObjectName); + var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _bucketNameManifestSource, _fixture.BucketNameSink, _manifestObjectName); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -65,6 +70,7 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + _fixture.Storage.DeleteBucket(_bucketNameManifestSource, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) { From 72872613914e196cd5053431d4330de4207da7e1 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Fri, 21 Feb 2025 02:56:11 -0800 Subject: [PATCH 33/49] linter changes in tests --- .../api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs | 1 + .../StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs | 1 + .../api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs | 1 + .../api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs | 1 + .../StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs | 1 + 5 files changed, 5 insertions(+) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 75c4f77918d..0313f5f3ebe 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -28,6 +28,7 @@ public class DownloadToPosixTest : IDisposable private readonly string _tempDirectory; private readonly string _gcsSourcePath; private readonly string _bucketNamePosixSource; + public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index be9980c3927..7aad45caf56 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -26,6 +26,7 @@ public class TransferBetweenPosixTest : IDisposable private string _transferJobName; private readonly string _tempDirectory; private readonly string _tempDestinationDirectory; + public TransferBetweenPosixTest(StorageFixture fixture) { _fixture = fixture; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 49e23e5a7dc..e8f416e80da 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -24,6 +24,7 @@ public class TransferFromPosixTest : IDisposable private readonly StorageFixture _fixture; private string _transferJobName; private readonly string _rootDirectory; + public TransferFromPosixTest(StorageFixture fixture) { _fixture = fixture; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index dd2617e5000..d9bfbefb88b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -24,6 +24,7 @@ public class TransferToNearlineTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + public TransferToNearlineTest(StorageFixture fixture) { _fixture = fixture; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 74a3b3ccb18..bef0970d34a 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -28,6 +28,7 @@ public class TransferUsingManifestTest : IDisposable private readonly string _rootDirectory; private readonly string _manifestObjectName; private readonly string _bucketNameManifestSource; + public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; From 904c9890789cbdf4b2870a47a2b49a45875a2d5a Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Fri, 21 Feb 2025 03:35:54 -0800 Subject: [PATCH 34/49] reuse of existing fixture's storage client in transfer to nearline test --- .../StorageTransfer.Samples.Tests/TransferToNearlineTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index d9bfbefb88b..c782bf237f5 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -38,11 +38,11 @@ public TransferToNearlineTest(StorageFixture fixture) public void TestTransferToNearline() { TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(); - var storage = StorageClient.Create(); + var storage = _fixture.Storage; var bucket = storage.GetBucket(_fixture.BucketNameSink); string storageClass = StorageClasses.Nearline; bucket.StorageClass = storageClass; - bucket = storage.UpdateBucket(bucket); + storage.UpdateBucket(bucket); var transferJob = transferToNearlineSample.TransferToNearline(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; From 1b0d4faa29e9ee6edf23ae8e8173246b2a518ed8 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 26 Feb 2025 05:11:30 -0800 Subject: [PATCH 35/49] linter changes as per .editorconfig file at the root of repository --- .../CheckLatestTransferOperationTest.cs | 2 +- .../CreateEventDrivenGcsTransferTest.cs | 4 ++-- .../StorageTransfer.Samples.Tests/DownloadToPosixTest.cs | 6 +++--- .../api/StorageTransfer.Samples.Tests/QuickstartTest.cs | 4 ++-- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 8 ++++---- .../TransferBetweenPosixTest.cs | 4 ++-- .../TransferFromPosixTest.cs | 4 ++-- .../TransferToNearlineTest.cs | 4 ++-- .../TransferUsingManifestTest.cs | 6 +++--- .../CheckLatestTransferOperationSample.cs | 4 ++-- .../CreateEventDrivenGcsTransferSample.cs | 4 ++-- .../api/StorageTransfer.Samples/DownloadToPosixSample.cs | 4 ++-- .../api/StorageTransfer.Samples/QuickstartSample.cs | 4 ++-- .../StorageTransfer.Samples/TransferBetweenPosixSample.cs | 4 ++-- .../StorageTransfer.Samples/TransferFromPosixSample.cs | 4 ++-- .../StorageTransfer.Samples/TransferToNearlineSample.cs | 4 ++-- .../TransferUsingManifestSample.cs | 4 ++-- 17 files changed, 37 insertions(+), 37 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index 9462c17b69f..723f566098d 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -64,4 +64,4 @@ private string CreateTransferJob() TransferJob response = _fixture.Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); return response.Name; } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 14bc4f1b267..cba7c2b99fd 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -14,9 +14,9 @@ * limitations under the License. */ -using System; using Google.Cloud.PubSub.V1; using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -110,4 +110,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 0313f5f3ebe..ac9116f3abc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; -using System.IO; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; +using System; +using System.IO; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -80,4 +80,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index f8f67687ef7..9efbb3334a1 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -14,8 +14,8 @@ * limitations under the License. */ -using System; using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; namespace StorageTransfer.Samples.Tests @@ -65,4 +65,4 @@ public void Dispose() } } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 43aff5dbbf4..fbd53117768 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -14,12 +14,12 @@ * limitations under the License. */ -using System; -using System.Collections.Generic; -using System.IO; using Google.Apis.Storage.v1.Data; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; +using System; +using System.Collections.Generic; +using System.IO; using Xunit; namespace StorageTransfer.Samples.Tests @@ -114,4 +114,4 @@ public void Dispose() } } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 7aad45caf56..f267930f566 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +using Google.Cloud.StorageTransfer.V1; using System; using System.IO; -using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -82,4 +82,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index e8f416e80da..88f763d25fc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -62,4 +62,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index c782bf237f5..eacce91d255 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -68,4 +68,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index bef0970d34a..58711707643 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; -using System.IO; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; +using System; +using System.IO; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -78,4 +78,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 611d4751998..4005ef76341 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -14,8 +14,8 @@ // [START storagetransfer_get_latest_transfer_operation] -using System; using Google.Cloud.StorageTransfer.V1; +using System; /// /// Checks the latest transfer operation for a given transfer job. @@ -59,4 +59,4 @@ public TransferJob CheckLatestTransferOperation( } } } -// [END storagetransfer_get_latest_transfer_operation] \ No newline at end of file +// [END storagetransfer_get_latest_transfer_operation] diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 9eac88090f4..49465a395f3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -16,8 +16,8 @@ // [START storagetransfer_create_event_driven_gcs_transfer] -using System; using Google.Cloud.StorageTransfer.V1; +using System; /// /// Create an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. @@ -56,4 +56,4 @@ public TransferJob CreateEventDrivenGcsTransfer( return response; } } -// [END storagetransfer_create_event_driven_gcs_transfer] \ No newline at end of file +// [END storagetransfer_create_event_driven_gcs_transfer] diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 83172e16f99..cd92b85e2c3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -14,8 +14,8 @@ // [START storagetransfer_download_to_posix] -using System; using Google.Cloud.StorageTransfer.V1; +using System; /// /// Create a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. @@ -66,4 +66,4 @@ public TransferJob DownloadToPosix( return response; } } -//[END storagetransfer_download_to_posix] \ No newline at end of file +//[END storagetransfer_download_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index d963a11a4c7..fe08f6082c2 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -16,8 +16,8 @@ // [START storagetransfer_quickstart] -using System; using Google.Cloud.StorageTransfer.V1; +using System; public class QuickstartSample { @@ -52,4 +52,4 @@ public TransferJob Quickstart( return response; } } -// [END storagetransfer_quickstart] \ No newline at end of file +// [END storagetransfer_quickstart] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 202c8fd3b29..5d8abed0ce5 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -14,8 +14,8 @@ // [START storagetransfer_transfer_posix_to_posix] -using System; using Google.Cloud.StorageTransfer.V1; +using System; /// /// Create a transfer to transfer objects from root directory to the destination directory between POSIX file systems. @@ -70,4 +70,4 @@ public TransferJob TransferBetweenPosix( return response; } } -//[END storagetransfer_transfer_posix_to_posix] \ No newline at end of file +//[END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 9bfc347ee27..bc82f8f74dc 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -14,8 +14,8 @@ // [START storagetransfer_transfer_from_posix] -using System; using Google.Cloud.StorageTransfer.V1; +using System; /// /// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket. @@ -64,4 +64,4 @@ public TransferJob TransferFromPosix( return response; } } -// [END storagetransfer_transfer_from_posix] \ No newline at end of file +// [END storagetransfer_transfer_from_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index be0f2176778..98dcfb0406d 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -14,9 +14,9 @@ // [START storagetransfer_transfer_to_nearline] -using System; using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; +using System; /// /// Creates a one-off transfer job that transfers objects from a standard gcs bucket that are more @@ -65,4 +65,4 @@ public TransferJob TransferToNearline( return response; } } -// [END storagetransfer_transfer_to_nearline] \ No newline at end of file +// [END storagetransfer_transfer_to_nearline] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 454836d1b81..c92860b2174 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -14,8 +14,8 @@ // [START storagetransfer_manifest_request] -using System; using Google.Cloud.StorageTransfer.V1; +using System; /// /// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. @@ -72,4 +72,4 @@ public TransferJob TransferUsingManifest( return response; } } -// [END storagetransfer_manifest_request] \ No newline at end of file +// [END storagetransfer_manifest_request] From dda9b52823e268e7dd39707219e393bcd90dfed4 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 12 Mar 2025 00:49:13 -0700 Subject: [PATCH 36/49] exception handling removed from CheckLatestTransferOperation --- .../CheckLatestTransferOperationTest.cs | 24 +++++++++++++- .../CheckLatestTransferOperationSample.cs | 33 +++++++------------ 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index 723f566098d..f9107e38d98 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -13,12 +13,13 @@ // limitations under the License. using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; namespace StorageTransfer.Samples.Tests; [Collection(nameof(StorageFixture))] -public class CheckLatestTransferOperationTest +public class CheckLatestTransferOperationTest : IDisposable { private readonly StorageFixture _fixture; private string _jobName; @@ -64,4 +65,25 @@ private string CreateTransferJob() TransferJob response = _fixture.Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); return response.Name; } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } } diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 4005ef76341..f0f188b04e2 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -28,35 +28,24 @@ public TransferJob CheckLatestTransferOperation( // The name of the job to check string jobName = "transferJobs/1234567890") { - if (string.IsNullOrEmpty(jobName)) - { - throw new Exception("JobName can not be null or empty"); - } - // Create a Transfer Service client StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.Create(); GetTransferJobRequest getTransferJobRequest = new GetTransferJobRequest { ProjectId = projectId, JobName = jobName }; - try - { - // Get Transfer job - TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); - // Get Latest operation name from transfer job - string latestOperationName = transferJob.LatestOperationName; - if (!string.IsNullOrEmpty(latestOperationName)) - { - Console.WriteLine("The latest operation for transfer job " + jobName + " is: " + latestOperationName + ""); - } - else - { - Console.WriteLine("Transfer job " + jobName + " hasn't run yet, try again after the job has started running."); - } - return transferJob; + // Get Transfer job + TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); + // Get Latest operation name from transfer job + string latestOperationName = transferJob.LatestOperationName; + + if (!string.IsNullOrEmpty(latestOperationName)) + { + Console.WriteLine("The latest operation for transfer job " + jobName + " is: " + latestOperationName + ""); } - catch (Exception) + else { - throw new Exception("Failed to get transfer job " + jobName + ""); + Console.WriteLine("Transfer job " + jobName + " hasn't run yet, try again after the job has started running."); } + return transferJob; } } // [END storagetransfer_get_latest_transfer_operation] From f1b05b37942aa131ccff60fdd9084adfedd39260 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 27 Mar 2025 03:51:36 -0700 Subject: [PATCH 37/49] Comments added in tests and sample description modified --- storagetransfer/api/.editorconfig | 281 ++++++++++++++++++ .../CheckLatestTransferOperationTest.cs | 2 + .../CreateEventDrivenGcsTransferTest.cs | 1 + .../DownloadToPosixTest.cs | 2 + .../QuickstartTest.cs | 1 + .../TransferBetweenPosixTest.cs | 1 + .../TransferFromPosixTest.cs | 1 + .../TransferToNearlineTest.cs | 1 + .../TransferUsingManifestTest.cs | 2 + .../CheckLatestTransferOperationSample.cs | 8 +- .../CreateEventDrivenGcsTransferSample.cs | 10 +- .../DownloadToPosixSample.cs | 11 +- .../QuickstartSample.cs | 6 + .../TransferBetweenPosixSample.cs | 18 +- .../TransferFromPosixSample.cs | 10 +- .../TransferToNearlineSample.cs | 11 +- .../TransferUsingManifestSample.cs | 13 +- testutil/TestBucket.cs | 4 +- testutil/TestUtil.cs | 26 +- 19 files changed, 369 insertions(+), 40 deletions(-) create mode 100644 storagetransfer/api/.editorconfig diff --git a/storagetransfer/api/.editorconfig b/storagetransfer/api/.editorconfig new file mode 100644 index 00000000000..0ec4d4be40e --- /dev/null +++ b/storagetransfer/api/.editorconfig @@ -0,0 +1,281 @@ +# To learn more about .editorconfig see https://aka.ms/editorconfigdocs +# top-most EditorConfig file +root = true +############################### +# Core EditorConfig Options # +############################### +# All files +[*] +insert_final_newline = true +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true + +# Generated code +[*{_AssemblyInfo.cs,.g.cs}] +generated_code = true + +# XML project files +[*.{csproj}] +indent_size = 2 + +# Code files + +############################### +# .NET Coding Conventions # +############################### +[*.{cs}] + +# Organize usings +dotnet_sort_system_directives_first = false + +# this. preferences +dotnet_style_qualification_for_field = false:suggestion +dotnet_style_qualification_for_property = false:suggestion +dotnet_style_qualification_for_method = false:suggestion +dotnet_style_qualification_for_event = false:suggestion + +# Language keywords vs BCL types preferences +dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion +dotnet_style_predefined_type_for_member_access = true:suggestion + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent + +# Modifier preferences +csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning +dotnet_style_require_accessibility_modifiers = for_non_interface_members:warning +dotnet_style_readonly_field = true:warning + +# Expression-level preferences +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_explicit_tuple_names = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion +dotnet_style_prefer_inferred_tuple_names = true:suggestion +dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion +dotnet_style_prefer_auto_properties = true:suggestion +dotnet_style_prefer_conditional_expression_over_assignment = true:silent +dotnet_style_prefer_conditional_expression_over_return = true:silent +csharp_prefer_simple_default_expression = true:suggestion + +############################### +# Naming Conventions # +############################### +# Style Definitions +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# Use PascalCase for constant fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.applicable_accessibilities = * +dotnet_naming_symbols.constant_fields.required_modifiers = const +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# static fields should have s_ prefix +dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion +dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields +dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style +dotnet_naming_symbols.static_fields.applicable_kinds = field +dotnet_naming_symbols.static_fields.required_modifiers = static +dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected +dotnet_naming_style.static_prefix_style.required_prefix = s_ +dotnet_naming_style.static_prefix_style.capitalization = camel_case + +# internal and private fields should be _camelCase +dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion +dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields +dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style +dotnet_naming_symbols.private_internal_fields.applicable_kinds = field +dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal +dotnet_naming_style.camel_case_underscore_style.required_prefix = _ +dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case + +# Locals and parameters are camelCase +dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion +dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters +dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style +dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local +dotnet_naming_style.camel_case_style.capitalization = camel_case + +# Public property and methods should be PascalCase +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.symbols = public_methods_and_properties +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.style = pascal_case +dotnet_naming_symbols.public_methods_and_properties.applicable_kinds = method, property +dotnet_naming_symbols.public_methods_and_properties.applicable_accessibilities = public +dotnet_naming_style.pascal_case.capitalization = pascal_case + +# Private and internal methods should be PascalCase +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.symbols = private_and_internal_methods +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.style = pascal_case +dotnet_naming_symbols.private_and_internal_methods.applicable_kinds = method +dotnet_naming_symbols.private_and_internal_methods.applicable_accessibilities = private, internal +############################### +# C# Coding Conventions # +############################### +[*.cs] +# var preferences +csharp_style_var_for_built_in_types = true:silent +csharp_style_var_when_type_is_apparent = true:silent +csharp_style_var_elsewhere = true:silent +dotnet_style_predefined_type_for_locals_parameters_members = true:silent +dotnet_style_predefined_type_for_member_access = true:silent + +csharp_using_directive_placement = outside_namespace:warning +csharp_prefer_braces = true:suggestion +csharp_preserve_single_line_blocks = true:none +csharp_preserve_single_line_statements = false:none +csharp_prefer_static_local_function = true:suggestion +csharp_prefer_simple_using_statement = true:suggestion +csharp_style_prefer_switch_expression = true:suggestion + +# Expression-bodied members +csharp_style_expression_bodied_methods = true:suggestion +csharp_style_expression_bodied_constructors = true:suggestion +csharp_style_expression_bodied_operators = true:suggestion +csharp_style_expression_bodied_properties = true:suggestion +csharp_style_expression_bodied_indexers = true:suggestion +csharp_style_expression_bodied_accessors = true:suggestion +csharp_style_expression_bodied_lambdas = true:suggestion +csharp_style_expression_bodied_local_functions = true:suggestion + +# Pattern matching preferences +csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion +csharp_style_pattern_matching_over_as_with_null_check = true:suggestion +csharp_style_inlined_variable_declaration = true:suggestion + +# Null-checking preferences +csharp_style_throw_expression = true:suggestion +csharp_style_conditional_delegate_call = true:suggestion + +# Other features +csharp_style_prefer_index_operator = true:suggestion +csharp_style_prefer_range_operator = true:suggestion +csharp_style_pattern_local_over_anonymous_function = false:none + +############################### +# C# Formatting Rules # +############################### +indent_size = 4 +insert_final_newline = true +charset = utf-8 + +# New line preferences +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation preferences +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_switch_labels = true +csharp_indent_labels = one_less_than_current + +# Space preferences +csharp_space_after_cast = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_after_comma = true +csharp_space_after_dot = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_after_semicolon_in_for_statement = true +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_parentheses = false +csharp_space_before_comma = false +csharp_space_before_dot = false +csharp_space_before_open_square_brackets = false +csharp_space_before_semicolon_in_for_statement = false +csharp_space_between_empty_square_brackets = false +csharp_space_around_declaration_statements = do_not_ignore +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_square_brackets = false + +# Wrapping preferences +csharp_preserve_single_line_statements = true +csharp_preserve_single_line_blocks = true + +# License header +file_header_template = Copyright 2024 Google LLC\n\nLicensed under the Apache License, Version 2.0 (the "License").\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at \n\nhttps://www.apache.org/licenses/LICENSE-2.0 \n\nUnless required by applicable law or agreed to in writing, software \ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and \nlimitations under the License. +csharp_style_namespace_declarations = file_scoped:suggestion +csharp_style_prefer_null_check_over_type_check = true:suggestion +csharp_style_prefer_local_over_anonymous_function = true:suggestion +csharp_style_prefer_tuple_swap = true:suggestion +csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion +csharp_style_unused_value_assignment_preference = discard_variable:suggestion +csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion +csharp_style_prefer_pattern_matching = true:suggestion +csharp_style_prefer_not_pattern = true:suggestion +csharp_style_prefer_extended_property_pattern = true:suggestion +csharp_style_prefer_parameter_null_checking = true:suggestion +csharp_style_prefer_method_group_conversion = true:suggestion + +# Change the severity to `error` for the rules that must not be violated, to fail the build. + +# IDE0005: Remove unnecessary using directives. +dotnet_diagnostic.IDE0005.severity = suggestion + +# IDE0011: Add braces +dotnet_diagnostic.IDE0011.severity = suggestion + +# IDE0017: Object initialization can be simplified. +dotnet_diagnostic.IDE0017.severity = suggestion + +# IDE0019: Use pattern matching. +dotnet_diagnostic.IDE0019.severity = suggestion + +# IDE0021: Use expression body for constructor. +dotnet_diagnostic.IDE0021.severity = suggestion + +# IDE0028: Collection initialization can be simplified. +dotnet_diagnostic.IDE0028.severity = suggestion + +# IDE0035: Remove unreachable code. +dotnet_diagnostic.IDE0035.severity = suggestion + +# IDE0040: Accessibility modifiers required. +dotnet_diagnostic.IDE0040.severity = suggestion + +# IDE0052: Remove unread private member +dotnet_diagnostic.IDE0052.severity = suggestion + +# IDE0053: Use expression body for lambda expression. +dotnet_diagnostic.IDE0053.severity = suggestion + +# IDE0055: All C# and .NET formatting rules. +dotnet_diagnostic.IDE0055.severity = suggestion + +# IDE0060: Remove unused parameter +dotnet_diagnostic.IDE0060.severity = suggestion + +# IDE0063: Using statement can be simplified +dotnet_diagnostic.IDE0063.severity = suggestion + +# IDE0065: Using statement can be simplified +dotnet_diagnostic.IDE0065.severity = suggestion + +# IDE0071: Interpolation can be simplified. +dotnet_diagnostic.IDE0071.severity = suggestion + +# IDE0075: Simplify conditional expression. +dotnet_diagnostic.IDE0075.severity = suggestion \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index f9107e38d98..f798dc5b882 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -35,6 +35,7 @@ public CheckLatestTransferOperationTest(StorageFixture fixture) _transferJobName = CreateTransferJob(); } + // Checks the latest transfer operation for a created transfer job. [Fact] public void CheckLatestTransferOperation() { @@ -44,6 +45,7 @@ public void CheckLatestTransferOperation() _jobName = transferJob.Name; } + // Creates one-time transfer job from a Google cloud storage bucket to another bucket. private string CreateTransferJob() { // Initialize request argument(s) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index cba7c2b99fd..54a85805ea3 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -76,6 +76,7 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) }); } + // Transfers object from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. [Fact] public void CreateEventDrivenGcsTransfer() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index ac9116f3abc..bc3c3247fdf 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -39,6 +39,7 @@ public DownloadToPosixTest(StorageFixture fixture) UploadObjectToPosixBucket(_bucketNamePosixSource); } + // Downloads an object from a GCS bucket to a POSIX file system. [Fact] public void DownloadToPosix() { @@ -50,6 +51,7 @@ public void DownloadToPosix() _transferJobName = transferJob.Name; } + // Uploads an object to the POSIX bucket. private void UploadObjectToPosixBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 9efbb3334a1..68f5868e5ae 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -35,6 +35,7 @@ public QuickstartTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } + // Creates a one-time transfer job from a Google cloud storage bucket to another bucket [Fact] public void TestQuickstart() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index f267930f566..1b5d8045cfe 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -36,6 +36,7 @@ public TransferBetweenPosixTest(StorageFixture fixture) _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } + // Transfers objects from a POSIX file system to another POSIX file system. [Fact] public void TransferBetweenPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 88f763d25fc..b81eeb7689b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -33,6 +33,7 @@ public TransferFromPosixTest(StorageFixture fixture) _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } + // Transfers objects from a POSIX file system to a Cloud Storage bucket. [Fact] public void TransferFromPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index eacce91d255..d0eaa9a6f00 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -34,6 +34,7 @@ public TransferToNearlineTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } + // Transfers objects from a Google Cloud Storage bucket to another bucket with Nearline storage class. [Fact] public void TestTransferToNearline() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 58711707643..dde52fc6548 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -41,6 +41,7 @@ public TransferUsingManifestTest(StorageFixture fixture) UploadObjectToManifestBucket(_bucketNameManifestSource); } + // Transfers objects from a POSIX file system to a sink bucket using a manifest file. [Fact] public void TransferUsingManifest() { @@ -50,6 +51,7 @@ public void TransferUsingManifest() _transferJobName = transferJob.Name; } + // Uploads an object to the manifest bucket. private void UploadObjectToManifestBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index f0f188b04e2..04df26cec05 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -17,11 +17,13 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Checks the latest transfer operation for a given transfer job. -/// public class CheckLatestTransferOperationSample { + /// + /// Sample that checks the latest transfer operation for a given transfer job. + /// + /// Your Google Cloud Project ID. + /// The name of the job to check. public TransferJob CheckLatestTransferOperation( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 49465a395f3..3619408aca0 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -19,11 +19,15 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. -/// public class CreateEventDrivenGcsTransferSample { + /// + /// Sample that creates an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. + /// + /// Your Google Cloud Project ID. + /// The GCS bucket to transfer data from. + /// The GCS bucket to transfer data to. + /// The subscription ID to a Pubsub queue to track. public TransferJob CreateEventDrivenGcsTransfer( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index cd92b85e2c3..e0d78bf742b 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -17,11 +17,16 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. -/// public class DownloadToPosixSample { + /// + /// Sample that creates a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. + /// + /// Your Google Cloud Project ID. + /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. + /// Your GCS source bucket name. + /// An optional path on the Google Cloud Storage bucket to download from. + /// The root directory path on the source filesystem. public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index fe08f6082c2..a2a5bef3d84 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -21,6 +21,12 @@ public class QuickstartSample { + /// + /// Sample that creates a one-time transfer job from a Google cloud storage bucket to another bucket. + /// + /// Your Google Cloud Project ID. + /// The GCS bucket to transfer data from. + /// The GCS bucket to transfer data to. public TransferJob Quickstart( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 5d8abed0ce5..ba00450564e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -17,11 +17,17 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer to transfer objects from root directory to the destination directory between POSIX file systems. -/// public class TransferBetweenPosixSample { + /// + /// Sample that creates a transfer to transfer objects from root directory to the destination directory between POSIX file systems. + /// + /// The ID of the project. + /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. + /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. + /// The root directory path on the source filesystem. + /// The root directory path on the sink filesystem. + /// The name of GCS bucket for intermediate storage. public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -34,7 +40,7 @@ public TransferJob TransferBetweenPosix( // The root directory path on the sink filesystem string destinationDirectory = "/directory/to/transfer/sink", // The name of GCS bucket for intermediate storage - string intermediate_bucket = "my-intermediate-bucket") + string intermediateBucket = "my-intermediate-bucket") { // A useful description for your transfer job string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; @@ -49,7 +55,7 @@ public TransferJob TransferBetweenPosix( SinkAgentPoolName = sinkAgentPoolName, PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, PosixDataSink = new PosixFilesystem { RootDirectory = destinationDirectory }, - GcsIntermediateDataLocation = new GcsData { BucketName = intermediate_bucket } + GcsIntermediateDataLocation = new GcsData { BucketName = intermediateBucket } }, Status = TransferJob.Types.Status.Enabled, }; @@ -70,4 +76,4 @@ public TransferJob TransferBetweenPosix( return response; } } -//[END storagetransfer_transfer_posix_to_posix] +// [END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index bc82f8f74dc..1307acff0b3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -17,11 +17,15 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket. -/// public class TransferFromPosixSample { + /// + /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket. + /// + /// The ID of the project. + /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. + /// The root directory path on the source filesystem. + /// The GCS bucket to transfer data to. public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 98dcfb0406d..deaeca3fe3e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -18,12 +18,15 @@ using Google.Protobuf.WellKnownTypes; using System; -/// -/// Creates a one-off transfer job that transfers objects from a standard gcs bucket that are more -/// than 30 days old to a nearline gcs bucket. -/// public class TransferToNearlineSample { + /// + /// Sample that creates a one-off transfer job that transfers objects from a standard gcs bucket that are more + /// than 30 days old to a nearline gcs bucket. + /// + /// Your Google Cloud Project ID. + /// The GCS bucket to transfer objects from. + /// The GCS Nearline bucket to transfer old objects to. public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index c92860b2174..8154a36605c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -17,11 +17,18 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. -/// public class TransferUsingManifestSample { + /// + /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. + /// + /// The ID of the project. + /// The agent pool associated with the POSIX data source. If not provided , + /// defaults to the default agent + /// The root directory path on the source filesystem. + /// The GCS bucket which has your manifest file. + /// The GCS bucket to transfer data to. + /// The name of the manifest file in manifestBucket that specifies which objects to transfer. public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/testutil/TestBucket.cs b/testutil/TestBucket.cs index c5afc307d6d..f07f6406574 100644 --- a/testutil/TestBucket.cs +++ b/testutil/TestBucket.cs @@ -1,4 +1,4 @@ -// Copyright(c) 2017 Google Inc. +// Copyright(c) 2017 Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); you may not // use this file except in compliance with the License. You may obtain a copy of @@ -146,4 +146,4 @@ public void Dispose() _garbage.Clear(); } } -} \ No newline at end of file +} diff --git a/testutil/TestUtil.cs b/testutil/TestUtil.cs index 4b366d196eb..763ff37a2cf 100644 --- a/testutil/TestUtil.cs +++ b/testutil/TestUtil.cs @@ -1,4 +1,4 @@ -/* +/* * Copyright (c) 2016 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not @@ -38,8 +38,8 @@ public static string RandomName() while (nextChar < randomChars.Length) { rng.GetBytes(randomByte); - if (legalChars.Contains((char)randomByte[0])) - randomChars[nextChar++] = (char)randomByte[0]; + if (legalChars.Contains((char) randomByte[0])) + randomChars[nextChar++] = (char) randomByte[0]; } return new string(randomChars); } @@ -76,21 +76,21 @@ public T Eventually(Func func) catch (Exception e) when (ShouldCatch(e) && i < MaxTryCount) { int jitteredDelayMs; - lock(_lock) + lock (_lock) { - jitteredDelayMs = delayMs/2 + (int)(_random.NextDouble() * delayMs); + jitteredDelayMs = delayMs / 2 + (int) (_random.NextDouble() * delayMs); } Thread.Sleep(jitteredDelayMs); - delayMs *= (int)DelayMultiplier; + delayMs *= (int) DelayMultiplier; } } } public void Eventually(Action action) => Eventually(() => - { + { action(); - return 0; + return 0; }); @@ -108,10 +108,10 @@ public async Task Eventually(Func> asyncFunc) int jitteredDelayMs; lock (_lock) { - jitteredDelayMs = delayMs / 2 + (int)(_random.NextDouble() * delayMs); + jitteredDelayMs = delayMs / 2 + (int) (_random.NextDouble() * delayMs); } await Task.Delay(jitteredDelayMs); - delayMs *= (int)DelayMultiplier; + delayMs *= (int) DelayMultiplier; } } } @@ -119,9 +119,9 @@ public async Task Eventually(Func> asyncFunc) public async Task Eventually(Func action) { await Eventually(async () => - { + { await action(); - return 0; + return 0; }); } @@ -265,4 +265,4 @@ public override string ToString() } } -} \ No newline at end of file +} From b05a6bc42cf5f474ee2370cfac46a4c78260ce15 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 27 Mar 2025 03:58:21 -0700 Subject: [PATCH 38/49] Revert "Comments added in tests and sample description modified" This reverts commit f1b05b37942aa131ccff60fdd9084adfedd39260. --- storagetransfer/api/.editorconfig | 281 ------------------ .../CheckLatestTransferOperationTest.cs | 2 - .../CreateEventDrivenGcsTransferTest.cs | 1 - .../DownloadToPosixTest.cs | 2 - .../QuickstartTest.cs | 1 - .../TransferBetweenPosixTest.cs | 1 - .../TransferFromPosixTest.cs | 1 - .../TransferToNearlineTest.cs | 1 - .../TransferUsingManifestTest.cs | 2 - .../CheckLatestTransferOperationSample.cs | 8 +- .../CreateEventDrivenGcsTransferSample.cs | 10 +- .../DownloadToPosixSample.cs | 11 +- .../QuickstartSample.cs | 6 - .../TransferBetweenPosixSample.cs | 18 +- .../TransferFromPosixSample.cs | 10 +- .../TransferToNearlineSample.cs | 11 +- .../TransferUsingManifestSample.cs | 13 +- testutil/TestBucket.cs | 4 +- testutil/TestUtil.cs | 26 +- 19 files changed, 40 insertions(+), 369 deletions(-) delete mode 100644 storagetransfer/api/.editorconfig diff --git a/storagetransfer/api/.editorconfig b/storagetransfer/api/.editorconfig deleted file mode 100644 index 0ec4d4be40e..00000000000 --- a/storagetransfer/api/.editorconfig +++ /dev/null @@ -1,281 +0,0 @@ -# To learn more about .editorconfig see https://aka.ms/editorconfigdocs -# top-most EditorConfig file -root = true -############################### -# Core EditorConfig Options # -############################### -# All files -[*] -insert_final_newline = true -indent_style = space -indent_size = 4 -trim_trailing_whitespace = true - -# Generated code -[*{_AssemblyInfo.cs,.g.cs}] -generated_code = true - -# XML project files -[*.{csproj}] -indent_size = 2 - -# Code files - -############################### -# .NET Coding Conventions # -############################### -[*.{cs}] - -# Organize usings -dotnet_sort_system_directives_first = false - -# this. preferences -dotnet_style_qualification_for_field = false:suggestion -dotnet_style_qualification_for_property = false:suggestion -dotnet_style_qualification_for_method = false:suggestion -dotnet_style_qualification_for_event = false:suggestion - -# Language keywords vs BCL types preferences -dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion -dotnet_style_predefined_type_for_member_access = true:suggestion - -# Parentheses preferences -dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent - -# Modifier preferences -csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning -dotnet_style_require_accessibility_modifiers = for_non_interface_members:warning -dotnet_style_readonly_field = true:warning - -# Expression-level preferences -dotnet_style_object_initializer = true:suggestion -dotnet_style_collection_initializer = true:suggestion -dotnet_style_explicit_tuple_names = true:suggestion -dotnet_style_null_propagation = true:suggestion -dotnet_style_coalesce_expression = true:suggestion -dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion -dotnet_style_prefer_inferred_tuple_names = true:suggestion -dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion -dotnet_style_prefer_auto_properties = true:suggestion -dotnet_style_prefer_conditional_expression_over_assignment = true:silent -dotnet_style_prefer_conditional_expression_over_return = true:silent -csharp_prefer_simple_default_expression = true:suggestion - -############################### -# Naming Conventions # -############################### -# Style Definitions -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -# Use PascalCase for constant fields -dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields -dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style -dotnet_naming_symbols.constant_fields.applicable_kinds = field -dotnet_naming_symbols.constant_fields.applicable_accessibilities = * -dotnet_naming_symbols.constant_fields.required_modifiers = const -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -# static fields should have s_ prefix -dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion -dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields -dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style -dotnet_naming_symbols.static_fields.applicable_kinds = field -dotnet_naming_symbols.static_fields.required_modifiers = static -dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected -dotnet_naming_style.static_prefix_style.required_prefix = s_ -dotnet_naming_style.static_prefix_style.capitalization = camel_case - -# internal and private fields should be _camelCase -dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion -dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields -dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style -dotnet_naming_symbols.private_internal_fields.applicable_kinds = field -dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal -dotnet_naming_style.camel_case_underscore_style.required_prefix = _ -dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case - -# Locals and parameters are camelCase -dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion -dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters -dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style -dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local -dotnet_naming_style.camel_case_style.capitalization = camel_case - -# Public property and methods should be PascalCase -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.symbols = public_methods_and_properties -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.style = pascal_case -dotnet_naming_symbols.public_methods_and_properties.applicable_kinds = method, property -dotnet_naming_symbols.public_methods_and_properties.applicable_accessibilities = public -dotnet_naming_style.pascal_case.capitalization = pascal_case - -# Private and internal methods should be PascalCase -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.symbols = private_and_internal_methods -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.style = pascal_case -dotnet_naming_symbols.private_and_internal_methods.applicable_kinds = method -dotnet_naming_symbols.private_and_internal_methods.applicable_accessibilities = private, internal -############################### -# C# Coding Conventions # -############################### -[*.cs] -# var preferences -csharp_style_var_for_built_in_types = true:silent -csharp_style_var_when_type_is_apparent = true:silent -csharp_style_var_elsewhere = true:silent -dotnet_style_predefined_type_for_locals_parameters_members = true:silent -dotnet_style_predefined_type_for_member_access = true:silent - -csharp_using_directive_placement = outside_namespace:warning -csharp_prefer_braces = true:suggestion -csharp_preserve_single_line_blocks = true:none -csharp_preserve_single_line_statements = false:none -csharp_prefer_static_local_function = true:suggestion -csharp_prefer_simple_using_statement = true:suggestion -csharp_style_prefer_switch_expression = true:suggestion - -# Expression-bodied members -csharp_style_expression_bodied_methods = true:suggestion -csharp_style_expression_bodied_constructors = true:suggestion -csharp_style_expression_bodied_operators = true:suggestion -csharp_style_expression_bodied_properties = true:suggestion -csharp_style_expression_bodied_indexers = true:suggestion -csharp_style_expression_bodied_accessors = true:suggestion -csharp_style_expression_bodied_lambdas = true:suggestion -csharp_style_expression_bodied_local_functions = true:suggestion - -# Pattern matching preferences -csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion -csharp_style_pattern_matching_over_as_with_null_check = true:suggestion -csharp_style_inlined_variable_declaration = true:suggestion - -# Null-checking preferences -csharp_style_throw_expression = true:suggestion -csharp_style_conditional_delegate_call = true:suggestion - -# Other features -csharp_style_prefer_index_operator = true:suggestion -csharp_style_prefer_range_operator = true:suggestion -csharp_style_pattern_local_over_anonymous_function = false:none - -############################### -# C# Formatting Rules # -############################### -indent_size = 4 -insert_final_newline = true -charset = utf-8 - -# New line preferences -csharp_new_line_before_open_brace = all -csharp_new_line_before_else = true -csharp_new_line_before_catch = true -csharp_new_line_before_finally = true -csharp_new_line_before_members_in_object_initializers = true -csharp_new_line_before_members_in_anonymous_types = true -csharp_new_line_between_query_expression_clauses = true - -# Indentation preferences -csharp_indent_block_contents = true -csharp_indent_braces = false -csharp_indent_case_contents = true -csharp_indent_case_contents_when_block = true -csharp_indent_switch_labels = true -csharp_indent_labels = one_less_than_current - -# Space preferences -csharp_space_after_cast = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_after_comma = true -csharp_space_after_dot = false -csharp_space_after_keywords_in_control_flow_statements = true -csharp_space_after_semicolon_in_for_statement = true -csharp_space_between_method_call_parameter_list_parentheses = false -csharp_space_between_method_declaration_parameter_list_parentheses = false -csharp_space_between_parentheses = false -csharp_space_before_comma = false -csharp_space_before_dot = false -csharp_space_before_open_square_brackets = false -csharp_space_before_semicolon_in_for_statement = false -csharp_space_between_empty_square_brackets = false -csharp_space_around_declaration_statements = do_not_ignore -csharp_space_before_colon_in_inheritance_clause = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_around_binary_operators = before_and_after -csharp_space_between_method_declaration_empty_parameter_list_parentheses = false -csharp_space_between_method_call_name_and_opening_parenthesis = false -csharp_space_between_method_call_empty_parameter_list_parentheses = false -csharp_space_between_method_declaration_name_and_open_parenthesis = false -csharp_space_between_square_brackets = false - -# Wrapping preferences -csharp_preserve_single_line_statements = true -csharp_preserve_single_line_blocks = true - -# License header -file_header_template = Copyright 2024 Google LLC\n\nLicensed under the Apache License, Version 2.0 (the "License").\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at \n\nhttps://www.apache.org/licenses/LICENSE-2.0 \n\nUnless required by applicable law or agreed to in writing, software \ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and \nlimitations under the License. -csharp_style_namespace_declarations = file_scoped:suggestion -csharp_style_prefer_null_check_over_type_check = true:suggestion -csharp_style_prefer_local_over_anonymous_function = true:suggestion -csharp_style_prefer_tuple_swap = true:suggestion -csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion -csharp_style_unused_value_assignment_preference = discard_variable:suggestion -csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion -csharp_style_prefer_pattern_matching = true:suggestion -csharp_style_prefer_not_pattern = true:suggestion -csharp_style_prefer_extended_property_pattern = true:suggestion -csharp_style_prefer_parameter_null_checking = true:suggestion -csharp_style_prefer_method_group_conversion = true:suggestion - -# Change the severity to `error` for the rules that must not be violated, to fail the build. - -# IDE0005: Remove unnecessary using directives. -dotnet_diagnostic.IDE0005.severity = suggestion - -# IDE0011: Add braces -dotnet_diagnostic.IDE0011.severity = suggestion - -# IDE0017: Object initialization can be simplified. -dotnet_diagnostic.IDE0017.severity = suggestion - -# IDE0019: Use pattern matching. -dotnet_diagnostic.IDE0019.severity = suggestion - -# IDE0021: Use expression body for constructor. -dotnet_diagnostic.IDE0021.severity = suggestion - -# IDE0028: Collection initialization can be simplified. -dotnet_diagnostic.IDE0028.severity = suggestion - -# IDE0035: Remove unreachable code. -dotnet_diagnostic.IDE0035.severity = suggestion - -# IDE0040: Accessibility modifiers required. -dotnet_diagnostic.IDE0040.severity = suggestion - -# IDE0052: Remove unread private member -dotnet_diagnostic.IDE0052.severity = suggestion - -# IDE0053: Use expression body for lambda expression. -dotnet_diagnostic.IDE0053.severity = suggestion - -# IDE0055: All C# and .NET formatting rules. -dotnet_diagnostic.IDE0055.severity = suggestion - -# IDE0060: Remove unused parameter -dotnet_diagnostic.IDE0060.severity = suggestion - -# IDE0063: Using statement can be simplified -dotnet_diagnostic.IDE0063.severity = suggestion - -# IDE0065: Using statement can be simplified -dotnet_diagnostic.IDE0065.severity = suggestion - -# IDE0071: Interpolation can be simplified. -dotnet_diagnostic.IDE0071.severity = suggestion - -# IDE0075: Simplify conditional expression. -dotnet_diagnostic.IDE0075.severity = suggestion \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index f798dc5b882..f9107e38d98 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -35,7 +35,6 @@ public CheckLatestTransferOperationTest(StorageFixture fixture) _transferJobName = CreateTransferJob(); } - // Checks the latest transfer operation for a created transfer job. [Fact] public void CheckLatestTransferOperation() { @@ -45,7 +44,6 @@ public void CheckLatestTransferOperation() _jobName = transferJob.Name; } - // Creates one-time transfer job from a Google cloud storage bucket to another bucket. private string CreateTransferJob() { // Initialize request argument(s) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 54a85805ea3..cba7c2b99fd 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -76,7 +76,6 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) }); } - // Transfers object from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. [Fact] public void CreateEventDrivenGcsTransfer() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index bc3c3247fdf..ac9116f3abc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -39,7 +39,6 @@ public DownloadToPosixTest(StorageFixture fixture) UploadObjectToPosixBucket(_bucketNamePosixSource); } - // Downloads an object from a GCS bucket to a POSIX file system. [Fact] public void DownloadToPosix() { @@ -51,7 +50,6 @@ public void DownloadToPosix() _transferJobName = transferJob.Name; } - // Uploads an object to the POSIX bucket. private void UploadObjectToPosixBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 68f5868e5ae..9efbb3334a1 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -35,7 +35,6 @@ public QuickstartTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } - // Creates a one-time transfer job from a Google cloud storage bucket to another bucket [Fact] public void TestQuickstart() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 1b5d8045cfe..f267930f566 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -36,7 +36,6 @@ public TransferBetweenPosixTest(StorageFixture fixture) _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } - // Transfers objects from a POSIX file system to another POSIX file system. [Fact] public void TransferBetweenPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index b81eeb7689b..88f763d25fc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -33,7 +33,6 @@ public TransferFromPosixTest(StorageFixture fixture) _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } - // Transfers objects from a POSIX file system to a Cloud Storage bucket. [Fact] public void TransferFromPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index d0eaa9a6f00..eacce91d255 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -34,7 +34,6 @@ public TransferToNearlineTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } - // Transfers objects from a Google Cloud Storage bucket to another bucket with Nearline storage class. [Fact] public void TestTransferToNearline() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index dde52fc6548..58711707643 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -41,7 +41,6 @@ public TransferUsingManifestTest(StorageFixture fixture) UploadObjectToManifestBucket(_bucketNameManifestSource); } - // Transfers objects from a POSIX file system to a sink bucket using a manifest file. [Fact] public void TransferUsingManifest() { @@ -51,7 +50,6 @@ public void TransferUsingManifest() _transferJobName = transferJob.Name; } - // Uploads an object to the manifest bucket. private void UploadObjectToManifestBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 04df26cec05..f0f188b04e2 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -17,13 +17,11 @@ using Google.Cloud.StorageTransfer.V1; using System; +/// +/// Checks the latest transfer operation for a given transfer job. +/// public class CheckLatestTransferOperationSample { - /// - /// Sample that checks the latest transfer operation for a given transfer job. - /// - /// Your Google Cloud Project ID. - /// The name of the job to check. public TransferJob CheckLatestTransferOperation( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 3619408aca0..49465a395f3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -19,15 +19,11 @@ using Google.Cloud.StorageTransfer.V1; using System; +/// +/// Create an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. +/// public class CreateEventDrivenGcsTransferSample { - /// - /// Sample that creates an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. - /// - /// Your Google Cloud Project ID. - /// The GCS bucket to transfer data from. - /// The GCS bucket to transfer data to. - /// The subscription ID to a Pubsub queue to track. public TransferJob CreateEventDrivenGcsTransfer( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index e0d78bf742b..cd92b85e2c3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -17,16 +17,11 @@ using Google.Cloud.StorageTransfer.V1; using System; +/// +/// Create a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. +/// public class DownloadToPosixSample { - /// - /// Sample that creates a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. - /// - /// Your Google Cloud Project ID. - /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. - /// Your GCS source bucket name. - /// An optional path on the Google Cloud Storage bucket to download from. - /// The root directory path on the source filesystem. public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index a2a5bef3d84..fe08f6082c2 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -21,12 +21,6 @@ public class QuickstartSample { - /// - /// Sample that creates a one-time transfer job from a Google cloud storage bucket to another bucket. - /// - /// Your Google Cloud Project ID. - /// The GCS bucket to transfer data from. - /// The GCS bucket to transfer data to. public TransferJob Quickstart( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index ba00450564e..5d8abed0ce5 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -17,17 +17,11 @@ using Google.Cloud.StorageTransfer.V1; using System; +/// +/// Create a transfer to transfer objects from root directory to the destination directory between POSIX file systems. +/// public class TransferBetweenPosixSample { - /// - /// Sample that creates a transfer to transfer objects from root directory to the destination directory between POSIX file systems. - /// - /// The ID of the project. - /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. - /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. - /// The root directory path on the source filesystem. - /// The root directory path on the sink filesystem. - /// The name of GCS bucket for intermediate storage. public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -40,7 +34,7 @@ public TransferJob TransferBetweenPosix( // The root directory path on the sink filesystem string destinationDirectory = "/directory/to/transfer/sink", // The name of GCS bucket for intermediate storage - string intermediateBucket = "my-intermediate-bucket") + string intermediate_bucket = "my-intermediate-bucket") { // A useful description for your transfer job string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; @@ -55,7 +49,7 @@ public TransferJob TransferBetweenPosix( SinkAgentPoolName = sinkAgentPoolName, PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, PosixDataSink = new PosixFilesystem { RootDirectory = destinationDirectory }, - GcsIntermediateDataLocation = new GcsData { BucketName = intermediateBucket } + GcsIntermediateDataLocation = new GcsData { BucketName = intermediate_bucket } }, Status = TransferJob.Types.Status.Enabled, }; @@ -76,4 +70,4 @@ public TransferJob TransferBetweenPosix( return response; } } -// [END storagetransfer_transfer_posix_to_posix] +//[END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 1307acff0b3..bc82f8f74dc 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -17,15 +17,11 @@ using Google.Cloud.StorageTransfer.V1; using System; +/// +/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket. +/// public class TransferFromPosixSample { - /// - /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket. - /// - /// The ID of the project. - /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. - /// The root directory path on the source filesystem. - /// The GCS bucket to transfer data to. public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index deaeca3fe3e..98dcfb0406d 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -18,15 +18,12 @@ using Google.Protobuf.WellKnownTypes; using System; +/// +/// Creates a one-off transfer job that transfers objects from a standard gcs bucket that are more +/// than 30 days old to a nearline gcs bucket. +/// public class TransferToNearlineSample { - /// - /// Sample that creates a one-off transfer job that transfers objects from a standard gcs bucket that are more - /// than 30 days old to a nearline gcs bucket. - /// - /// Your Google Cloud Project ID. - /// The GCS bucket to transfer objects from. - /// The GCS Nearline bucket to transfer old objects to. public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 8154a36605c..c92860b2174 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -17,18 +17,11 @@ using Google.Cloud.StorageTransfer.V1; using System; +/// +/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. +/// public class TransferUsingManifestSample { - /// - /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. - /// - /// The ID of the project. - /// The agent pool associated with the POSIX data source. If not provided , - /// defaults to the default agent - /// The root directory path on the source filesystem. - /// The GCS bucket which has your manifest file. - /// The GCS bucket to transfer data to. - /// The name of the manifest file in manifestBucket that specifies which objects to transfer. public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/testutil/TestBucket.cs b/testutil/TestBucket.cs index f07f6406574..c5afc307d6d 100644 --- a/testutil/TestBucket.cs +++ b/testutil/TestBucket.cs @@ -1,4 +1,4 @@ -// Copyright(c) 2017 Google Inc. +// Copyright(c) 2017 Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); you may not // use this file except in compliance with the License. You may obtain a copy of @@ -146,4 +146,4 @@ public void Dispose() _garbage.Clear(); } } -} +} \ No newline at end of file diff --git a/testutil/TestUtil.cs b/testutil/TestUtil.cs index 763ff37a2cf..4b366d196eb 100644 --- a/testutil/TestUtil.cs +++ b/testutil/TestUtil.cs @@ -1,4 +1,4 @@ -/* +/* * Copyright (c) 2016 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not @@ -38,8 +38,8 @@ public static string RandomName() while (nextChar < randomChars.Length) { rng.GetBytes(randomByte); - if (legalChars.Contains((char) randomByte[0])) - randomChars[nextChar++] = (char) randomByte[0]; + if (legalChars.Contains((char)randomByte[0])) + randomChars[nextChar++] = (char)randomByte[0]; } return new string(randomChars); } @@ -76,21 +76,21 @@ public T Eventually(Func func) catch (Exception e) when (ShouldCatch(e) && i < MaxTryCount) { int jitteredDelayMs; - lock (_lock) + lock(_lock) { - jitteredDelayMs = delayMs / 2 + (int) (_random.NextDouble() * delayMs); + jitteredDelayMs = delayMs/2 + (int)(_random.NextDouble() * delayMs); } Thread.Sleep(jitteredDelayMs); - delayMs *= (int) DelayMultiplier; + delayMs *= (int)DelayMultiplier; } } } public void Eventually(Action action) => Eventually(() => - { + { action(); - return 0; + return 0; }); @@ -108,10 +108,10 @@ public async Task Eventually(Func> asyncFunc) int jitteredDelayMs; lock (_lock) { - jitteredDelayMs = delayMs / 2 + (int) (_random.NextDouble() * delayMs); + jitteredDelayMs = delayMs / 2 + (int)(_random.NextDouble() * delayMs); } await Task.Delay(jitteredDelayMs); - delayMs *= (int) DelayMultiplier; + delayMs *= (int)DelayMultiplier; } } } @@ -119,9 +119,9 @@ public async Task Eventually(Func> asyncFunc) public async Task Eventually(Func action) { await Eventually(async () => - { + { await action(); - return 0; + return 0; }); } @@ -265,4 +265,4 @@ public override string ToString() } } -} +} \ No newline at end of file From 526ddade33e7445a3a2079dd00e52564a834003d Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 27 Mar 2025 04:39:48 -0700 Subject: [PATCH 39/49] Comments are added and sample description modified --- .../CheckLatestTransferOperationTest.cs | 2 ++ .../CreateEventDrivenGcsTransferTest.cs | 1 + .../DownloadToPosixTest.cs | 2 ++ .../TransferBetweenPosixTest.cs | 1 + .../TransferFromPosixTest.cs | 1 + .../TransferToNearlineTest.cs | 1 + .../TransferUsingManifestTest.cs | 2 ++ .../CheckLatestTransferOperationSample.cs | 8 +++++--- .../CreateEventDrivenGcsTransferSample.cs | 10 +++++++--- .../DownloadToPosixSample.cs | 11 ++++++++--- .../TransferBetweenPosixSample.cs | 18 ++++++++++++------ .../TransferFromPosixSample.cs | 10 +++++++--- .../TransferToNearlineSample.cs | 11 +++++++---- .../TransferUsingManifestSample.cs | 13 ++++++++++--- 14 files changed, 66 insertions(+), 25 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index f9107e38d98..f798dc5b882 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -35,6 +35,7 @@ public CheckLatestTransferOperationTest(StorageFixture fixture) _transferJobName = CreateTransferJob(); } + // Checks the latest transfer operation for a created transfer job. [Fact] public void CheckLatestTransferOperation() { @@ -44,6 +45,7 @@ public void CheckLatestTransferOperation() _jobName = transferJob.Name; } + // Creates one-time transfer job from a Google cloud storage bucket to another bucket. private string CreateTransferJob() { // Initialize request argument(s) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index cba7c2b99fd..54a85805ea3 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -76,6 +76,7 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) }); } + // Transfers object from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. [Fact] public void CreateEventDrivenGcsTransfer() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index ac9116f3abc..bc3c3247fdf 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -39,6 +39,7 @@ public DownloadToPosixTest(StorageFixture fixture) UploadObjectToPosixBucket(_bucketNamePosixSource); } + // Downloads an object from a GCS bucket to a POSIX file system. [Fact] public void DownloadToPosix() { @@ -50,6 +51,7 @@ public void DownloadToPosix() _transferJobName = transferJob.Name; } + // Uploads an object to the POSIX bucket. private void UploadObjectToPosixBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index f267930f566..1b5d8045cfe 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -36,6 +36,7 @@ public TransferBetweenPosixTest(StorageFixture fixture) _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } + // Transfers objects from a POSIX file system to another POSIX file system. [Fact] public void TransferBetweenPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 88f763d25fc..b81eeb7689b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -33,6 +33,7 @@ public TransferFromPosixTest(StorageFixture fixture) _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } + // Transfers objects from a POSIX file system to a Cloud Storage bucket. [Fact] public void TransferFromPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index eacce91d255..d0eaa9a6f00 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -34,6 +34,7 @@ public TransferToNearlineTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } + // Transfers objects from a Google Cloud Storage bucket to another bucket with Nearline storage class. [Fact] public void TestTransferToNearline() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 58711707643..dde52fc6548 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -41,6 +41,7 @@ public TransferUsingManifestTest(StorageFixture fixture) UploadObjectToManifestBucket(_bucketNameManifestSource); } + // Transfers objects from a POSIX file system to a sink bucket using a manifest file. [Fact] public void TransferUsingManifest() { @@ -50,6 +51,7 @@ public void TransferUsingManifest() _transferJobName = transferJob.Name; } + // Uploads an object to the manifest bucket. private void UploadObjectToManifestBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index f0f188b04e2..04df26cec05 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -17,11 +17,13 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Checks the latest transfer operation for a given transfer job. -/// public class CheckLatestTransferOperationSample { + /// + /// Sample that checks the latest transfer operation for a given transfer job. + /// + /// Your Google Cloud Project ID. + /// The name of the job to check. public TransferJob CheckLatestTransferOperation( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 49465a395f3..3619408aca0 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -19,11 +19,15 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. -/// public class CreateEventDrivenGcsTransferSample { + /// + /// Sample that creates an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. + /// + /// Your Google Cloud Project ID. + /// The GCS bucket to transfer data from. + /// The GCS bucket to transfer data to. + /// The subscription ID to a Pubsub queue to track. public TransferJob CreateEventDrivenGcsTransfer( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index cd92b85e2c3..e0d78bf742b 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -17,11 +17,16 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. -/// public class DownloadToPosixSample { + /// + /// Sample that creates a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. + /// + /// Your Google Cloud Project ID. + /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. + /// Your GCS source bucket name. + /// An optional path on the Google Cloud Storage bucket to download from. + /// The root directory path on the source filesystem. public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 5d8abed0ce5..ba00450564e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -17,11 +17,17 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer to transfer objects from root directory to the destination directory between POSIX file systems. -/// public class TransferBetweenPosixSample { + /// + /// Sample that creates a transfer to transfer objects from root directory to the destination directory between POSIX file systems. + /// + /// The ID of the project. + /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. + /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. + /// The root directory path on the source filesystem. + /// The root directory path on the sink filesystem. + /// The name of GCS bucket for intermediate storage. public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -34,7 +40,7 @@ public TransferJob TransferBetweenPosix( // The root directory path on the sink filesystem string destinationDirectory = "/directory/to/transfer/sink", // The name of GCS bucket for intermediate storage - string intermediate_bucket = "my-intermediate-bucket") + string intermediateBucket = "my-intermediate-bucket") { // A useful description for your transfer job string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; @@ -49,7 +55,7 @@ public TransferJob TransferBetweenPosix( SinkAgentPoolName = sinkAgentPoolName, PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, PosixDataSink = new PosixFilesystem { RootDirectory = destinationDirectory }, - GcsIntermediateDataLocation = new GcsData { BucketName = intermediate_bucket } + GcsIntermediateDataLocation = new GcsData { BucketName = intermediateBucket } }, Status = TransferJob.Types.Status.Enabled, }; @@ -70,4 +76,4 @@ public TransferJob TransferBetweenPosix( return response; } } -//[END storagetransfer_transfer_posix_to_posix] +// [END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index bc82f8f74dc..1307acff0b3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -17,11 +17,15 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket. -/// public class TransferFromPosixSample { + /// + /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket. + /// + /// The ID of the project. + /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. + /// The root directory path on the source filesystem. + /// The GCS bucket to transfer data to. public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 98dcfb0406d..deaeca3fe3e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -18,12 +18,15 @@ using Google.Protobuf.WellKnownTypes; using System; -/// -/// Creates a one-off transfer job that transfers objects from a standard gcs bucket that are more -/// than 30 days old to a nearline gcs bucket. -/// public class TransferToNearlineSample { + /// + /// Sample that creates a one-off transfer job that transfers objects from a standard gcs bucket that are more + /// than 30 days old to a nearline gcs bucket. + /// + /// Your Google Cloud Project ID. + /// The GCS bucket to transfer objects from. + /// The GCS Nearline bucket to transfer old objects to. public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index c92860b2174..8154a36605c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -17,11 +17,18 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. -/// public class TransferUsingManifestSample { + /// + /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. + /// + /// The ID of the project. + /// The agent pool associated with the POSIX data source. If not provided , + /// defaults to the default agent + /// The root directory path on the source filesystem. + /// The GCS bucket which has your manifest file. + /// The GCS bucket to transfer data to. + /// The name of the manifest file in manifestBucket that specifies which objects to transfer. public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", From c6950d8b00ece45343e86e257b58c3131148d557 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 27 Mar 2025 22:18:18 -0700 Subject: [PATCH 40/49] Quickstart test and sample changes --- .../api/StorageTransfer.Samples.Tests/QuickstartTest.cs | 1 + .../api/StorageTransfer.Samples/QuickstartSample.cs | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 9efbb3334a1..3c6fb011cf7 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -35,6 +35,7 @@ public QuickstartTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } + // Creates a one-time transfer job from a Google cloud storage bucket to another bucket. [Fact] public void TestQuickstart() { diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index fe08f6082c2..a2a5bef3d84 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -21,6 +21,12 @@ public class QuickstartSample { + /// + /// Sample that creates a one-time transfer job from a Google cloud storage bucket to another bucket. + /// + /// Your Google Cloud Project ID. + /// The GCS bucket to transfer data from. + /// The GCS bucket to transfer data to. public TransferJob Quickstart( // Your Google Cloud Project ID string projectId = "my-project-id", From d043b9e8358030305693f6f101601822ba180b20 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Fri, 28 Mar 2025 00:22:23 -0700 Subject: [PATCH 41/49] minor changes --- .../api/StorageTransfer.Samples/DownloadToPosixSample.cs | 2 +- .../api/StorageTransfer.Samples/TransferBetweenPosixSample.cs | 2 +- .../StorageTransfer.Samples/TransferUsingManifestSample.cs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index e0d78bf742b..31e1937309f 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -71,4 +71,4 @@ public TransferJob DownloadToPosix( return response; } } -//[END storagetransfer_download_to_posix] +// [END storagetransfer_download_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index ba00450564e..78973c0d919 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -20,7 +20,7 @@ public class TransferBetweenPosixSample { /// - /// Sample that creates a transfer to transfer objects from root directory to the destination directory between POSIX file systems. + /// Sample that creates a transfer that transfer objects from root directory to the destination directory between POSIX file systems. /// /// The ID of the project. /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 8154a36605c..cf75fb9a2e4 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -20,10 +20,10 @@ public class TransferUsingManifestSample { /// - /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. + /// Sample that creates a transfer job that transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. /// /// The ID of the project. - /// The agent pool associated with the POSIX data source. If not provided , + /// The agent pool associated with the POSIX data source. If not provided, /// defaults to the default agent /// The root directory path on the source filesystem. /// The GCS bucket which has your manifest file. From 869274aae549912b2667a51eb4fee47aef0d4a3a Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 14 Apr 2025 03:39:05 -0700 Subject: [PATCH 42/49] Sample and Test changes --- .../CheckLatestTransferOperationTest.cs | 5 +-- .../CreateEventDrivenGcsTransferTest.cs | 36 ++++++++--------- .../DownloadToPosixTest.cs | 2 - .../QuickstartTest.cs | 1 - .../StorageFixture.cs | 40 ++++++++++++++----- .../TransferBetweenPosixTest.cs | 1 - .../TransferFromPosixTest.cs | 1 - .../TransferToNearlineTest.cs | 3 +- .../TransferUsingManifestTest.cs | 2 - .../CheckLatestTransferOperationSample.cs | 11 ++--- .../CreateEventDrivenGcsTransferSample.cs | 11 ++--- .../DownloadToPosixSample.cs | 14 +------ .../QuickstartSample.cs | 7 +--- .../TransferBetweenPosixSample.cs | 15 +------ .../TransferFromPosixSample.cs | 13 +----- .../TransferToNearlineSample.cs | 11 +---- .../TransferUsingManifestSample.cs | 15 +------ 17 files changed, 66 insertions(+), 122 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index f798dc5b882..b5f3e934a31 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -35,7 +35,6 @@ public CheckLatestTransferOperationTest(StorageFixture fixture) _transferJobName = CreateTransferJob(); } - // Checks the latest transfer operation for a created transfer job. [Fact] public void CheckLatestTransferOperation() { @@ -45,10 +44,8 @@ public void CheckLatestTransferOperation() _jobName = transferJob.Name; } - // Creates one-time transfer job from a Google cloud storage bucket to another bucket. private string CreateTransferJob() { - // Initialize request argument(s) TransferJob transferJob = new TransferJob { ProjectId = _fixture.ProjectId, @@ -63,7 +60,7 @@ private string CreateTransferJob() { TransferJob = transferJob }; - // Make the request + TransferJob response = _fixture.Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); return response.Name; } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 54a85805ea3..c8687af21d7 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -27,8 +27,8 @@ public class CreateEventDrivenGcsTransferTest : IDisposable private readonly StorageFixture _fixture; private readonly string _pubSubId; private string _transferJobName; - private string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); - private string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); + private string TopicId { get; } = Guid.NewGuid().ToString(); + private string SubscriptionId { get; } = Guid.NewGuid().ToString(); private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); private PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); @@ -40,25 +40,33 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); _pubSubId = $"projects/{_fixture.ProjectId}/subscriptions/{SubscriptionId}"; + CreatePubSubResourcesAndGrantStsPermissions(); + } + + [Fact] + public void CreateEventDrivenGcsTransfer() + { + CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); + var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _pubSubId); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + private void CreatePubSubResourcesAndGrantStsPermissions() + { string email = _fixture.Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest() { ProjectId = _fixture.ProjectId }).AccountEmail; string memberServiceAccount = "serviceAccount:" + email; - // Create subscription name SubscriptionName subscriptionName = new SubscriptionName(_fixture.ProjectId, SubscriptionId); - // Create topic name TopicName topicName = new TopicName(_fixture.ProjectId, TopicId); - // Create topic PublisherClient.CreateTopic(topicName); - // Create subscription. SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500); var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy(); - policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount); - PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest { ResourceAsResourceName = topicName, @@ -66,9 +74,7 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) }); var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy(); - policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount); - PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest { ResourceAsResourceName = subscriptionName, @@ -76,16 +82,6 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) }); } - // Transfers object from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. - [Fact] - public void CreateEventDrivenGcsTransfer() - { - CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); - var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _pubSubId); - Assert.Contains("transferJobs/", transferJob.Name); - _transferJobName = transferJob.Name; - } - public void Dispose() { try diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index bc3c3247fdf..ac9116f3abc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -39,7 +39,6 @@ public DownloadToPosixTest(StorageFixture fixture) UploadObjectToPosixBucket(_bucketNamePosixSource); } - // Downloads an object from a GCS bucket to a POSIX file system. [Fact] public void DownloadToPosix() { @@ -51,7 +50,6 @@ public void DownloadToPosix() _transferJobName = transferJob.Name; } - // Uploads an object to the POSIX bucket. private void UploadObjectToPosixBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 3c6fb011cf7..9efbb3334a1 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -35,7 +35,6 @@ public QuickstartTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } - // Creates a one-time transfer job from a Google cloud storage bucket to another bucket. [Fact] public void TestQuickstart() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index fbd53117768..df85646e2eb 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -14,12 +14,15 @@ * limitations under the License. */ +using Google; using Google.Apis.Storage.v1.Data; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using System; using System.Collections.Generic; using System.IO; +using System.Threading; +using System.Xml.Linq; using Xunit; namespace StorageTransfer.Samples.Tests @@ -32,11 +35,14 @@ public class StorageFixture : IDisposable, ICollectionFixture public string BucketNameSink { get; set; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } + public StorageClient Client { get; } public StorageClient Storage { get; } = StorageClient.Create(); public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); + private readonly List _bucketsToDelete = []; public StorageFixture() { + Client = StorageClient.Create(); ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); SourceAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; SinkAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; @@ -46,7 +52,7 @@ public StorageFixture() } } - internal void CreateBucketAndGrantStsPermissions(string bucketName) + internal void CreateBucketAndGrantStsPermissions(string bucketName, bool registerForDeletion = true) { var bucket = Storage.CreateBucket(ProjectId, new Bucket { @@ -88,30 +94,44 @@ internal void CreateBucketAndGrantStsPermissions(string bucketName) policy.Bindings.Add(bucketReaderBinding); policy.Bindings.Add(bucketWriterBinding); Storage.SetBucketIamPolicy(bucketName, policy); + SleepAfterBucketCreateDelete(); + if (registerForDeletion) + { + RegisterBucketToDelete(bucketName); + } } internal string GenerateBucketName() => Guid.NewGuid().ToString(); internal string GetCurrentUserTempFolderPath() => System.IO.Path.GetTempPath(); internal string GenerateTempFolderPath() => Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + /// + /// Bucket creation/deletion is rate-limited. To avoid making the tests flaky, we sleep after each operation. + /// + internal static void SleepAfterBucketCreateDelete() => Thread.Sleep(2000); + + internal void RegisterBucketToDelete(string bucket) => _bucketsToDelete.Add(bucket); + public void Dispose() { - try + foreach (var bucket in _bucketsToDelete) { - Storage.DeleteBucket(BucketNameSink, new DeleteBucketOptions { DeleteObjects = true }); - } - catch (Exception) - { - // Do nothing, we delete on a best effort basis. + DeleteBucket(Client, bucket, null); } + } + + private void DeleteBucket(StorageClient client, string bucket, string userProject) + { try { - Storage.DeleteBucket(BucketNameSource, new DeleteBucketOptions { DeleteObjects = true }); + client.DeleteBucket(bucket, new DeleteBucketOptions { UserProject = userProject, DeleteObjects = true }); } - catch (Exception) + catch (GoogleApiException) { - // Do nothing, we delete on a best effort basis. + // Some tests fail to delete buckets due to object retention locks etc. + // They can be cleaned up later. } + SleepAfterBucketCreateDelete(); } } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 1b5d8045cfe..f267930f566 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -36,7 +36,6 @@ public TransferBetweenPosixTest(StorageFixture fixture) _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } - // Transfers objects from a POSIX file system to another POSIX file system. [Fact] public void TransferBetweenPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index b81eeb7689b..88f763d25fc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -33,7 +33,6 @@ public TransferFromPosixTest(StorageFixture fixture) _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } - // Transfers objects from a POSIX file system to a Cloud Storage bucket. [Fact] public void TransferFromPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index d0eaa9a6f00..b239ad868eb 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -34,9 +34,8 @@ public TransferToNearlineTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } - // Transfers objects from a Google Cloud Storage bucket to another bucket with Nearline storage class. [Fact] - public void TestTransferToNearline() + public void TransferToNearline() { TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(); var storage = _fixture.Storage; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index dde52fc6548..58711707643 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -41,7 +41,6 @@ public TransferUsingManifestTest(StorageFixture fixture) UploadObjectToManifestBucket(_bucketNameManifestSource); } - // Transfers objects from a POSIX file system to a sink bucket using a manifest file. [Fact] public void TransferUsingManifest() { @@ -51,7 +50,6 @@ public void TransferUsingManifest() _transferJobName = transferJob.Name; } - // Uploads an object to the manifest bucket. private void UploadObjectToManifestBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 04df26cec05..da9a1bdeaaa 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -20,23 +20,18 @@ public class CheckLatestTransferOperationSample { /// - /// Sample that checks the latest transfer operation for a given transfer job. + /// Checks the latest transfer operation for a given transfer job. /// - /// Your Google Cloud Project ID. - /// The name of the job to check. + /// The ID of the Google Cloud project. + /// The name of the transfer job. public TransferJob CheckLatestTransferOperation( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The name of the job to check string jobName = "transferJobs/1234567890") { - // Create a Transfer Service client StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.Create(); GetTransferJobRequest getTransferJobRequest = new GetTransferJobRequest { ProjectId = projectId, JobName = jobName }; - // Get Transfer job TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); - // Get Latest operation name from transfer job string latestOperationName = transferJob.LatestOperationName; if (!string.IsNullOrEmpty(latestOperationName)) diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 3619408aca0..34c57b3afbf 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -22,23 +22,18 @@ public class CreateEventDrivenGcsTransferSample { /// - /// Sample that creates an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. + /// Creates an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. /// - /// Your Google Cloud Project ID. + /// The ID of the Google Cloud project. /// The GCS bucket to transfer data from. /// The GCS bucket to transfer data to. - /// The subscription ID to a Pubsub queue to track. + /// The subscription ID to a PubSub queue to track. public TransferJob CreateEventDrivenGcsTransfer( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The GCS bucket to transfer data from string sourceBucket = "my-source-bucket", - // The GCS bucket to transfer data to string sinkBucket = "my-sink-bucket", - // The subscription ID to a Pubsub queue to track string pubSubId = "projects/PROJECT_NAME/subscriptions/SUBSCRIPTION_ID") { - // A useful description for your transfer job string jobDescription = $"Event driven gcs data transfer from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} "; TransferJob transferJob = new TransferJob diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 31e1937309f..5b554a759e2 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -20,26 +20,20 @@ public class DownloadToPosixSample { /// - /// Sample that creates a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. + /// Creates a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. /// - /// Your Google Cloud Project ID. + /// The ID of the Google Cloud project. /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. /// Your GCS source bucket name. /// An optional path on the Google Cloud Storage bucket to download from. /// The root directory path on the source filesystem. public TransferJob DownloadToPosix( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // Your GCS source bucket name string gcsSourceBucket = "my-gcs-source-bucket", - // An optional path on the Google Cloud Storage bucket to download from string gcsSourcePath = "foo/bar/", - // The root directory path on the source filesystem string rootDirectory = "/tmp/uploads") { - // A useful description for your transfer job string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; TransferJob transferJob = new TransferJob @@ -55,12 +49,8 @@ public TransferJob DownloadToPosix( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - - // Create a Transfer job TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest { JobName = response.Name, diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index a2a5bef3d84..276d6177c10 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -22,17 +22,14 @@ public class QuickstartSample { /// - /// Sample that creates a one-time transfer job from a Google cloud storage bucket to another bucket. + /// Creates an one-time transfer job from a Google Cloud storage bucket to another bucket. /// - /// Your Google Cloud Project ID. + /// The ID of the Google Cloud project. /// The GCS bucket to transfer data from. /// The GCS bucket to transfer data to. public TransferJob Quickstart( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The GCS bucket to transfer data from string sourceBucket = "my-source-bucket", - // The GCS bucket to transfer data to string sinkBucket = "my-sink-bucket") { TransferJob transferJob = new TransferJob diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 78973c0d919..b1410a2a92f 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -20,29 +20,22 @@ public class TransferBetweenPosixSample { /// - /// Sample that creates a transfer that transfer objects from root directory to the destination directory between POSIX file systems. + /// Creates a transfer that transfer objects from root directory to the destination directory between POSIX file systems. /// - /// The ID of the project. + /// The ID of the Google Cloud project. /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. /// The root directory path on the source filesystem. /// The root directory path on the sink filesystem. /// The name of GCS bucket for intermediate storage. public TransferJob TransferBetweenPosix( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // The root directory path on the source filesystem string rootDirectory = "/tmp/uploads", - // The root directory path on the sink filesystem string destinationDirectory = "/directory/to/transfer/sink", - // The name of GCS bucket for intermediate storage string intermediateBucket = "my-intermediate-bucket") { - // A useful description for your transfer job string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; TransferJob transferJob = new TransferJob @@ -60,12 +53,8 @@ public TransferJob TransferBetweenPosix( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - - // Create a Transfer job TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest { JobName = response.Name, diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 1307acff0b3..8d03df85a69 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -20,23 +20,18 @@ public class TransferFromPosixSample { /// - /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket. + /// Creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket. /// - /// The ID of the project. + /// The ID of the Google Cloud project. /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. /// The root directory path on the source filesystem. /// The GCS bucket to transfer data to. public TransferJob TransferFromPosix( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // The root directory path on the source filesystem string rootDirectory = "/tmp/uploads", - // The GCS bucket to transfer data to string sinkBucket = "my-sink-bucket") { - // A useful description for your transfer job string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket})"; TransferJob transferJob = new TransferJob @@ -52,12 +47,8 @@ public TransferJob TransferFromPosix( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - - // Create a Transfer job TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest { JobName = response.Name, diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index deaeca3fe3e..0e32ed5f087 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -21,21 +21,17 @@ public class TransferToNearlineSample { /// - /// Sample that creates a one-off transfer job that transfers objects from a standard gcs bucket that are more + /// Creates an one-off transfer job that transfers objects from a standard gcs bucket that are more /// than 30 days old to a nearline gcs bucket. /// - /// Your Google Cloud Project ID. + /// The ID of the Google Cloud project. /// The GCS bucket to transfer objects from. /// The GCS Nearline bucket to transfer old objects to. public TransferJob TransferToNearline( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The GCS bucket to transfer objects from string sourceBucket = "my-source-bucket", - // The GCS Nearline bucket to transfer old objects to string sinkBucket = "my-sink-bucket") { - // A description of this job string jobDescription = $"Transfers old objects from standard bucket ({sourceBucket}) that haven't been modified in the last 30 days to a Nearline bucket ({sinkBucket})"; TransferJob transferJob = new TransferJob @@ -53,10 +49,7 @@ public TransferJob TransferToNearline( Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)), ScheduleEndDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - - // Create a Transfer job TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); client.RunTransferJob(new RunTransferJobRequest { diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index cf75fb9a2e4..f21fb6d6faa 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -20,9 +20,9 @@ public class TransferUsingManifestSample { /// - /// Sample that creates a transfer job that transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. + /// Creates a transfer job that transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. /// - /// The ID of the project. + /// The ID of the Google Cloud project. /// The agent pool associated with the POSIX data source. If not provided, /// defaults to the default agent /// The root directory path on the source filesystem. @@ -30,22 +30,15 @@ public class TransferUsingManifestSample /// The GCS bucket to transfer data to. /// The name of the manifest file in manifestBucket that specifies which objects to transfer. public TransferJob TransferUsingManifest( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // The root directory path on the source filesystem string rootDirectory = "/tmp/uploads", - // The GCS bucket which has your manifest file string manifestBucket = "my-source-bucket", - // The GCS bucket to transfer data to string sinkBucket = "my-sink-bucket", - // The name of the manifest file in manifestBucket that specifies which objects to transfer string manifestObjectName = "path/to/manifest.csv") { string manifestLocation = $"gs://{manifestBucket}/{manifestObjectName}"; - // A useful description for your transfer job string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file"; TransferJob transferJob = new TransferJob @@ -63,12 +56,8 @@ public TransferJob TransferUsingManifest( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - - // Create a Transfer job TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest { JobName = response.Name, From dece92adce3a1f2f41196adc3b5da12294ad0aac Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 16 Apr 2025 02:07:48 -0700 Subject: [PATCH 43/49] minor changes --- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 4 +--- .../TransferBetweenPosixTest.cs | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index df85646e2eb..83897fd8628 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -35,14 +35,12 @@ public class StorageFixture : IDisposable, ICollectionFixture public string BucketNameSink { get; set; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } - public StorageClient Client { get; } public StorageClient Storage { get; } = StorageClient.Create(); public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); private readonly List _bucketsToDelete = []; public StorageFixture() { - Client = StorageClient.Create(); ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); SourceAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; SinkAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; @@ -116,7 +114,7 @@ public void Dispose() { foreach (var bucket in _bucketsToDelete) { - DeleteBucket(Client, bucket, null); + DeleteBucket(Storage, bucket, null); } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index f267930f566..43489ba8223 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -42,15 +42,14 @@ public void TransferBetweenPosix() TransferBetweenPosixSample transferBetweenPosixSample = new TransferBetweenPosixSample(); Directory.CreateDirectory(_tempDirectory); Directory.CreateDirectory(_tempDestinationDirectory); - string fileName = Path.Combine(_tempDirectory, "test.txt"); - // Check if file already exists. If yes, delete it. + string fileName = Path.Combine(_tempDirectory, $"{Guid.NewGuid().ToString()}.txt"); if (File.Exists(fileName)) { File.Delete(fileName); } using (StreamWriter sw = new StreamWriter(fileName)) { - sw.WriteLine("test message"); + sw.WriteLine(Guid.NewGuid().ToString()); } var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _tempDirectory, _tempDestinationDirectory, _fixture.BucketNameSource); Assert.Contains("transferJobs/", transferJob.Name); From 40d28d14c90ce219b4efb2f38934314836277da8 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 16 Apr 2025 03:15:49 -0700 Subject: [PATCH 44/49] individual bucket creation and deletion test changes --- .../CheckLatestTransferOperationTest.cs | 16 ++++++++++------ .../CreateEventDrivenGcsTransferTest.cs | 14 +++++++++----- .../QuickstartTest.cs | 14 +++++++++----- .../StorageFixture.cs | 2 -- .../TransferBetweenPosixTest.cs | 8 +++++--- .../TransferFromPosixTest.cs | 8 +++++--- .../TransferToNearlineTest.cs | 16 ++++++++++------ .../TransferUsingManifestTest.cs | 18 ++++++++++-------- .../DownloadToPosixSample.cs | 10 +++++----- 9 files changed, 63 insertions(+), 43 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index b5f3e934a31..adaf2f838da 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -24,14 +24,16 @@ public class CheckLatestTransferOperationTest : IDisposable private readonly StorageFixture _fixture; private string _jobName; private readonly string _transferJobName; + private readonly string _sourceBucket; + private readonly string _sinkBucket; public CheckLatestTransferOperationTest(StorageFixture fixture) { _fixture = fixture; - _fixture.BucketNameSource = _fixture.GenerateBucketName(); - _fixture.BucketNameSink = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _sourceBucket = _fixture.GenerateBucketName(); + _sinkBucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_sourceBucket); + _fixture.CreateBucketAndGrantStsPermissions(_sinkBucket); _transferJobName = CreateTransferJob(); } @@ -51,8 +53,8 @@ private string CreateTransferJob() ProjectId = _fixture.ProjectId, TransferSpec = new TransferSpec { - GcsDataSink = new GcsData { BucketName = _fixture.BucketNameSource }, - GcsDataSource = new GcsData { BucketName = _fixture.BucketNameSink } + GcsDataSink = new GcsData { BucketName = _sinkBucket }, + GcsDataSource = new GcsData { BucketName = _sourceBucket } }, Status = TransferJob.Types.Status.Enabled }; @@ -79,6 +81,8 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + _fixture.Storage.DeleteBucket(_sourceBucket); + _fixture.Storage.DeleteBucket(_sinkBucket); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index c8687af21d7..8e65a07b7fa 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -27,6 +27,8 @@ public class CreateEventDrivenGcsTransferTest : IDisposable private readonly StorageFixture _fixture; private readonly string _pubSubId; private string _transferJobName; + private readonly string _sourceBucket; + private readonly string _sinkBucket; private string TopicId { get; } = Guid.NewGuid().ToString(); private string SubscriptionId { get; } = Guid.NewGuid().ToString(); private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); @@ -35,10 +37,10 @@ public class CreateEventDrivenGcsTransferTest : IDisposable public CreateEventDrivenGcsTransferTest(StorageFixture fixture) { _fixture = fixture; - _fixture.BucketNameSource = _fixture.GenerateBucketName(); - _fixture.BucketNameSink = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _sourceBucket = _fixture.GenerateBucketName(); + _sinkBucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_sourceBucket); + _fixture.CreateBucketAndGrantStsPermissions(_sinkBucket); _pubSubId = $"projects/{_fixture.ProjectId}/subscriptions/{SubscriptionId}"; CreatePubSubResourcesAndGrantStsPermissions(); } @@ -47,7 +49,7 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) public void CreateEventDrivenGcsTransfer() { CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); - var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _pubSubId); + var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _sourceBucket, _sinkBucket, _pubSubId); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -101,6 +103,8 @@ public void Dispose() PublisherClient.DeleteTopic(topicName); SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(_fixture.ProjectId, SubscriptionId); SubscriberClient.DeleteSubscription(subscriptionName); + _fixture.Storage.DeleteBucket(_sourceBucket); + _fixture.Storage.DeleteBucket(_sinkBucket); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 9efbb3334a1..7f32bd64055 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -25,21 +25,23 @@ public class QuickstartTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly string _sourceBucket; + private readonly string _sinkBucket; public QuickstartTest(StorageFixture fixture) { _fixture = fixture; - _fixture.BucketNameSource = _fixture.GenerateBucketName(); - _fixture.BucketNameSink = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _sourceBucket = _fixture.GenerateBucketName(); + _sinkBucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_sourceBucket); + _fixture.CreateBucketAndGrantStsPermissions(_sinkBucket); } [Fact] public void TestQuickstart() { QuickstartSample quickstartSample = new QuickstartSample(); - var transferJob = quickstartSample.Quickstart(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink); + var transferJob = quickstartSample.Quickstart(_fixture.ProjectId, _sourceBucket, _sinkBucket); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -58,6 +60,8 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + _fixture.Storage.DeleteBucket(_sourceBucket); + _fixture.Storage.DeleteBucket(_sinkBucket); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 83897fd8628..7c3e0588386 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -31,8 +31,6 @@ namespace StorageTransfer.Samples.Tests public class StorageFixture : IDisposable, ICollectionFixture { public string ProjectId { get; } - public string BucketNameSource { get; set; } - public string BucketNameSink { get; set; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } public StorageClient Storage { get; } = StorageClient.Create(); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 43489ba8223..42f39d043be 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -26,12 +26,13 @@ public class TransferBetweenPosixTest : IDisposable private string _transferJobName; private readonly string _tempDirectory; private readonly string _tempDestinationDirectory; + private readonly string _bucket; public TransferBetweenPosixTest(StorageFixture fixture) { _fixture = fixture; - _fixture.BucketNameSource = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); + _bucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_bucket); _tempDirectory = fixture.GenerateTempFolderPath(); _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } @@ -51,7 +52,7 @@ public void TransferBetweenPosix() { sw.WriteLine(Guid.NewGuid().ToString()); } - var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _tempDirectory, _tempDestinationDirectory, _fixture.BucketNameSource); + var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _tempDirectory, _tempDestinationDirectory, _bucket); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; Assert.True(Directory.Exists(_tempDirectory)); @@ -75,6 +76,7 @@ public void Dispose() }); Directory.Delete(_tempDirectory, true); Directory.Delete(_tempDestinationDirectory, true); + _fixture.Storage.DeleteBucket(_bucket); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 88f763d25fc..3f72ac58dc1 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -24,12 +24,13 @@ public class TransferFromPosixTest : IDisposable private readonly StorageFixture _fixture; private string _transferJobName; private readonly string _rootDirectory; + private readonly string _bucket; public TransferFromPosixTest(StorageFixture fixture) { _fixture = fixture; - _fixture.BucketNameSink = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _bucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_bucket); _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } @@ -37,7 +38,7 @@ public TransferFromPosixTest(StorageFixture fixture) public void TransferFromPosix() { TransferFromPosixSample transferFromPosixSample = new TransferFromPosixSample(); - var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _fixture.BucketNameSink); + var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _bucket); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -56,6 +57,7 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + _fixture.Storage.DeleteBucket(_bucket); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index b239ad868eb..2ceddc570b3 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -24,14 +24,16 @@ public class TransferToNearlineTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly string _sourceBucket; + private readonly string _sinkBucket; public TransferToNearlineTest(StorageFixture fixture) { _fixture = fixture; - _fixture.BucketNameSource = _fixture.GenerateBucketName(); - _fixture.BucketNameSink = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _sourceBucket = _fixture.GenerateBucketName(); + _sinkBucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_sourceBucket); + _fixture.CreateBucketAndGrantStsPermissions(_sinkBucket); } [Fact] @@ -39,11 +41,11 @@ public void TransferToNearline() { TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(); var storage = _fixture.Storage; - var bucket = storage.GetBucket(_fixture.BucketNameSink); + var bucket = storage.GetBucket(_sinkBucket); string storageClass = StorageClasses.Nearline; bucket.StorageClass = storageClass; storage.UpdateBucket(bucket); - var transferJob = transferToNearlineSample.TransferToNearline(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink); + var transferJob = transferToNearlineSample.TransferToNearline(_fixture.ProjectId, _sourceBucket, _sinkBucket); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -62,6 +64,8 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + _fixture.Storage.DeleteBucket(_sourceBucket); + _fixture.Storage.DeleteBucket(_sinkBucket); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 58711707643..811acf71385 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -27,25 +27,26 @@ public class TransferUsingManifestTest : IDisposable private string _transferJobName; private readonly string _rootDirectory; private readonly string _manifestObjectName; - private readonly string _bucketNameManifestSource; + private readonly string _manifestBucket; + private readonly string _sinkBucket; public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; - _bucketNameManifestSource = _fixture.GenerateBucketName(); - _fixture.BucketNameSink = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_bucketNameManifestSource); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _manifestBucket = _fixture.GenerateBucketName(); + _sinkBucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_manifestBucket); + _fixture.CreateBucketAndGrantStsPermissions(_sinkBucket); _rootDirectory = fixture.GetCurrentUserTempFolderPath(); _manifestObjectName = $@"{Guid.NewGuid()}.csv"; - UploadObjectToManifestBucket(_bucketNameManifestSource); + UploadObjectToManifestBucket(_manifestBucket); } [Fact] public void TransferUsingManifest() { TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(); - var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _bucketNameManifestSource, _fixture.BucketNameSink, _manifestObjectName); + var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _manifestBucket, _sinkBucket, _manifestObjectName); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -71,7 +72,8 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); - _fixture.Storage.DeleteBucket(_bucketNameManifestSource, new DeleteBucketOptions { DeleteObjects = true }); + _fixture.Storage.DeleteBucket(_manifestBucket, new DeleteBucketOptions { DeleteObjects = true }); + _fixture.Storage.DeleteBucket(_sinkBucket, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 5b554a759e2..e0ddd6fa376 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -24,17 +24,17 @@ public class DownloadToPosixSample /// /// The ID of the Google Cloud project. /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. - /// Your GCS source bucket name. + /// Your GCS source bucket name. /// An optional path on the Google Cloud Storage bucket to download from. /// The root directory path on the source filesystem. public TransferJob DownloadToPosix( string projectId = "my-project-id", string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - string gcsSourceBucket = "my-gcs-source-bucket", + string sourceBucket = "my-source-bucket", string gcsSourcePath = "foo/bar/", string rootDirectory = "/tmp/uploads") { - string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; + string jobDescription = $"Download objects from a GCS source bucket ({sourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; TransferJob transferJob = new TransferJob { @@ -42,7 +42,7 @@ public TransferJob DownloadToPosix( Description = jobDescription, TransferSpec = new TransferSpec { - GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath }, + GcsDataSource = new GcsData { BucketName = sourceBucket, Path = gcsSourcePath }, SinkAgentPoolName = sinkAgentPoolName, PosixDataSink = new PosixFilesystem { RootDirectory = rootDirectory } }, @@ -57,7 +57,7 @@ public TransferJob DownloadToPosix( ProjectId = projectId }); - Console.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); + Console.WriteLine($"Created and ran transfer job from ({sourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); return response; } } From c2b42b8ce53c43c45cecb5135ea40ecfbd200576 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 16 Apr 2025 04:50:44 -0700 Subject: [PATCH 45/49] QuickStart changes --- .../DownloadToPosixTest.cs | 14 +++++++------- .../QuickstartTest.cs | 10 +++++----- .../StorageTransfer.Samples/QuickstartSample.cs | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index ac9116f3abc..0ee2a072e2f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -27,16 +27,16 @@ public class DownloadToPosixTest : IDisposable private string _transferJobName; private readonly string _tempDirectory; private readonly string _gcsSourcePath; - private readonly string _bucketNamePosixSource; + private readonly string _sourceBucket; public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; - _bucketNamePosixSource = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_bucketNamePosixSource); + _sourceBucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_sourceBucket); _tempDirectory = _fixture.GenerateTempFolderPath(); _gcsSourcePath = $"{Guid.NewGuid()}/{Guid.NewGuid()}/"; - UploadObjectToPosixBucket(_bucketNamePosixSource); + UploadObjectToBucket(_sourceBucket); } [Fact] @@ -44,13 +44,13 @@ public void DownloadToPosix() { DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(); Directory.CreateDirectory(_tempDirectory); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _bucketNamePosixSource, _gcsSourcePath, _tempDirectory); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _sourceBucket, _gcsSourcePath, _tempDirectory); Assert.Contains("transferJobs/", transferJob.Name); Assert.True(Directory.Exists(_tempDirectory)); _transferJobName = transferJob.Name; } - private void UploadObjectToPosixBucket(string bucketName) + private void UploadObjectToBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); MemoryStream stream = new MemoryStream(byteArray); @@ -73,7 +73,7 @@ public void Dispose() } }); Directory.Delete(_tempDirectory, true); - _fixture.Storage.DeleteBucket(_bucketNamePosixSource, new DeleteBucketOptions { DeleteObjects = true }); + _fixture.Storage.DeleteBucket(_sourceBucket, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 7f32bd64055..53048c0d578 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -21,14 +21,14 @@ namespace StorageTransfer.Samples.Tests { [Collection(nameof(StorageFixture))] - public class QuickstartTest : IDisposable + public class QuickStartTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; private readonly string _sourceBucket; private readonly string _sinkBucket; - public QuickstartTest(StorageFixture fixture) + public QuickStartTest(StorageFixture fixture) { _fixture = fixture; _sourceBucket = _fixture.GenerateBucketName(); @@ -38,10 +38,10 @@ public QuickstartTest(StorageFixture fixture) } [Fact] - public void TestQuickstart() + public void QuickStart() { - QuickstartSample quickstartSample = new QuickstartSample(); - var transferJob = quickstartSample.Quickstart(_fixture.ProjectId, _sourceBucket, _sinkBucket); + QuickStartSample quickStartSample = new QuickStartSample(); + var transferJob = quickStartSample.QuickStart(_fixture.ProjectId, _sourceBucket, _sinkBucket); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index 276d6177c10..115e71231da 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -19,7 +19,7 @@ using Google.Cloud.StorageTransfer.V1; using System; -public class QuickstartSample +public class QuickStartSample { /// /// Creates an one-time transfer job from a Google Cloud storage bucket to another bucket. @@ -27,7 +27,7 @@ public class QuickstartSample /// The ID of the Google Cloud project. /// The GCS bucket to transfer data from. /// The GCS bucket to transfer data to. - public TransferJob Quickstart( + public TransferJob QuickStart( string projectId = "my-project-id", string sourceBucket = "my-source-bucket", string sinkBucket = "my-sink-bucket") From 5e9cca3270961487df5e2a2c7edae03049b43d41 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 16 Apr 2025 05:07:00 -0700 Subject: [PATCH 46/49] event driven transfer code changes --- storage/api/.editorconfig | 281 ++++++++++++++++++ storagetransfer/api/.editorconfig | 281 ++++++++++++++++++ .../CreateEventDrivenGcsTransferTest.cs | 4 +- 3 files changed, 564 insertions(+), 2 deletions(-) create mode 100644 storage/api/.editorconfig create mode 100644 storagetransfer/api/.editorconfig diff --git a/storage/api/.editorconfig b/storage/api/.editorconfig new file mode 100644 index 00000000000..a0b75fc4dff --- /dev/null +++ b/storage/api/.editorconfig @@ -0,0 +1,281 @@ +# To learn more about .editorconfig see https://aka.ms/editorconfigdocs +# top-most EditorConfig file +root = true +############################### +# Core EditorConfig Options # +############################### +# All files +[*] +insert_final_newline = true +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true + +# Generated code +[*{_AssemblyInfo.cs,.g.cs}] +generated_code = true + +# XML project files +[*.{csproj}] +indent_size = 2 + +# Code files + +############################### +# .NET Coding Conventions # +############################### +[*.{cs}] + +# Organize usings +dotnet_sort_system_directives_first = false + +# this. preferences +dotnet_style_qualification_for_field = false:suggestion +dotnet_style_qualification_for_property = false:suggestion +dotnet_style_qualification_for_method = false:suggestion +dotnet_style_qualification_for_event = false:suggestion + +# Language keywords vs BCL types preferences +dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion +dotnet_style_predefined_type_for_member_access = true:suggestion + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent + +# Modifier preferences +csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning +dotnet_style_require_accessibility_modifiers = for_non_interface_members:warning +dotnet_style_readonly_field = true:warning + +# Expression-level preferences +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_explicit_tuple_names = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion +dotnet_style_prefer_inferred_tuple_names = true:suggestion +dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion +dotnet_style_prefer_auto_properties = true:suggestion +dotnet_style_prefer_conditional_expression_over_assignment = true:silent +dotnet_style_prefer_conditional_expression_over_return = true:silent +csharp_prefer_simple_default_expression = true:suggestion + +############################### +# Naming Conventions # +############################### +# Style Definitions +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# Use PascalCase for constant fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.applicable_accessibilities = * +dotnet_naming_symbols.constant_fields.required_modifiers = const +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# static fields should have s_ prefix +dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion +dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields +dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style +dotnet_naming_symbols.static_fields.applicable_kinds = field +dotnet_naming_symbols.static_fields.required_modifiers = static +dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected +dotnet_naming_style.static_prefix_style.required_prefix = s_ +dotnet_naming_style.static_prefix_style.capitalization = camel_case + +# internal and private fields should be _camelCase +dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion +dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields +dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style +dotnet_naming_symbols.private_internal_fields.applicable_kinds = field +dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal +dotnet_naming_style.camel_case_underscore_style.required_prefix = _ +dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case + +# Locals and parameters are camelCase +dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion +dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters +dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style +dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local +dotnet_naming_style.camel_case_style.capitalization = camel_case + +# Public property and methods should be PascalCase +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.symbols = public_methods_and_properties +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.style = pascal_case +dotnet_naming_symbols.public_methods_and_properties.applicable_kinds = method, property +dotnet_naming_symbols.public_methods_and_properties.applicable_accessibilities = public +dotnet_naming_style.pascal_case.capitalization = pascal_case + +# Private and internal methods should be PascalCase +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.symbols = private_and_internal_methods +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.style = pascal_case +dotnet_naming_symbols.private_and_internal_methods.applicable_kinds = method +dotnet_naming_symbols.private_and_internal_methods.applicable_accessibilities = private, internal +############################### +# C# Coding Conventions # +############################### +[*.cs] +# var preferences +csharp_style_var_for_built_in_types = true:silent +csharp_style_var_when_type_is_apparent = true:silent +csharp_style_var_elsewhere = true:silent +dotnet_style_predefined_type_for_locals_parameters_members = true:silent +dotnet_style_predefined_type_for_member_access = true:silent + +csharp_using_directive_placement = outside_namespace:warning +csharp_prefer_braces = true:suggestion +csharp_preserve_single_line_blocks = true:none +csharp_preserve_single_line_statements = false:none +csharp_prefer_static_local_function = true:suggestion +csharp_prefer_simple_using_statement = true:suggestion +csharp_style_prefer_switch_expression = true:suggestion + +# Expression-bodied members +csharp_style_expression_bodied_methods = true:suggestion +csharp_style_expression_bodied_constructors = true:suggestion +csharp_style_expression_bodied_operators = true:suggestion +csharp_style_expression_bodied_properties = true:suggestion +csharp_style_expression_bodied_indexers = true:suggestion +csharp_style_expression_bodied_accessors = true:suggestion +csharp_style_expression_bodied_lambdas = true:suggestion +csharp_style_expression_bodied_local_functions = true:suggestion + +# Pattern matching preferences +csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion +csharp_style_pattern_matching_over_as_with_null_check = true:suggestion +csharp_style_inlined_variable_declaration = true:suggestion + +# Null-checking preferences +csharp_style_throw_expression = true:suggestion +csharp_style_conditional_delegate_call = true:suggestion + +# Other features +csharp_style_prefer_index_operator = true:suggestion +csharp_style_prefer_range_operator = true:suggestion +csharp_style_pattern_local_over_anonymous_function = false:none + +############################### +# C# Formatting Rules # +############################### +indent_size = 4 +insert_final_newline = true +charset = utf-8 + +# New line preferences +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation preferences +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_switch_labels = true +csharp_indent_labels = one_less_than_current + +# Space preferences +csharp_space_after_cast = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_after_comma = true +csharp_space_after_dot = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_after_semicolon_in_for_statement = true +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_parentheses = false +csharp_space_before_comma = false +csharp_space_before_dot = false +csharp_space_before_open_square_brackets = false +csharp_space_before_semicolon_in_for_statement = false +csharp_space_between_empty_square_brackets = false +csharp_space_around_declaration_statements = do_not_ignore +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_square_brackets = false + +# Wrapping preferences +csharp_preserve_single_line_statements = true +csharp_preserve_single_line_blocks = true + +# License header +file_header_template = Copyright 2024 Google LLC\n\nLicensed under the Apache License, Version 2.0 (the "License").\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at \n\nhttps://www.apache.org/licenses/LICENSE-2.0 \n\nUnless required by applicable law or agreed to in writing, software \ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and \nlimitations under the License. +csharp_style_namespace_declarations = file_scoped:suggestion +csharp_style_prefer_null_check_over_type_check = true:suggestion +csharp_style_prefer_local_over_anonymous_function = true:suggestion +csharp_style_prefer_tuple_swap = true:suggestion +csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion +csharp_style_unused_value_assignment_preference = discard_variable:suggestion +csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion +csharp_style_prefer_pattern_matching = true:suggestion +csharp_style_prefer_not_pattern = true:suggestion +csharp_style_prefer_extended_property_pattern = true:suggestion +csharp_style_prefer_parameter_null_checking = true:suggestion +csharp_style_prefer_method_group_conversion = true:suggestion + +# Change the severity to `error` for the rules that must not be violated, to fail the build. + +# IDE0005: Remove unnecessary using directives. +dotnet_diagnostic.IDE0005.severity = suggestion + +# IDE0011: Add braces +dotnet_diagnostic.IDE0011.severity = suggestion + +# IDE0017: Object initialization can be simplified. +dotnet_diagnostic.IDE0017.severity = suggestion + +# IDE0019: Use pattern matching. +dotnet_diagnostic.IDE0019.severity = suggestion + +# IDE0021: Use expression body for constructor. +dotnet_diagnostic.IDE0021.severity = suggestion + +# IDE0028: Collection initialization can be simplified. +dotnet_diagnostic.IDE0028.severity = suggestion + +# IDE0035: Remove unreachable code. +dotnet_diagnostic.IDE0035.severity = suggestion + +# IDE0040: Accessibility modifiers required. +dotnet_diagnostic.IDE0040.severity = suggestion + +# IDE0052: Remove unread private member +dotnet_diagnostic.IDE0052.severity = suggestion + +# IDE0053: Use expression body for lambda expression. +dotnet_diagnostic.IDE0053.severity = suggestion + +# IDE0055: All C# and .NET formatting rules. +dotnet_diagnostic.IDE0055.severity = suggestion + +# IDE0060: Remove unused parameter +dotnet_diagnostic.IDE0060.severity = suggestion + +# IDE0063: Using statement can be simplified +dotnet_diagnostic.IDE0063.severity = suggestion + +# IDE0065: Using statement can be simplified +dotnet_diagnostic.IDE0065.severity = suggestion + +# IDE0071: Interpolation can be simplified. +dotnet_diagnostic.IDE0071.severity = suggestion + +# IDE0075: Simplify conditional expression. +dotnet_diagnostic.IDE0075.severity = suggestion diff --git a/storagetransfer/api/.editorconfig b/storagetransfer/api/.editorconfig new file mode 100644 index 00000000000..a0b75fc4dff --- /dev/null +++ b/storagetransfer/api/.editorconfig @@ -0,0 +1,281 @@ +# To learn more about .editorconfig see https://aka.ms/editorconfigdocs +# top-most EditorConfig file +root = true +############################### +# Core EditorConfig Options # +############################### +# All files +[*] +insert_final_newline = true +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true + +# Generated code +[*{_AssemblyInfo.cs,.g.cs}] +generated_code = true + +# XML project files +[*.{csproj}] +indent_size = 2 + +# Code files + +############################### +# .NET Coding Conventions # +############################### +[*.{cs}] + +# Organize usings +dotnet_sort_system_directives_first = false + +# this. preferences +dotnet_style_qualification_for_field = false:suggestion +dotnet_style_qualification_for_property = false:suggestion +dotnet_style_qualification_for_method = false:suggestion +dotnet_style_qualification_for_event = false:suggestion + +# Language keywords vs BCL types preferences +dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion +dotnet_style_predefined_type_for_member_access = true:suggestion + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent + +# Modifier preferences +csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning +dotnet_style_require_accessibility_modifiers = for_non_interface_members:warning +dotnet_style_readonly_field = true:warning + +# Expression-level preferences +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_explicit_tuple_names = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion +dotnet_style_prefer_inferred_tuple_names = true:suggestion +dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion +dotnet_style_prefer_auto_properties = true:suggestion +dotnet_style_prefer_conditional_expression_over_assignment = true:silent +dotnet_style_prefer_conditional_expression_over_return = true:silent +csharp_prefer_simple_default_expression = true:suggestion + +############################### +# Naming Conventions # +############################### +# Style Definitions +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# Use PascalCase for constant fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.applicable_accessibilities = * +dotnet_naming_symbols.constant_fields.required_modifiers = const +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# static fields should have s_ prefix +dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion +dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields +dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style +dotnet_naming_symbols.static_fields.applicable_kinds = field +dotnet_naming_symbols.static_fields.required_modifiers = static +dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected +dotnet_naming_style.static_prefix_style.required_prefix = s_ +dotnet_naming_style.static_prefix_style.capitalization = camel_case + +# internal and private fields should be _camelCase +dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion +dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields +dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style +dotnet_naming_symbols.private_internal_fields.applicable_kinds = field +dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal +dotnet_naming_style.camel_case_underscore_style.required_prefix = _ +dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case + +# Locals and parameters are camelCase +dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion +dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters +dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style +dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local +dotnet_naming_style.camel_case_style.capitalization = camel_case + +# Public property and methods should be PascalCase +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.symbols = public_methods_and_properties +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.style = pascal_case +dotnet_naming_symbols.public_methods_and_properties.applicable_kinds = method, property +dotnet_naming_symbols.public_methods_and_properties.applicable_accessibilities = public +dotnet_naming_style.pascal_case.capitalization = pascal_case + +# Private and internal methods should be PascalCase +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.symbols = private_and_internal_methods +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.style = pascal_case +dotnet_naming_symbols.private_and_internal_methods.applicable_kinds = method +dotnet_naming_symbols.private_and_internal_methods.applicable_accessibilities = private, internal +############################### +# C# Coding Conventions # +############################### +[*.cs] +# var preferences +csharp_style_var_for_built_in_types = true:silent +csharp_style_var_when_type_is_apparent = true:silent +csharp_style_var_elsewhere = true:silent +dotnet_style_predefined_type_for_locals_parameters_members = true:silent +dotnet_style_predefined_type_for_member_access = true:silent + +csharp_using_directive_placement = outside_namespace:warning +csharp_prefer_braces = true:suggestion +csharp_preserve_single_line_blocks = true:none +csharp_preserve_single_line_statements = false:none +csharp_prefer_static_local_function = true:suggestion +csharp_prefer_simple_using_statement = true:suggestion +csharp_style_prefer_switch_expression = true:suggestion + +# Expression-bodied members +csharp_style_expression_bodied_methods = true:suggestion +csharp_style_expression_bodied_constructors = true:suggestion +csharp_style_expression_bodied_operators = true:suggestion +csharp_style_expression_bodied_properties = true:suggestion +csharp_style_expression_bodied_indexers = true:suggestion +csharp_style_expression_bodied_accessors = true:suggestion +csharp_style_expression_bodied_lambdas = true:suggestion +csharp_style_expression_bodied_local_functions = true:suggestion + +# Pattern matching preferences +csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion +csharp_style_pattern_matching_over_as_with_null_check = true:suggestion +csharp_style_inlined_variable_declaration = true:suggestion + +# Null-checking preferences +csharp_style_throw_expression = true:suggestion +csharp_style_conditional_delegate_call = true:suggestion + +# Other features +csharp_style_prefer_index_operator = true:suggestion +csharp_style_prefer_range_operator = true:suggestion +csharp_style_pattern_local_over_anonymous_function = false:none + +############################### +# C# Formatting Rules # +############################### +indent_size = 4 +insert_final_newline = true +charset = utf-8 + +# New line preferences +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation preferences +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_switch_labels = true +csharp_indent_labels = one_less_than_current + +# Space preferences +csharp_space_after_cast = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_after_comma = true +csharp_space_after_dot = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_after_semicolon_in_for_statement = true +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_parentheses = false +csharp_space_before_comma = false +csharp_space_before_dot = false +csharp_space_before_open_square_brackets = false +csharp_space_before_semicolon_in_for_statement = false +csharp_space_between_empty_square_brackets = false +csharp_space_around_declaration_statements = do_not_ignore +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_square_brackets = false + +# Wrapping preferences +csharp_preserve_single_line_statements = true +csharp_preserve_single_line_blocks = true + +# License header +file_header_template = Copyright 2024 Google LLC\n\nLicensed under the Apache License, Version 2.0 (the "License").\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at \n\nhttps://www.apache.org/licenses/LICENSE-2.0 \n\nUnless required by applicable law or agreed to in writing, software \ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and \nlimitations under the License. +csharp_style_namespace_declarations = file_scoped:suggestion +csharp_style_prefer_null_check_over_type_check = true:suggestion +csharp_style_prefer_local_over_anonymous_function = true:suggestion +csharp_style_prefer_tuple_swap = true:suggestion +csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion +csharp_style_unused_value_assignment_preference = discard_variable:suggestion +csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion +csharp_style_prefer_pattern_matching = true:suggestion +csharp_style_prefer_not_pattern = true:suggestion +csharp_style_prefer_extended_property_pattern = true:suggestion +csharp_style_prefer_parameter_null_checking = true:suggestion +csharp_style_prefer_method_group_conversion = true:suggestion + +# Change the severity to `error` for the rules that must not be violated, to fail the build. + +# IDE0005: Remove unnecessary using directives. +dotnet_diagnostic.IDE0005.severity = suggestion + +# IDE0011: Add braces +dotnet_diagnostic.IDE0011.severity = suggestion + +# IDE0017: Object initialization can be simplified. +dotnet_diagnostic.IDE0017.severity = suggestion + +# IDE0019: Use pattern matching. +dotnet_diagnostic.IDE0019.severity = suggestion + +# IDE0021: Use expression body for constructor. +dotnet_diagnostic.IDE0021.severity = suggestion + +# IDE0028: Collection initialization can be simplified. +dotnet_diagnostic.IDE0028.severity = suggestion + +# IDE0035: Remove unreachable code. +dotnet_diagnostic.IDE0035.severity = suggestion + +# IDE0040: Accessibility modifiers required. +dotnet_diagnostic.IDE0040.severity = suggestion + +# IDE0052: Remove unread private member +dotnet_diagnostic.IDE0052.severity = suggestion + +# IDE0053: Use expression body for lambda expression. +dotnet_diagnostic.IDE0053.severity = suggestion + +# IDE0055: All C# and .NET formatting rules. +dotnet_diagnostic.IDE0055.severity = suggestion + +# IDE0060: Remove unused parameter +dotnet_diagnostic.IDE0060.severity = suggestion + +# IDE0063: Using statement can be simplified +dotnet_diagnostic.IDE0063.severity = suggestion + +# IDE0065: Using statement can be simplified +dotnet_diagnostic.IDE0065.severity = suggestion + +# IDE0071: Interpolation can be simplified. +dotnet_diagnostic.IDE0071.severity = suggestion + +# IDE0075: Simplify conditional expression. +dotnet_diagnostic.IDE0075.severity = suggestion diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 8e65a07b7fa..8815afa51df 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -29,8 +29,8 @@ public class CreateEventDrivenGcsTransferTest : IDisposable private string _transferJobName; private readonly string _sourceBucket; private readonly string _sinkBucket; - private string TopicId { get; } = Guid.NewGuid().ToString(); - private string SubscriptionId { get; } = Guid.NewGuid().ToString(); + private string TopicId { get; } = $"Topic-{Guid.NewGuid().ToString()}"; + private string SubscriptionId { get; } = $"Subscription-{Guid.NewGuid().ToString()}"; private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); private PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); From c7e1a3576b0af45880b8710b53d19c21e683f2a2 Mon Sep 17 00:00:00 2001 From: Mahendra Date: Wed, 16 Apr 2025 17:38:59 +0530 Subject: [PATCH 47/49] Delete storage/api/.editorconfig --- storage/api/.editorconfig | 281 -------------------------------------- 1 file changed, 281 deletions(-) delete mode 100644 storage/api/.editorconfig diff --git a/storage/api/.editorconfig b/storage/api/.editorconfig deleted file mode 100644 index a0b75fc4dff..00000000000 --- a/storage/api/.editorconfig +++ /dev/null @@ -1,281 +0,0 @@ -# To learn more about .editorconfig see https://aka.ms/editorconfigdocs -# top-most EditorConfig file -root = true -############################### -# Core EditorConfig Options # -############################### -# All files -[*] -insert_final_newline = true -indent_style = space -indent_size = 4 -trim_trailing_whitespace = true - -# Generated code -[*{_AssemblyInfo.cs,.g.cs}] -generated_code = true - -# XML project files -[*.{csproj}] -indent_size = 2 - -# Code files - -############################### -# .NET Coding Conventions # -############################### -[*.{cs}] - -# Organize usings -dotnet_sort_system_directives_first = false - -# this. preferences -dotnet_style_qualification_for_field = false:suggestion -dotnet_style_qualification_for_property = false:suggestion -dotnet_style_qualification_for_method = false:suggestion -dotnet_style_qualification_for_event = false:suggestion - -# Language keywords vs BCL types preferences -dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion -dotnet_style_predefined_type_for_member_access = true:suggestion - -# Parentheses preferences -dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent - -# Modifier preferences -csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning -dotnet_style_require_accessibility_modifiers = for_non_interface_members:warning -dotnet_style_readonly_field = true:warning - -# Expression-level preferences -dotnet_style_object_initializer = true:suggestion -dotnet_style_collection_initializer = true:suggestion -dotnet_style_explicit_tuple_names = true:suggestion -dotnet_style_null_propagation = true:suggestion -dotnet_style_coalesce_expression = true:suggestion -dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion -dotnet_style_prefer_inferred_tuple_names = true:suggestion -dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion -dotnet_style_prefer_auto_properties = true:suggestion -dotnet_style_prefer_conditional_expression_over_assignment = true:silent -dotnet_style_prefer_conditional_expression_over_return = true:silent -csharp_prefer_simple_default_expression = true:suggestion - -############################### -# Naming Conventions # -############################### -# Style Definitions -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -# Use PascalCase for constant fields -dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields -dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style -dotnet_naming_symbols.constant_fields.applicable_kinds = field -dotnet_naming_symbols.constant_fields.applicable_accessibilities = * -dotnet_naming_symbols.constant_fields.required_modifiers = const -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -# static fields should have s_ prefix -dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion -dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields -dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style -dotnet_naming_symbols.static_fields.applicable_kinds = field -dotnet_naming_symbols.static_fields.required_modifiers = static -dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected -dotnet_naming_style.static_prefix_style.required_prefix = s_ -dotnet_naming_style.static_prefix_style.capitalization = camel_case - -# internal and private fields should be _camelCase -dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion -dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields -dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style -dotnet_naming_symbols.private_internal_fields.applicable_kinds = field -dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal -dotnet_naming_style.camel_case_underscore_style.required_prefix = _ -dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case - -# Locals and parameters are camelCase -dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion -dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters -dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style -dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local -dotnet_naming_style.camel_case_style.capitalization = camel_case - -# Public property and methods should be PascalCase -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.symbols = public_methods_and_properties -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.style = pascal_case -dotnet_naming_symbols.public_methods_and_properties.applicable_kinds = method, property -dotnet_naming_symbols.public_methods_and_properties.applicable_accessibilities = public -dotnet_naming_style.pascal_case.capitalization = pascal_case - -# Private and internal methods should be PascalCase -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.symbols = private_and_internal_methods -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.style = pascal_case -dotnet_naming_symbols.private_and_internal_methods.applicable_kinds = method -dotnet_naming_symbols.private_and_internal_methods.applicable_accessibilities = private, internal -############################### -# C# Coding Conventions # -############################### -[*.cs] -# var preferences -csharp_style_var_for_built_in_types = true:silent -csharp_style_var_when_type_is_apparent = true:silent -csharp_style_var_elsewhere = true:silent -dotnet_style_predefined_type_for_locals_parameters_members = true:silent -dotnet_style_predefined_type_for_member_access = true:silent - -csharp_using_directive_placement = outside_namespace:warning -csharp_prefer_braces = true:suggestion -csharp_preserve_single_line_blocks = true:none -csharp_preserve_single_line_statements = false:none -csharp_prefer_static_local_function = true:suggestion -csharp_prefer_simple_using_statement = true:suggestion -csharp_style_prefer_switch_expression = true:suggestion - -# Expression-bodied members -csharp_style_expression_bodied_methods = true:suggestion -csharp_style_expression_bodied_constructors = true:suggestion -csharp_style_expression_bodied_operators = true:suggestion -csharp_style_expression_bodied_properties = true:suggestion -csharp_style_expression_bodied_indexers = true:suggestion -csharp_style_expression_bodied_accessors = true:suggestion -csharp_style_expression_bodied_lambdas = true:suggestion -csharp_style_expression_bodied_local_functions = true:suggestion - -# Pattern matching preferences -csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion -csharp_style_pattern_matching_over_as_with_null_check = true:suggestion -csharp_style_inlined_variable_declaration = true:suggestion - -# Null-checking preferences -csharp_style_throw_expression = true:suggestion -csharp_style_conditional_delegate_call = true:suggestion - -# Other features -csharp_style_prefer_index_operator = true:suggestion -csharp_style_prefer_range_operator = true:suggestion -csharp_style_pattern_local_over_anonymous_function = false:none - -############################### -# C# Formatting Rules # -############################### -indent_size = 4 -insert_final_newline = true -charset = utf-8 - -# New line preferences -csharp_new_line_before_open_brace = all -csharp_new_line_before_else = true -csharp_new_line_before_catch = true -csharp_new_line_before_finally = true -csharp_new_line_before_members_in_object_initializers = true -csharp_new_line_before_members_in_anonymous_types = true -csharp_new_line_between_query_expression_clauses = true - -# Indentation preferences -csharp_indent_block_contents = true -csharp_indent_braces = false -csharp_indent_case_contents = true -csharp_indent_case_contents_when_block = true -csharp_indent_switch_labels = true -csharp_indent_labels = one_less_than_current - -# Space preferences -csharp_space_after_cast = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_after_comma = true -csharp_space_after_dot = false -csharp_space_after_keywords_in_control_flow_statements = true -csharp_space_after_semicolon_in_for_statement = true -csharp_space_between_method_call_parameter_list_parentheses = false -csharp_space_between_method_declaration_parameter_list_parentheses = false -csharp_space_between_parentheses = false -csharp_space_before_comma = false -csharp_space_before_dot = false -csharp_space_before_open_square_brackets = false -csharp_space_before_semicolon_in_for_statement = false -csharp_space_between_empty_square_brackets = false -csharp_space_around_declaration_statements = do_not_ignore -csharp_space_before_colon_in_inheritance_clause = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_around_binary_operators = before_and_after -csharp_space_between_method_declaration_empty_parameter_list_parentheses = false -csharp_space_between_method_call_name_and_opening_parenthesis = false -csharp_space_between_method_call_empty_parameter_list_parentheses = false -csharp_space_between_method_declaration_name_and_open_parenthesis = false -csharp_space_between_square_brackets = false - -# Wrapping preferences -csharp_preserve_single_line_statements = true -csharp_preserve_single_line_blocks = true - -# License header -file_header_template = Copyright 2024 Google LLC\n\nLicensed under the Apache License, Version 2.0 (the "License").\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at \n\nhttps://www.apache.org/licenses/LICENSE-2.0 \n\nUnless required by applicable law or agreed to in writing, software \ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and \nlimitations under the License. -csharp_style_namespace_declarations = file_scoped:suggestion -csharp_style_prefer_null_check_over_type_check = true:suggestion -csharp_style_prefer_local_over_anonymous_function = true:suggestion -csharp_style_prefer_tuple_swap = true:suggestion -csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion -csharp_style_unused_value_assignment_preference = discard_variable:suggestion -csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion -csharp_style_prefer_pattern_matching = true:suggestion -csharp_style_prefer_not_pattern = true:suggestion -csharp_style_prefer_extended_property_pattern = true:suggestion -csharp_style_prefer_parameter_null_checking = true:suggestion -csharp_style_prefer_method_group_conversion = true:suggestion - -# Change the severity to `error` for the rules that must not be violated, to fail the build. - -# IDE0005: Remove unnecessary using directives. -dotnet_diagnostic.IDE0005.severity = suggestion - -# IDE0011: Add braces -dotnet_diagnostic.IDE0011.severity = suggestion - -# IDE0017: Object initialization can be simplified. -dotnet_diagnostic.IDE0017.severity = suggestion - -# IDE0019: Use pattern matching. -dotnet_diagnostic.IDE0019.severity = suggestion - -# IDE0021: Use expression body for constructor. -dotnet_diagnostic.IDE0021.severity = suggestion - -# IDE0028: Collection initialization can be simplified. -dotnet_diagnostic.IDE0028.severity = suggestion - -# IDE0035: Remove unreachable code. -dotnet_diagnostic.IDE0035.severity = suggestion - -# IDE0040: Accessibility modifiers required. -dotnet_diagnostic.IDE0040.severity = suggestion - -# IDE0052: Remove unread private member -dotnet_diagnostic.IDE0052.severity = suggestion - -# IDE0053: Use expression body for lambda expression. -dotnet_diagnostic.IDE0053.severity = suggestion - -# IDE0055: All C# and .NET formatting rules. -dotnet_diagnostic.IDE0055.severity = suggestion - -# IDE0060: Remove unused parameter -dotnet_diagnostic.IDE0060.severity = suggestion - -# IDE0063: Using statement can be simplified -dotnet_diagnostic.IDE0063.severity = suggestion - -# IDE0065: Using statement can be simplified -dotnet_diagnostic.IDE0065.severity = suggestion - -# IDE0071: Interpolation can be simplified. -dotnet_diagnostic.IDE0071.severity = suggestion - -# IDE0075: Simplify conditional expression. -dotnet_diagnostic.IDE0075.severity = suggestion From e4f5c85c1dd8d3ba5cd1074c4afb87161f9ea523 Mon Sep 17 00:00:00 2001 From: Mahendra Date: Wed, 16 Apr 2025 17:39:33 +0530 Subject: [PATCH 48/49] Delete storagetransfer/api/.editorconfig --- storagetransfer/api/.editorconfig | 281 ------------------------------ 1 file changed, 281 deletions(-) delete mode 100644 storagetransfer/api/.editorconfig diff --git a/storagetransfer/api/.editorconfig b/storagetransfer/api/.editorconfig deleted file mode 100644 index a0b75fc4dff..00000000000 --- a/storagetransfer/api/.editorconfig +++ /dev/null @@ -1,281 +0,0 @@ -# To learn more about .editorconfig see https://aka.ms/editorconfigdocs -# top-most EditorConfig file -root = true -############################### -# Core EditorConfig Options # -############################### -# All files -[*] -insert_final_newline = true -indent_style = space -indent_size = 4 -trim_trailing_whitespace = true - -# Generated code -[*{_AssemblyInfo.cs,.g.cs}] -generated_code = true - -# XML project files -[*.{csproj}] -indent_size = 2 - -# Code files - -############################### -# .NET Coding Conventions # -############################### -[*.{cs}] - -# Organize usings -dotnet_sort_system_directives_first = false - -# this. preferences -dotnet_style_qualification_for_field = false:suggestion -dotnet_style_qualification_for_property = false:suggestion -dotnet_style_qualification_for_method = false:suggestion -dotnet_style_qualification_for_event = false:suggestion - -# Language keywords vs BCL types preferences -dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion -dotnet_style_predefined_type_for_member_access = true:suggestion - -# Parentheses preferences -dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent - -# Modifier preferences -csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning -dotnet_style_require_accessibility_modifiers = for_non_interface_members:warning -dotnet_style_readonly_field = true:warning - -# Expression-level preferences -dotnet_style_object_initializer = true:suggestion -dotnet_style_collection_initializer = true:suggestion -dotnet_style_explicit_tuple_names = true:suggestion -dotnet_style_null_propagation = true:suggestion -dotnet_style_coalesce_expression = true:suggestion -dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion -dotnet_style_prefer_inferred_tuple_names = true:suggestion -dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion -dotnet_style_prefer_auto_properties = true:suggestion -dotnet_style_prefer_conditional_expression_over_assignment = true:silent -dotnet_style_prefer_conditional_expression_over_return = true:silent -csharp_prefer_simple_default_expression = true:suggestion - -############################### -# Naming Conventions # -############################### -# Style Definitions -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -# Use PascalCase for constant fields -dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields -dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style -dotnet_naming_symbols.constant_fields.applicable_kinds = field -dotnet_naming_symbols.constant_fields.applicable_accessibilities = * -dotnet_naming_symbols.constant_fields.required_modifiers = const -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -# static fields should have s_ prefix -dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion -dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields -dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style -dotnet_naming_symbols.static_fields.applicable_kinds = field -dotnet_naming_symbols.static_fields.required_modifiers = static -dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected -dotnet_naming_style.static_prefix_style.required_prefix = s_ -dotnet_naming_style.static_prefix_style.capitalization = camel_case - -# internal and private fields should be _camelCase -dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion -dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields -dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style -dotnet_naming_symbols.private_internal_fields.applicable_kinds = field -dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal -dotnet_naming_style.camel_case_underscore_style.required_prefix = _ -dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case - -# Locals and parameters are camelCase -dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion -dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters -dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style -dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local -dotnet_naming_style.camel_case_style.capitalization = camel_case - -# Public property and methods should be PascalCase -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.symbols = public_methods_and_properties -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.style = pascal_case -dotnet_naming_symbols.public_methods_and_properties.applicable_kinds = method, property -dotnet_naming_symbols.public_methods_and_properties.applicable_accessibilities = public -dotnet_naming_style.pascal_case.capitalization = pascal_case - -# Private and internal methods should be PascalCase -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.symbols = private_and_internal_methods -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.style = pascal_case -dotnet_naming_symbols.private_and_internal_methods.applicable_kinds = method -dotnet_naming_symbols.private_and_internal_methods.applicable_accessibilities = private, internal -############################### -# C# Coding Conventions # -############################### -[*.cs] -# var preferences -csharp_style_var_for_built_in_types = true:silent -csharp_style_var_when_type_is_apparent = true:silent -csharp_style_var_elsewhere = true:silent -dotnet_style_predefined_type_for_locals_parameters_members = true:silent -dotnet_style_predefined_type_for_member_access = true:silent - -csharp_using_directive_placement = outside_namespace:warning -csharp_prefer_braces = true:suggestion -csharp_preserve_single_line_blocks = true:none -csharp_preserve_single_line_statements = false:none -csharp_prefer_static_local_function = true:suggestion -csharp_prefer_simple_using_statement = true:suggestion -csharp_style_prefer_switch_expression = true:suggestion - -# Expression-bodied members -csharp_style_expression_bodied_methods = true:suggestion -csharp_style_expression_bodied_constructors = true:suggestion -csharp_style_expression_bodied_operators = true:suggestion -csharp_style_expression_bodied_properties = true:suggestion -csharp_style_expression_bodied_indexers = true:suggestion -csharp_style_expression_bodied_accessors = true:suggestion -csharp_style_expression_bodied_lambdas = true:suggestion -csharp_style_expression_bodied_local_functions = true:suggestion - -# Pattern matching preferences -csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion -csharp_style_pattern_matching_over_as_with_null_check = true:suggestion -csharp_style_inlined_variable_declaration = true:suggestion - -# Null-checking preferences -csharp_style_throw_expression = true:suggestion -csharp_style_conditional_delegate_call = true:suggestion - -# Other features -csharp_style_prefer_index_operator = true:suggestion -csharp_style_prefer_range_operator = true:suggestion -csharp_style_pattern_local_over_anonymous_function = false:none - -############################### -# C# Formatting Rules # -############################### -indent_size = 4 -insert_final_newline = true -charset = utf-8 - -# New line preferences -csharp_new_line_before_open_brace = all -csharp_new_line_before_else = true -csharp_new_line_before_catch = true -csharp_new_line_before_finally = true -csharp_new_line_before_members_in_object_initializers = true -csharp_new_line_before_members_in_anonymous_types = true -csharp_new_line_between_query_expression_clauses = true - -# Indentation preferences -csharp_indent_block_contents = true -csharp_indent_braces = false -csharp_indent_case_contents = true -csharp_indent_case_contents_when_block = true -csharp_indent_switch_labels = true -csharp_indent_labels = one_less_than_current - -# Space preferences -csharp_space_after_cast = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_after_comma = true -csharp_space_after_dot = false -csharp_space_after_keywords_in_control_flow_statements = true -csharp_space_after_semicolon_in_for_statement = true -csharp_space_between_method_call_parameter_list_parentheses = false -csharp_space_between_method_declaration_parameter_list_parentheses = false -csharp_space_between_parentheses = false -csharp_space_before_comma = false -csharp_space_before_dot = false -csharp_space_before_open_square_brackets = false -csharp_space_before_semicolon_in_for_statement = false -csharp_space_between_empty_square_brackets = false -csharp_space_around_declaration_statements = do_not_ignore -csharp_space_before_colon_in_inheritance_clause = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_around_binary_operators = before_and_after -csharp_space_between_method_declaration_empty_parameter_list_parentheses = false -csharp_space_between_method_call_name_and_opening_parenthesis = false -csharp_space_between_method_call_empty_parameter_list_parentheses = false -csharp_space_between_method_declaration_name_and_open_parenthesis = false -csharp_space_between_square_brackets = false - -# Wrapping preferences -csharp_preserve_single_line_statements = true -csharp_preserve_single_line_blocks = true - -# License header -file_header_template = Copyright 2024 Google LLC\n\nLicensed under the Apache License, Version 2.0 (the "License").\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at \n\nhttps://www.apache.org/licenses/LICENSE-2.0 \n\nUnless required by applicable law or agreed to in writing, software \ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and \nlimitations under the License. -csharp_style_namespace_declarations = file_scoped:suggestion -csharp_style_prefer_null_check_over_type_check = true:suggestion -csharp_style_prefer_local_over_anonymous_function = true:suggestion -csharp_style_prefer_tuple_swap = true:suggestion -csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion -csharp_style_unused_value_assignment_preference = discard_variable:suggestion -csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion -csharp_style_prefer_pattern_matching = true:suggestion -csharp_style_prefer_not_pattern = true:suggestion -csharp_style_prefer_extended_property_pattern = true:suggestion -csharp_style_prefer_parameter_null_checking = true:suggestion -csharp_style_prefer_method_group_conversion = true:suggestion - -# Change the severity to `error` for the rules that must not be violated, to fail the build. - -# IDE0005: Remove unnecessary using directives. -dotnet_diagnostic.IDE0005.severity = suggestion - -# IDE0011: Add braces -dotnet_diagnostic.IDE0011.severity = suggestion - -# IDE0017: Object initialization can be simplified. -dotnet_diagnostic.IDE0017.severity = suggestion - -# IDE0019: Use pattern matching. -dotnet_diagnostic.IDE0019.severity = suggestion - -# IDE0021: Use expression body for constructor. -dotnet_diagnostic.IDE0021.severity = suggestion - -# IDE0028: Collection initialization can be simplified. -dotnet_diagnostic.IDE0028.severity = suggestion - -# IDE0035: Remove unreachable code. -dotnet_diagnostic.IDE0035.severity = suggestion - -# IDE0040: Accessibility modifiers required. -dotnet_diagnostic.IDE0040.severity = suggestion - -# IDE0052: Remove unread private member -dotnet_diagnostic.IDE0052.severity = suggestion - -# IDE0053: Use expression body for lambda expression. -dotnet_diagnostic.IDE0053.severity = suggestion - -# IDE0055: All C# and .NET formatting rules. -dotnet_diagnostic.IDE0055.severity = suggestion - -# IDE0060: Remove unused parameter -dotnet_diagnostic.IDE0060.severity = suggestion - -# IDE0063: Using statement can be simplified -dotnet_diagnostic.IDE0063.severity = suggestion - -# IDE0065: Using statement can be simplified -dotnet_diagnostic.IDE0065.severity = suggestion - -# IDE0071: Interpolation can be simplified. -dotnet_diagnostic.IDE0071.severity = suggestion - -# IDE0075: Simplify conditional expression. -dotnet_diagnostic.IDE0075.severity = suggestion From 54197f5f78c71c43e6dea6c9dd338e05e1db924c Mon Sep 17 00:00:00 2001 From: Mahendra Date: Thu, 17 Apr 2025 11:11:31 +0530 Subject: [PATCH 49/49] QuickStart sample and tests files renamed --- .../{QuickstartTest.cs => QuickStartTest.cs} | 0 .../{QuickstartSample.cs => QuickStartSample.cs} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename storagetransfer/api/StorageTransfer.Samples.Tests/{QuickstartTest.cs => QuickStartTest.cs} (100%) rename storagetransfer/api/StorageTransfer.Samples/{QuickstartSample.cs => QuickStartSample.cs} (100%) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickStartTest.cs similarity index 100% rename from storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs rename to storagetransfer/api/StorageTransfer.Samples.Tests/QuickStartTest.cs diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickStartSample.cs similarity index 100% rename from storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs rename to storagetransfer/api/StorageTransfer.Samples/QuickStartSample.cs