Skip to content

Commit 000b432

Browse files
mahendra-googlemahendra-google
authored
feat (storage transfer) : addition of samples and test cases for transfer to nearline and check latest transfer operation (#1)
2 parents fd311f1 + e3c0619 commit 000b432

8 files changed

+279
-12
lines changed

‎storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License").
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
using Xunit;
15+
using Xunit.Abstractions;
16+
17+
namespace StorageTransfer.Samples.Tests;
18+
19+
[Collection(nameof(StorageFixture))]
20+
public class CheckLatestTransferOperationTest
21+
{
22+
23+
private readonly StorageFixture _fixture;
24+
private string _jobName;
25+
private readonly ITestOutputHelper _outputHelper;
26+
public CheckLatestTransferOperationTest(StorageFixture fixture, ITestOutputHelper outputHelper)
27+
{
28+
_outputHelper = outputHelper;
29+
_fixture = fixture;
30+
}
31+
32+
[Fact]
33+
public void CheckLatestTransferOperation()
34+
{
35+
CheckLatestTransferOperationSample checkLatestTransferOperationSample = new CheckLatestTransferOperationSample(_outputHelper);
36+
var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId,_fixture.JobName);
37+
Assert.Contains("transferJobs/", transferJob.Name);
38+
_jobName = transferJob.Name;
39+
}
40+
}

‎storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs

+7-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
/**
2-
* Copyright 2021 Google Inc.
1+
/**
2+
* Copyright 2024 Google Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
1717
using System;
1818
using Google.Cloud.StorageTransfer.V1;
1919
using Xunit;
20+
using Xunit.Abstractions;
2021

2122
namespace StorageTransfer.Samples.Tests
2223
{
@@ -25,16 +26,18 @@ public class QuickstartTest : IDisposable
2526
{
2627
private readonly StorageFixture _fixture;
2728
private string _transferJobName;
29+
private readonly ITestOutputHelper _outputHelper;
2830

29-
public QuickstartTest(StorageFixture fixture)
31+
public QuickstartTest(StorageFixture fixture , ITestOutputHelper outputHelper)
3032
{
3133
_fixture = fixture;
34+
_outputHelper = outputHelper;
3235
}
3336

3437
[Fact]
3538
public void TestQuickstart()
3639
{
37-
QuickstartSample quickstartSample = new QuickstartSample();
40+
QuickstartSample quickstartSample = new QuickstartSample(_outputHelper);
3841
var transferJob = quickstartSample.Quickstart(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink);
3942
Assert.Contains("transferJobs/", transferJob.Name);
4043
_transferJobName = transferJob.Name;

‎storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs

+6-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/**
1+
/**
22
* Copyright 2021 Google Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,11 +28,16 @@ public class StorageFixture : IDisposable, ICollectionFixture<StorageFixture>
2828
public string ProjectId { get; }
2929
public string BucketNameSource { get; } = Guid.NewGuid().ToString();
3030
public string BucketNameSink { get; } = Guid.NewGuid().ToString();
31+
public string JobName { get; }
3132
public StorageClient Storage { get; } = StorageClient.Create();
3233
public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create();
3334

3435
public StorageFixture()
3536
{
37+
// Instantiate random number generator
38+
Random random = new Random();
39+
JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " ";
40+
3641
ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID");
3742
if (string.IsNullOrWhiteSpace(ProjectId))
3843
{

‎storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License").
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
16+
using Google.Cloud.Storage.V1;
17+
using Google.Cloud.StorageTransfer.V1;
18+
using System;
19+
using Xunit;
20+
using Xunit.Abstractions;
21+
22+
namespace StorageTransfer.Samples.Tests;
23+
24+
[Collection(nameof(StorageFixture))]
25+
public class TransferToNearlineTest : IDisposable
26+
{
27+
private readonly StorageFixture _fixture;
28+
private string _transferJobName;
29+
private readonly ITestOutputHelper _outputHelper;
30+
public TransferToNearlineTest(StorageFixture fixture, ITestOutputHelper outputHelper)
31+
{
32+
_fixture = fixture;
33+
_outputHelper = outputHelper;
34+
}
35+
36+
[Fact]
37+
public void TestTransferToNearline()
38+
{
39+
TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(_outputHelper);
40+
var storage = StorageClient.Create();
41+
var bucket = storage.GetBucket(_fixture.BucketNameSink);
42+
string storageClass = StorageClasses.Nearline;
43+
bucket.StorageClass = storageClass;
44+
bucket = storage.UpdateBucket(bucket);
45+
var transferJob = transferToNearlineSample.TransferToNearline(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink);
46+
Assert.Contains("transferJobs/", transferJob.Name);
47+
_transferJobName = transferJob.Name;
48+
}
49+
50+
public void Dispose()
51+
{
52+
try
53+
{
54+
_fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest()
55+
{
56+
ProjectId = _fixture.ProjectId,
57+
JobName = _transferJobName,
58+
TransferJob = new TransferJob()
59+
{
60+
Name = _transferJobName,
61+
Status = TransferJob.Types.Status.Deleted
62+
}
63+
});
64+
}
65+
catch (Exception)
66+
{
67+
// Do nothing, we delete on a best effort basis.
68+
}
69+
}
70+
}

‎storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License").
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
// [START storagetransfer_get_latest_transfer_operation]
15+
16+
using Google.Cloud.StorageTransfer.V1;
17+
using System;
18+
using Xunit.Abstractions;
19+
20+
namespace StorageTransfer.Samples
21+
{
22+
public class CheckLatestTransferOperationSample
23+
{
24+
private readonly ITestOutputHelper _output;
25+
public CheckLatestTransferOperationSample(ITestOutputHelper output)
26+
{
27+
_output = output;
28+
29+
}
30+
//Checks the latest transfer operation for a given transfer job.
31+
public TransferJob CheckLatestTransferOperation(
32+
// Your Google Cloud Project ID
33+
string projectId = "my-project-id",
34+
// The name of the job to check
35+
string jobName = "transferJobs/1234567890")
36+
{
37+
if(string.IsNullOrEmpty(jobName))
38+
{
39+
throw new Exception("JobName can not be null or empty");
40+
}
41+
// Create a Transfer Service client
42+
StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.Create();
43+
44+
GetTransferJobRequest getTransferJobRequest = new GetTransferJobRequest { ProjectId = projectId, JobName = jobName };
45+
try
46+
{
47+
// Get Transfer job
48+
TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest);
49+
// Get Latest operation name from tranfer job
50+
string latestOperationName = transferJob.LatestOperationName;
51+
52+
53+
if (!string.IsNullOrEmpty(latestOperationName))
54+
{
55+
_output.WriteLine("The latest operation for transfer job " +jobName+ " is: " +latestOperationName+ "");
56+
}
57+
else
58+
{
59+
_output.WriteLine("Transfer job "+ jobName +" hasn't run yet, try again once after job started running.");
60+
}
61+
return transferJob;
62+
}
63+
catch (Exception)
64+
{
65+
throw new Exception("Failed to get transfer job "+ jobName + "");
66+
}
67+
}
68+
}
69+
}
70+
// [END storagetransfer_get_latest_transfer_operation]

‎storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs

+11-7
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
/**
2-
* Copyright 2021 Google Inc.
1+
/**
2+
* Copyright 2024 Google Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -14,15 +14,19 @@
1414
* limitations under the License.
1515
*/
1616

17-
1817
// [START storagetransfer_quickstart]
19-
using System;
2018
using Google.Cloud.StorageTransfer.V1;
19+
using Xunit.Abstractions;
2120

2221
namespace StorageTransfer.Samples
2322
{
2423
public class QuickstartSample
2524
{
25+
private readonly ITestOutputHelper _output;
26+
public QuickstartSample(ITestOutputHelper output)
27+
{
28+
_output = output;
29+
}
2630
public TransferJob Quickstart(
2731
// Your Google Cloud Project ID
2832
string projectId = "my-project-id",
@@ -36,8 +40,8 @@ public TransferJob Quickstart(
3640
ProjectId = projectId,
3741
TransferSpec = new TransferSpec
3842
{
39-
GcsDataSink = new GcsData { BucketName = sourceBucket },
40-
GcsDataSource = new GcsData { BucketName = sinkBucket }
43+
GcsDataSink = new GcsData { BucketName = sinkBucket },
44+
GcsDataSource = new GcsData { BucketName = sourceBucket }
4145
},
4246
Status = TransferJob.Types.Status.Enabled
4347
};
@@ -50,7 +54,7 @@ public TransferJob Quickstart(
5054
ProjectId = projectId
5155
});
5256

53-
Console.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}");
57+
_output.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}");
5458

5559
return response;
5660
}

‎storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,6 @@
66

77
<ItemGroup>
88
<PackageReference Include="Google.Cloud.StorageTransfer.V1" Version="2.7.0" />
9+
<PackageReference Include="xunit.abstractions" Version="2.0.3" />
910
</ItemGroup>
1011
</Project>

‎storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License").
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
// [START storagetransfer_transfer_to_nearline]
15+
16+
using Google.Cloud.StorageTransfer.V1;
17+
using Google.Protobuf.WellKnownTypes;
18+
using System;
19+
using Xunit.Abstractions;
20+
21+
22+
namespace StorageTransfer.Samples
23+
{
24+
25+
public class TransferToNearlineSample
26+
{
27+
/*Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more
28+
than 30 days old to a Nearline GCS bucket.*/
29+
private readonly ITestOutputHelper _output;
30+
public TransferToNearlineSample(ITestOutputHelper output)
31+
{
32+
_output = output;
33+
}
34+
public TransferJob TransferToNearline(
35+
// Your Google Cloud Project ID
36+
string projectId = "my-project-id",
37+
// The GCS bucket to transfer objects from
38+
string sourceBucket = "my-source-bucket",
39+
// The GCS Nearline bucket to transfer old objects to
40+
string sinkBucket = "my-sink-bucket")
41+
{
42+
// A description of this job
43+
string jobDescription = $"Transfers old objects from standard bucket ({sourceBucket}) that haven't been modified in the last 30 days to a Nearline bucket ({sinkBucket})";
44+
45+
TransferJob transferJob = new TransferJob
46+
{
47+
ProjectId = projectId,
48+
Description = jobDescription,
49+
TransferSpec = new TransferSpec
50+
{
51+
GcsDataSink = new GcsData { BucketName = sinkBucket },
52+
GcsDataSource = new GcsData { BucketName = sourceBucket },
53+
ObjectConditions = new ObjectConditions { MinTimeElapsedSinceLastModification = Duration.FromTimeSpan(TimeSpan.FromSeconds(2592000)) },
54+
TransferOptions = new TransferOptions { DeleteObjectsFromSourceAfterTransfer = true },
55+
},
56+
Status = TransferJob.Types.Status.Enabled,
57+
Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)), ScheduleEndDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) }
58+
};
59+
// Create a Transfer Service client
60+
StorageTransferServiceClient client = StorageTransferServiceClient.Create();
61+
// Create a Transfer job
62+
TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob });
63+
client.RunTransferJob(new RunTransferJobRequest
64+
{
65+
JobName = response.Name,
66+
ProjectId = projectId
67+
});
68+
69+
_output.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with the name {response.Name}");
70+
return response;
71+
}
72+
}
73+
}
74+
// [END storagetransfer_transfer_to_nearline]

0 commit comments

Comments
 (0)