Skip to content

Commit f4b73e0

Browse files
mahendra-googlemahendra-google
authored
samples(storage transfer): transfer from POSIX file system to GCS bucket using manifest file (#2)
2 parents 398b3d5 + 3c9d243 commit f4b73e0

File tree

3 files changed

+171
-5
lines changed

3 files changed

+171
-5
lines changed

‎storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs

+18-5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/**
2-
* Copyright 2021 Google Inc.
2+
* Copyright 2024 Google Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -29,16 +29,19 @@ public class StorageFixture : IDisposable, ICollectionFixture<StorageFixture>
2929
public string BucketNameSource { get; } = Guid.NewGuid().ToString();
3030
public string BucketNameSink { get; } = Guid.NewGuid().ToString();
3131
public string JobName { get; }
32+
public string SourceAgentPoolName { get; }
33+
public string RootDirectory { get; } = "/tmp/uploads";
3234
public StorageClient Storage { get; } = StorageClient.Create();
35+
public string ManifestObjectName { get; } = "manifest.csv";
3336
public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create();
3437

3538
public StorageFixture()
3639
{
3740
// Instantiate random number generator
3841
Random random = new Random();
39-
JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " ";
40-
42+
JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " ";
4143
ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID");
44+
SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet";
4245
if (string.IsNullOrWhiteSpace(ProjectId))
4346
{
4447
throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id.");
@@ -102,15 +105,25 @@ public void Dispose()
102105
}
103106
catch (Exception)
104107
{
105-
// Do nothing, we delete on a best effort basis.
108+
// If bucket is not empty, we delete on a best effort basis.
109+
foreach (var storageObject in Storage.ListObjects(BucketNameSink, ""))
110+
{
111+
Storage.DeleteObject(BucketNameSink, storageObject.Name);
112+
}
113+
Storage.DeleteBucket(BucketNameSink);
106114
}
107115
try
108116
{
109117
Storage.DeleteBucket(BucketNameSource);
110118
}
111119
catch (Exception)
112120
{
113-
// Do nothing, we delete on a best effort basis.
121+
// If bucket is not empty, we delete on a best effort basis.
122+
foreach (var storageObject in Storage.ListObjects(BucketNameSource, ""))
123+
{
124+
Storage.DeleteObject(BucketNameSource, storageObject.Name);
125+
}
126+
Storage.DeleteBucket(BucketNameSource);
114127
}
115128
}
116129
}

‎storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License").
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
using Google.Cloud.Storage.V1;
16+
using Google.Cloud.StorageTransfer.V1;
17+
using System;
18+
using System.Text;
19+
using Xunit.Abstractions;
20+
using Xunit;
21+
using System.IO;
22+
23+
namespace StorageTransfer.Samples.Tests;
24+
[Collection(nameof(StorageFixture))]
25+
public class TransferUsingManifestTest : IDisposable
26+
{
27+
private readonly StorageFixture _fixture;
28+
private string _transferJobName;
29+
private readonly ITestOutputHelper _outputHelper;
30+
public TransferUsingManifestTest(StorageFixture fixture, ITestOutputHelper outputHelper)
31+
{
32+
_fixture = fixture;
33+
_outputHelper = outputHelper;
34+
}
35+
36+
[Fact]
37+
public void TransferUsingManifest()
38+
{
39+
TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(_outputHelper);
40+
var storage = StorageClient.Create();
41+
byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg");
42+
MemoryStream stream = new MemoryStream(byteArray);
43+
storage.UploadObject(_fixture.BucketNameSource, _fixture.ManifestObjectName, "application/octet-stream", stream);
44+
var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.ManifestObjectName);
45+
Assert.Contains("transferJobs/", transferJob.Name);
46+
_transferJobName = transferJob.Name;
47+
}
48+
49+
public void Dispose()
50+
{
51+
try
52+
{
53+
_fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest()
54+
{
55+
ProjectId = _fixture.ProjectId,
56+
JobName = _transferJobName,
57+
TransferJob = new TransferJob()
58+
{
59+
Name = _transferJobName,
60+
Status = TransferJob.Types.Status.Deleted
61+
}
62+
});
63+
}
64+
catch (Exception)
65+
{
66+
// Do nothing, we delete on a best effort basis.
67+
}
68+
}
69+
}

‎storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License").
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
// [START storagetransfer_manifest_request]
15+
using Google.Cloud.StorageTransfer.V1;
16+
using Xunit.Abstractions;
17+
18+
19+
namespace StorageTransfer.Samples
20+
{
21+
public class TransferUsingManifestSample
22+
{
23+
/*Create a transfer from a POSIX file system to a GCS bucket using
24+
a manifest file*/
25+
private readonly ITestOutputHelper _output;
26+
public TransferUsingManifestSample(ITestOutputHelper output)
27+
{
28+
_output = output;
29+
}
30+
public TransferJob TransferUsingManifest(
31+
// Your Google Cloud Project ID
32+
string projectId = "my-project-id",
33+
// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent
34+
string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default",
35+
// The root directory path on the source filesystem
36+
string rootDirectory = "/tmp/uploads",
37+
// The GCS bucket which has your manifest file
38+
string manifestBucket = "my-source-bucket",
39+
// The GCS bucket to transfer data to
40+
string sinkBucket = "my-sink-bucket",
41+
// The name of the manifest file in manifestBucket that specifies which objects to transfer
42+
string manifestObjectName = "path/to/manifest.csv")
43+
{
44+
string manifestLocation = "gs://" + manifestBucket + "/" + manifestObjectName;
45+
46+
// # A useful description for your transfer job
47+
string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file";
48+
49+
TransferJob transferJob = new TransferJob
50+
{
51+
ProjectId = projectId,
52+
Description = jobDescription,
53+
TransferSpec = new TransferSpec
54+
{
55+
GcsDataSink = new GcsData { BucketName = sinkBucket },
56+
GcsDataSource = new GcsData { BucketName = manifestBucket },
57+
SourceAgentPoolName = sourceAgentPoolName,
58+
PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory },
59+
TransferManifest = new TransferManifest { Location = manifestLocation }
60+
},
61+
Status = TransferJob.Types.Status.Enabled,
62+
};
63+
64+
65+
// Create a Transfer Service client
66+
StorageTransferServiceClient client = StorageTransferServiceClient.Create();
67+
68+
// Create a Transfer job
69+
TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob });
70+
71+
client.RunTransferJob(new RunTransferJobRequest
72+
{
73+
JobName = response.Name,
74+
ProjectId = projectId
75+
});
76+
77+
_output.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}");
78+
return response;
79+
80+
81+
}
82+
}
83+
}
84+
// [END storagetransfer_manifest_request]

0 commit comments

Comments
 (0)