Skip to content

Commit bf6c299

Browse files
evildourevildour
authored
Signed URL resumable upload support (#640)
* Signed URL resumable upload support * Derived uploader for signed URLs * Address PR feedback
1 parent 5322797 commit bf6c299

File tree

3 files changed

+278
-5
lines changed

3 files changed

+278
-5
lines changed

‎apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/UrlSignerTest.cs

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15+
using Google.Apis.Upload;
1516
using System;
17+
using System.Collections.Generic;
1618
using System.IO;
1719
using System.Linq;
1820
using System.Net;
@@ -476,5 +478,138 @@ private void PutWithCustomHeadersTest_InitDelayTest()
476478
Assert.Null(obj);
477479
});
478480
}
481+
482+
[Fact]
483+
public async Task ResumableUploadTest() => await _fixture.FinishDelayTest(GetTestName());
484+
485+
private void ResumableUploadTest_InitDelayTest()
486+
{
487+
var bucket = _fixture.SingleVersionBucket;
488+
var name = GenerateName();
489+
var data = _fixture.SmallContent;
490+
string url = null;
491+
492+
_fixture.RegisterDelayTest(_duration,
493+
beforeDelay: async duration =>
494+
{
495+
url = _fixture.UrlSigner.Sign(bucket, name, duration, UrlSigner.ResumableHttpMethod);
496+
497+
// Verify that the URL works initially.
498+
var uploader = SignedUrlResumableUpload.Create(url, new MemoryStream(data));
499+
var progress = await uploader.UploadAsync();
500+
Assert.Equal(UploadStatus.Completed, progress.Status);
501+
502+
var result = new MemoryStream();
503+
await _fixture.Client.DownloadObjectAsync(bucket, name, result);
504+
Assert.Equal(result.ToArray(), data);
505+
506+
// Reset the state.
507+
await _fixture.Client.DeleteObjectAsync(bucket, name);
508+
},
509+
afterDelay: async () =>
510+
{
511+
var uploader = SignedUrlResumableUpload.Create(url, new MemoryStream(data));
512+
513+
// Verify that the URL no longer works.
514+
var progress = await uploader.UploadAsync();
515+
Assert.Equal(UploadStatus.Failed, progress.Status);
516+
Assert.IsType(typeof(GoogleApiException), progress.Exception);
517+
518+
var obj = await _fixture.Client.ListObjectsAsync(bucket, name).FirstOrDefault(o => o.Name == name);
519+
Assert.Null(obj);
520+
});
521+
}
522+
523+
[Fact]
524+
public async Task ResumableUploadResumeTest() => await _fixture.FinishDelayTest(GetTestName());
525+
526+
private void ResumableUploadResumeTest_InitDelayTest()
527+
{
528+
var bucket = _fixture.SingleVersionBucket;
529+
var name = GenerateName();
530+
var data = _fixture.SmallContent;
531+
string url = null;
532+
533+
_fixture.RegisterDelayTest(_duration,
534+
beforeDelay: async duration =>
535+
{
536+
url = _fixture.UrlSigner.Sign(bucket, name, duration, UrlSigner.ResumableHttpMethod);
537+
var sessionUri = await SignedUrlResumableUpload.InitiateSessionAsync(url);
538+
539+
// Verify that the URL works initially.
540+
var uploader = ResumableUpload.CreateFromUploadUri(sessionUri, new MemoryStream(data));
541+
await uploader.ResumeAsync(sessionUri);
542+
var result = new MemoryStream();
543+
await _fixture.Client.DownloadObjectAsync(bucket, name, result);
544+
Assert.Equal(result.ToArray(), data);
545+
546+
// Reset the state.
547+
await _fixture.Client.DeleteObjectAsync(bucket, name);
548+
},
549+
afterDelay: async () =>
550+
{
551+
// Verify that the URL no longer works.
552+
await Assert.ThrowsAsync<GoogleApiException>(() => SignedUrlResumableUpload.InitiateSessionAsync(url));
553+
554+
var obj = await _fixture.Client.ListObjectsAsync(bucket, name).FirstOrDefault(o => o.Name == name);
555+
Assert.Null(obj);
556+
});
557+
}
558+
559+
[Fact]
560+
public async Task ResumableUploadWithCustomerSuppliedEncryptionKeysTest() => await _fixture.FinishDelayTest(GetTestName());
561+
562+
private void ResumableUploadWithCustomerSuppliedEncryptionKeysTest_InitDelayTest()
563+
{
564+
var bucket = _fixture.SingleVersionBucket;
565+
var name = GenerateName();
566+
var data = _fixture.SmallContent;
567+
string url = null;
568+
569+
EncryptionKey key = EncryptionKey.Generate();
570+
571+
_fixture.RegisterDelayTest(_duration,
572+
beforeDelay: async duration =>
573+
{
574+
url = _fixture.UrlSigner.Sign(
575+
bucket,
576+
name,
577+
duration,
578+
UrlSigner.ResumableHttpMethod,
579+
requestHeaders: new Dictionary<string, IEnumerable<string>> {
580+
{ "x-goog-encryption-algorithm", new [] { "AES256" } },
581+
{ "x-goog-encryption-key", new [] { key.Base64Key } },
582+
{ "x-goog-encryption-key-sha256", new []{ key.Base64Hash } }
583+
});
584+
585+
// Verify that the URL works initially.
586+
var uploader = SignedUrlResumableUpload.Create(
587+
url,
588+
new MemoryStream(data),
589+
new ResumableUploadOptions { ModifySessionInitiationRequest = key.ModifyRequest });
590+
var progress = await uploader.UploadAsync();
591+
Assert.Equal(UploadStatus.Completed, progress.Status);
592+
593+
// Make sure the encryption succeeded.
594+
var downloadedData = new MemoryStream();
595+
await Assert.ThrowsAsync<GoogleApiException>(
596+
() => _fixture.Client.DownloadObjectAsync(bucket, name, downloadedData));
597+
598+
await _fixture.Client.DownloadObjectAsync(bucket, name, downloadedData, new DownloadObjectOptions { EncryptionKey = key });
599+
Assert.Equal(data, downloadedData.ToArray());
600+
},
601+
afterDelay: async () =>
602+
{
603+
var uploader = SignedUrlResumableUpload.Create(
604+
url,
605+
new MemoryStream(data),
606+
new ResumableUploadOptions { ModifySessionInitiationRequest = key.ModifyRequest });
607+
608+
// Verify that the URL no longer works.
609+
var progress = await uploader.UploadAsync();
610+
Assert.Equal(UploadStatus.Failed, progress.Status);
611+
Assert.IsType(typeof(GoogleApiException), progress.Exception);
612+
});
613+
}
479614
}
480615
}

‎apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/SignedUrlResumableUpload.cs

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// Copyright 2017 Google Inc. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
using Google.Api.Gax;
16+
using Google.Apis.Upload;
17+
using System;
18+
using System.IO;
19+
using System.Linq;
20+
using System.Net.Http;
21+
using System.Threading.Tasks;
22+
using System.Threading;
23+
24+
namespace Google.Cloud.Storage.V1
25+
{
26+
/// <summary>
27+
/// Class which can perform a resumable upload using a signed URL to initiate the session.
28+
/// </summary>
29+
/// <seealso cref="UrlSigner"/>
30+
public sealed class SignedUrlResumableUpload : ResumableUpload
31+
{
32+
private string SignedUrl { get; set; }
33+
34+
private SignedUrlResumableUpload(string signedUrl, Stream contentStream, ResumableUploadOptions options)
35+
: base(contentStream, options)
36+
{
37+
SignedUrl = signedUrl;
38+
}
39+
40+
/// <summary>
41+
/// Creates a <see cref="SignedUrlResumableUpload"/> instance.
42+
/// </summary>
43+
/// <param name="signedUrl">
44+
/// The signed URL which can be used to initiate a resumable upload session. See
45+
/// <see cref="UrlSigner.ResumableHttpMethod">UrlSigner.ResumableHttpMethod</see> for more information.
46+
/// </param>
47+
/// <param name="contentStream">The data to be uploaded.</param>
48+
/// <param name="options">The options for the upload operation.</param>
49+
/// <returns>The instance which can be used to upload the specified content.</returns>
50+
public static SignedUrlResumableUpload Create(
51+
string signedUrl,
52+
Stream contentStream,
53+
ResumableUploadOptions options = null)
54+
{
55+
return new SignedUrlResumableUpload(
56+
GaxPreconditions.CheckNotNull(signedUrl, nameof(signedUrl)),
57+
contentStream,
58+
options);
59+
}
60+
61+
/// <inheritdoc/>
62+
protected override async Task<Uri> InitiateSessionAsync(CancellationToken cancellationToken)
63+
{
64+
var httpClient = Options?.HttpClient ?? new HttpClient();
65+
var request = new HttpRequestMessage(HttpMethod.Post, SignedUrl);
66+
request.Headers.Add("x-goog-resumable", "start");
67+
Options?.ModifySessionInitiationRequest?.Invoke(request);
68+
var result = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false);
69+
if (!result.IsSuccessStatusCode)
70+
{
71+
throw await ExceptionForResponseAsync(result).ConfigureAwait(false);
72+
}
73+
return result.Headers.Location;
74+
}
75+
76+
/// <summary>
77+
/// Initiates the resumable upload session by posting to the signed URL and returns the session URI.
78+
/// </summary>
79+
/// <param name="signedUrl">
80+
/// The signed URL which can be used to initiate a resumable upload session. See
81+
/// <see cref="UrlSigner.ResumableHttpMethod">UrlSigner.ResumableHttpMethod</see> for more information.
82+
/// </param>
83+
/// <param name="options">The options for the upload operation.</param>
84+
/// <returns>
85+
/// The session URI to use for the resumable upload.
86+
/// </returns>
87+
public static Uri InitiateSession(string signedUrl, ResumableUploadOptions options = null) =>
88+
InitiateSessionAsync(signedUrl, options).ResultWithUnwrappedExceptions();
89+
90+
/// <summary>
91+
/// Initiates the resumable upload session by posting to the signed URL and returns the session URI.
92+
/// </summary>
93+
/// <param name="signedUrl">
94+
/// The signed URL which can be used to initiate a resumable upload session. See
95+
/// <see cref="UrlSigner.ResumableHttpMethod">UrlSigner.ResumableHttpMethod</see> for more information.
96+
/// </param>
97+
/// <param name="options">The options for the upload operation.</param>
98+
/// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
99+
/// <returns>
100+
/// A task containing the session URI to use for the resumable upload.
101+
/// </returns>
102+
public static Task<Uri> InitiateSessionAsync(
103+
string signedUrl,
104+
ResumableUploadOptions options = null,
105+
CancellationToken cancellationToken = default(CancellationToken))
106+
{
107+
// We need an instance to call ExceptionForResponseAsync if the initiate request fails, so create a
108+
// temporary instance to initiate the session.
109+
var uploader = new SignedUrlResumableUpload(signedUrl, new MemoryStream(), options);
110+
return uploader.InitiateSessionAsync(cancellationToken);
111+
}
112+
}
113+
}

‎apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/UrlSigner.cs

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,19 @@
1414

1515
using Google.Api.Gax;
1616
using Google.Apis.Auth.OAuth2;
17-
using Google.Apis.Json;
1817
using System;
1918
using System.Collections.Generic;
2019
using System.Globalization;
2120
using System.IO;
2221
using System.Linq;
2322
using System.Net;
2423
using System.Net.Http;
25-
using System.Net.Http.Headers;
26-
using System.Security.Cryptography;
2724
using System.Text;
28-
using System.Threading.Tasks;
2925

3026
namespace Google.Cloud.Storage.V1
3127
{
28+
// TODO: Add unit tests for this
29+
3230
/// <summary>
3331
/// Class which helps create signed URLs which can be used to provide limited access to specific buckets and objects
3432
/// to anyone in possession of the URL, regardless of whether they have a Google account.
@@ -40,6 +38,18 @@ public sealed class UrlSigner
4038
{
4139
private const string GoogHeaderPrefix = "x-goog-";
4240
private const string StorageHost = "https://storage.googleapis.com";
41+
42+
/// <summary>
43+
/// Gets a special HTTP method which can be used to create a signed URL for initiating a resumable upload.
44+
/// See https://cloud.google.com/storage/docs/access-control/signed-urls#signing-resumable for more information.
45+
/// </summary>
46+
/// <remarks>
47+
/// Note: When using the RESUMABLE method to create a signed URL, a URL will actually be signed for the POST method with a header of
48+
/// 'x-goog-resumable:start'. The caller must perform a POST request with this URL and specify the 'x-goog-resumable:start' header as
49+
/// well or signature validation will fail.
50+
/// </remarks>
51+
public static HttpMethod ResumableHttpMethod { get; } = new HttpMethod("RESUMABLE");
52+
4353
private static readonly DateTimeOffset UnixEpoch = new DateTimeOffset(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc), TimeSpan.Zero);
4454

4555
private readonly ServiceAccountCredential _credentials;
@@ -289,20 +299,35 @@ public string Sign(
289299
{
290300
StorageClientImpl.ValidateBucketName(bucket);
291301

302+
bool isResumableUpload = false;
303+
if (requestMethod == null)
304+
{
305+
requestMethod = HttpMethod.Get;
306+
}
307+
else if (requestMethod == ResumableHttpMethod)
308+
{
309+
isResumableUpload = true;
310+
requestMethod = HttpMethod.Post;
311+
}
312+
292313
var expiryUnixSeconds = ((int?)((expiration - UnixEpoch)?.TotalSeconds))?.ToString(CultureInfo.InvariantCulture);
293314
var resourcePath = $"/{bucket}";
294315
if (objectName != null)
295316
{
296317
resourcePath += $"/{Uri.EscapeDataString(objectName)}";
297318
}
298319
var extensionHeaders = GetExtensionHeaders(requestHeaders, contentHeaders);
320+
if (isResumableUpload)
321+
{
322+
extensionHeaders["x-goog-resumable"] = new StringBuilder("start");
323+
}
299324

300325
var contentMD5 = GetFirstHeaderValue(contentHeaders, "Content-MD5");
301326
var contentType = GetFirstHeaderValue(contentHeaders, "Content-Type");
302327

303328
var signatureLines = new List<string>
304329
{
305-
(requestMethod ?? HttpMethod.Get).ToString(),
330+
requestMethod.ToString(),
306331
contentMD5,
307332
contentType,
308333
expiryUnixSeconds

0 commit comments

Comments
 (0)