Signed URL resumable upload support

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/UrlSignerTest.cs

@@ -12,7 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+using Google.Apis.Upload;
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net;
@@ -476,5 +478,138 @@ private void PutWithCustomHeadersTest_InitDelayTest()
                     Assert.Null(obj);
                 });
         }
+
+        [Fact]
+        public async Task ResumableUploadTest() => await _fixture.FinishDelayTest(GetTestName());
+
+        private void ResumableUploadTest_InitDelayTest()
+        {
+            var bucket = _fixture.SingleVersionBucket;
+            var name = GenerateName();
+            var data = _fixture.SmallContent;
+            string url = null;
+
+            _fixture.RegisterDelayTest(_duration,
+                beforeDelay: async duration =>
+                {
+                    url = _fixture.UrlSigner.Sign(bucket, name, duration, UrlSigner.ResumableHttpMethod);
+
+                    // Verify that the URL works initially.
+                    var uploader = SignedUrlResumableUpload.Create(url, new MemoryStream(data));
+                    var progress = await uploader.UploadAsync();
+                    Assert.Equal(UploadStatus.Completed, progress.Status);
+
+                    var result = new MemoryStream();
+                    await _fixture.Client.DownloadObjectAsync(bucket, name, result);
+                    Assert.Equal(result.ToArray(), data);
+
+                    // Reset the state.
+                    await _fixture.Client.DeleteObjectAsync(bucket, name);
+                },
+                afterDelay: async () =>
+                {
+                    var uploader = SignedUrlResumableUpload.Create(url, new MemoryStream(data));
+
+                    // Verify that the URL no longer works.
+                    var progress = await uploader.UploadAsync();
+                    Assert.Equal(UploadStatus.Failed, progress.Status);
+                    Assert.IsType(typeof(GoogleApiException), progress.Exception);
+
+                    var obj = await _fixture.Client.ListObjectsAsync(bucket, name).FirstOrDefault(o => o.Name == name);
+                    Assert.Null(obj);
+                });
+        }
+
+        [Fact]
+        public async Task ResumableUploadResumeTest() => await _fixture.FinishDelayTest(GetTestName());
+
+        private void ResumableUploadResumeTest_InitDelayTest()
+        {
+            var bucket = _fixture.SingleVersionBucket;
+            var name = GenerateName();
+            var data = _fixture.SmallContent;
+            string url = null;
+
+            _fixture.RegisterDelayTest(_duration,
+                beforeDelay: async duration =>
+                {
+                    url = _fixture.UrlSigner.Sign(bucket, name, duration, UrlSigner.ResumableHttpMethod);
+                    var sessionUri = await SignedUrlResumableUpload.InitiateSessionAsync(url);
+
+                    // Verify that the URL works initially.
+                    var uploader = ResumableUpload.CreateFromUploadUri(sessionUri, new MemoryStream(data));
+                    await uploader.ResumeAsync(sessionUri);
+                    var result = new MemoryStream();
+                    await _fixture.Client.DownloadObjectAsync(bucket, name, result);
+                    Assert.Equal(result.ToArray(), data);
+
+                    // Reset the state.
+                    await _fixture.Client.DeleteObjectAsync(bucket, name);
+                },
+                afterDelay: async () =>
+                {
+                    // Verify that the URL no longer works.
+                    await Assert.ThrowsAsync<GoogleApiException>(() => SignedUrlResumableUpload.InitiateSessionAsync(url));
+
+                    var obj = await _fixture.Client.ListObjectsAsync(bucket, name).FirstOrDefault(o => o.Name == name);
+                    Assert.Null(obj);
+                });
+        }
+
+        [Fact]
+        public async Task ResumableUploadWithCustomerSuppliedEncryptionKeysTest() => await _fixture.FinishDelayTest(GetTestName());
+
+        private void ResumableUploadWithCustomerSuppliedEncryptionKeysTest_InitDelayTest()
+        {
+            var bucket = _fixture.SingleVersionBucket;
+            var name = GenerateName();
+            var data = _fixture.SmallContent;
+            string url = null;
+
+            EncryptionKey key = EncryptionKey.Generate();
+
+            _fixture.RegisterDelayTest(_duration,
+                beforeDelay: async duration =>
+                {
+                    url = _fixture.UrlSigner.Sign(
+                        bucket,
+                        name,
+                        duration,
+                        UrlSigner.ResumableHttpMethod,
+                        requestHeaders: new Dictionary<string, IEnumerable<string>> {
+                            { "x-goog-encryption-algorithm", new [] { "AES256" } },
+                            { "x-goog-encryption-key", new [] { key.Base64Key } },
+                            { "x-goog-encryption-key-sha256", new []{ key.Base64Hash } }
+                        });
+
+                    // Verify that the URL works initially.
+                    var uploader = SignedUrlResumableUpload.Create(
+                        url,
+                        new MemoryStream(data),
+                        new ResumableUploadOptions { ModifySessionInitiationRequest = key.ModifyRequest });
+                    var progress = await uploader.UploadAsync();
+                    Assert.Equal(UploadStatus.Completed, progress.Status);
+
+                    // Make sure the encryption succeeded.
+                    var downloadedData = new MemoryStream();
+                    await Assert.ThrowsAsync<GoogleApiException>(
+                        () => _fixture.Client.DownloadObjectAsync(bucket, name, downloadedData));
+
+                    await _fixture.Client.DownloadObjectAsync(bucket, name, downloadedData, new DownloadObjectOptions { EncryptionKey = key });
+                    Assert.Equal(data, downloadedData.ToArray());
+                },
+                afterDelay: async () =>
+                {
+                    var uploader = SignedUrlResumableUpload.Create(
+                        url,
+                        new MemoryStream(data),
+                        new ResumableUploadOptions { ModifySessionInitiationRequest = key.ModifyRequest });
+
+                    // Verify that the URL no longer works.
+                    var progress = await uploader.UploadAsync();
+                    Assert.Equal(UploadStatus.Failed, progress.Status);
+                    Assert.IsType(typeof(GoogleApiException), progress.Exception);
+                });
+        }
     }
 }

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/SignedUrlResumableUpload.cs

@@ -0,0 +1,113 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using Google.Api.Gax;
+using Google.Apis.Upload;
+using System;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Threading.Tasks;
+using System.Threading;
+
+namespace Google.Cloud.Storage.V1
+{
+    /// <summary>
+    /// Class which can perform a resumable upload using a signed URL to initiate the session.
+    /// </summary>
+    /// <seealso cref="UrlSigner"/>
+    public sealed class SignedUrlResumableUpload : ResumableUpload
+    {
+        private string SignedUrl { get; set; }
+
+        private SignedUrlResumableUpload(string signedUrl, Stream contentStream, ResumableUploadOptions options)
+            : base(contentStream, options)
+        {
+            SignedUrl = signedUrl;
+        }
+
+        /// <summary>
+        /// Creates a <see cref="SignedUrlResumableUpload"/> instance.
+        /// </summary>
+        /// <param name="signedUrl">
+        /// The signed URL which can be used to initiate a resumable upload session. See
+        /// <see cref="UrlSigner.ResumableHttpMethod">UrlSigner.ResumableHttpMethod</see> for more information.
+        /// </param>
+        /// <param name="contentStream">The data to be uploaded.</param>
+        /// <param name="options">The options for the upload operation.</param>
+        /// <returns>The instance which can be used to upload the specified content.</returns>
+        public static SignedUrlResumableUpload Create(
+            string signedUrl,
+            Stream contentStream,
+            ResumableUploadOptions options = null)
+        {
+            return new SignedUrlResumableUpload(
+                GaxPreconditions.CheckNotNull(signedUrl, nameof(signedUrl)),
+                contentStream,
+                options);
+        }
+
+        /// <inheritdoc/>
+        protected override async Task<Uri> InitiateSessionAsync(CancellationToken cancellationToken)
+        {
+            var httpClient = Options?.HttpClient ?? new HttpClient();
+            var request = new HttpRequestMessage(HttpMethod.Post, SignedUrl);
+            request.Headers.Add("x-goog-resumable", "start");
+            Options?.ModifySessionInitiationRequest?.Invoke(request);
+            var result = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false);
+            if (!result.IsSuccessStatusCode)
+            {
+                throw await ExceptionForResponseAsync(result).ConfigureAwait(false);
+            }
+            return result.Headers.Location;
+        }
+
+        /// <summary>
+        /// Initiates the resumable upload session by posting to the signed URL and returns the session URI.
+        /// </summary>
+        /// <param name="signedUrl">
+        /// The signed URL which can be used to initiate a resumable upload session. See
+        /// <see cref="UrlSigner.ResumableHttpMethod">UrlSigner.ResumableHttpMethod</see> for more information.
+        /// </param>
+        /// <param name="options">The options for the upload operation.</param>
+        /// <returns>
+        /// The session URI to use for the resumable upload.
+        /// </returns>
+        public static Uri InitiateSession(string signedUrl, ResumableUploadOptions options = null) =>
+            InitiateSessionAsync(signedUrl, options).ResultWithUnwrappedExceptions();
+
+        /// <summary>
+        /// Initiates the resumable upload session by posting to the signed URL and returns the session URI.
+        /// </summary>
+        /// <param name="signedUrl">
+        /// The signed URL which can be used to initiate a resumable upload session. See
+        /// <see cref="UrlSigner.ResumableHttpMethod">UrlSigner.ResumableHttpMethod</see> for more information.
+        /// </param>
+        /// <param name="options">The options for the upload operation.</param>
+        /// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
+        /// <returns>
+        /// A task containing the session URI to use for the resumable upload.
+        /// </returns>
+        public static Task<Uri> InitiateSessionAsync(
+            string signedUrl,
+            ResumableUploadOptions options = null,
+            CancellationToken cancellationToken = default(CancellationToken))
+        {
+            // We need an instance to call ExceptionForResponseAsync if the initiate request fails, so create a
+            // temporary instance to initiate the session.
+            var uploader = new SignedUrlResumableUpload(signedUrl, new MemoryStream(), options);
+            return uploader.InitiateSessionAsync(cancellationToken);
+        }
+    }
+}

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/UrlSigner.cs

@@ -14,21 +14,19 @@
 
 using Google.Api.Gax;
 using Google.Apis.Auth.OAuth2;
-using Google.Apis.Json;
 using System;
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
-using System.Net.Http.Headers;
-using System.Security.Cryptography;
 using System.Text;
-using System.Threading.Tasks;
 
 namespace Google.Cloud.Storage.V1
 {
+    // TODO: Add unit tests for this
+
     /// <summary>
     /// Class which helps create signed URLs which can be used to provide limited access to specific buckets and objects
     /// to anyone in possession of the URL, regardless of whether they have a Google account.
@@ -40,6 +38,18 @@ public sealed class UrlSigner
     {
         private const string GoogHeaderPrefix = "x-goog-";
         private const string StorageHost = "https://storage.googleapis.com";
+
+        /// <summary>
+        /// Gets a special HTTP method which can be used to create a signed URL for initiating a resumable upload.
+        /// See https://cloud.google.com/storage/docs/access-control/signed-urls#signing-resumable for more information.
+        /// </summary>
+        /// <remarks>
+        /// Note: When using the RESUMABLE method to create a signed URL, a URL will actually be signed for the POST method with a header of
+        /// 'x-goog-resumable:start'. The caller must perform a POST request with this URL and specify the 'x-goog-resumable:start' header as
+        /// well or signature validation will fail.
+        /// </remarks>
+        public static HttpMethod ResumableHttpMethod { get; } = new HttpMethod("RESUMABLE");
+
         private static readonly DateTimeOffset UnixEpoch = new DateTimeOffset(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc), TimeSpan.Zero);
 
         private readonly ServiceAccountCredential _credentials;
@@ -289,20 +299,35 @@ public string Sign(
         {
             StorageClientImpl.ValidateBucketName(bucket);
 
+            bool isResumableUpload = false;
+            if (requestMethod == null)
+            {
+                requestMethod = HttpMethod.Get;
+            }
+            else if (requestMethod == ResumableHttpMethod)
+            {
+                isResumableUpload = true;
+                requestMethod = HttpMethod.Post;
+            }
+
             var expiryUnixSeconds = ((int?)((expiration - UnixEpoch)?.TotalSeconds))?.ToString(CultureInfo.InvariantCulture);
             var resourcePath = $"/{bucket}";
             if (objectName != null)
             {
                 resourcePath += $"/{Uri.EscapeDataString(objectName)}";
             }
             var extensionHeaders = GetExtensionHeaders(requestHeaders, contentHeaders);
+            if (isResumableUpload)
+            {
+                extensionHeaders["x-goog-resumable"] = new StringBuilder("start");
+            }
 
             var contentMD5 = GetFirstHeaderValue(contentHeaders, "Content-MD5");
             var contentType = GetFirstHeaderValue(contentHeaders, "Content-Type");
 
             var signatureLines = new List<string>
             {
-                (requestMethod ?? HttpMethod.Get).ToString(),
+                requestMethod.ToString(),
                 contentMD5,
                 contentType,
                 expiryUnixSeconds