Skip to content

Commit 8c1ae3c

Browse files
committed
add retry configuration to blob.create_resumable_upload_session
1 parent b57b863 commit 8c1ae3c

File tree

2 files changed

+103
-4
lines changed

2 files changed

+103
-4
lines changed

google/cloud/storage/blob.py

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2782,6 +2782,11 @@ def create_resumable_upload_session(
27822782
client=None,
27832783
timeout=_DEFAULT_TIMEOUT,
27842784
checksum=None,
2785+
if_generation_match=None,
2786+
if_generation_not_match=None,
2787+
if_metageneration_match=None,
2788+
if_metageneration_not_match=None,
2789+
retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,
27852790
):
27862791
"""Create a resumable upload session.
27872792
@@ -2857,6 +2862,41 @@ def create_resumable_upload_session(
28572862
delete the uploaded object automatically. Supported values
28582863
are "md5", "crc32c" and None. The default is None.
28592864
2865+
:type if_generation_match: long
2866+
:param if_generation_match:
2867+
(Optional) See :ref:`using-if-generation-match`
2868+
2869+
:type if_generation_not_match: long
2870+
:param if_generation_not_match:
2871+
(Optional) See :ref:`using-if-generation-not-match`
2872+
2873+
:type if_metageneration_match: long
2874+
:param if_metageneration_match:
2875+
(Optional) See :ref:`using-if-metageneration-match`
2876+
2877+
:type if_metageneration_not_match: long
2878+
:param if_metageneration_not_match:
2879+
(Optional) See :ref:`using-if-metageneration-not-match`
2880+
2881+
:type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
2882+
:param retry: (Optional) How to retry the RPC. A None value will disable
2883+
retries. A google.api_core.retry.Retry value will enable retries,
2884+
and the object will define retriable response codes and errors and
2885+
configure backoff and timeout options.
2886+
A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
2887+
Retry object and activates it only if certain conditions are met.
2888+
This class exists to provide safe defaults for RPC calls that are
2889+
not technically safe to retry normally (due to potential data
2890+
duplication or other side-effects) but become safe to retry if a
2891+
condition such as if_metageneration_match is set.
2892+
See the retry.py source code and docstrings in this package
2893+
(google.cloud.storage.retry) for information on retry types and how
2894+
to configure them.
2895+
Media operations (downloads and uploads) do not support non-default
2896+
predicates in a Retry object. The default will always be used. Other
2897+
configuration changes for Retry objects such as delays and deadlines
2898+
are respected.
2899+
28602900
:rtype: str
28612901
:returns: The resumable upload session URL. The upload can be
28622902
completed by making an HTTP PUT request with the
@@ -2865,6 +2905,19 @@ def create_resumable_upload_session(
28652905
:raises: :class:`google.cloud.exceptions.GoogleCloudError`
28662906
if the session creation response returns an error status.
28672907
"""
2908+
2909+
# Handle ConditionalRetryPolicy.
2910+
if isinstance(retry, ConditionalRetryPolicy):
2911+
# Conditional retries are designed for non-media calls, which change
2912+
# arguments into query_params dictionaries. Media operations work
2913+
# differently, so here we make a "fake" query_params to feed to the
2914+
# ConditionalRetryPolicy.
2915+
query_params = {
2916+
"ifGenerationMatch": if_generation_match,
2917+
"ifMetagenerationMatch": if_metageneration_match,
2918+
}
2919+
retry = retry.get_retry_policy_if_conditions_met(query_params=query_params)
2920+
28682921
extra_headers = {}
28692922
if origin is not None:
28702923
# This header is specifically for client-side uploads, it
@@ -2883,10 +2936,15 @@ def create_resumable_upload_session(
28832936
size,
28842937
None,
28852938
predefined_acl=None,
2939+
if_generation_match=if_generation_match,
2940+
if_generation_not_match=if_generation_not_match,
2941+
if_metageneration_match=if_metageneration_match,
2942+
if_metageneration_not_match=if_metageneration_not_match,
28862943
extra_headers=extra_headers,
28872944
chunk_size=self._CHUNK_SIZE_MULTIPLE,
28882945
timeout=timeout,
28892946
checksum=checksum,
2947+
retry=retry,
28902948
)
28912949

28922950
return upload.resumable_url

tests/unit/test_blob.py

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3249,8 +3249,18 @@ def test_upload_from_string_w_text_w_num_retries(self):
32493249
self._upload_from_string_helper(data, num_retries=2)
32503250

32513251
def _create_resumable_upload_session_helper(
3252-
self, origin=None, side_effect=None, timeout=None
3252+
self,
3253+
origin=None,
3254+
side_effect=None,
3255+
timeout=None,
3256+
if_generation_match=None,
3257+
if_generation_not_match=None,
3258+
if_metageneration_match=None,
3259+
if_metageneration_not_match=None,
3260+
retry=None,
32533261
):
3262+
from six.moves.urllib.parse import urlencode
3263+
32543264
bucket = _Bucket(name="alex-trebek")
32553265
blob = self._make_one("blob-name", bucket=bucket)
32563266
chunk_size = 99 * blob._CHUNK_SIZE_MULTIPLE
@@ -3276,11 +3286,19 @@ def _create_resumable_upload_session_helper(
32763286
expected_timeout = timeout
32773287
timeout_kwarg = {"timeout": timeout}
32783288

3289+
if retry is DEFAULT_RETRY_IF_GENERATION_SPECIFIED:
3290+
retry = DEFAULT_RETRY if if_generation_match else None
3291+
32793292
new_url = blob.create_resumable_upload_session(
32803293
content_type=content_type,
32813294
size=size,
32823295
origin=origin,
32833296
client=client,
3297+
if_generation_match=if_generation_match,
3298+
if_generation_not_match=if_generation_not_match,
3299+
if_metageneration_match=if_metageneration_match,
3300+
if_metageneration_not_match=if_metageneration_not_match,
3301+
retry=retry,
32843302
**timeout_kwarg
32853303
)
32863304

@@ -3290,10 +3308,23 @@ def _create_resumable_upload_session_helper(
32903308

32913309
# Check the mocks.
32923310
upload_url = (
3293-
"https://storage.googleapis.com/upload/storage/v1"
3294-
+ bucket.path
3295-
+ "/o?uploadType=resumable"
3311+
"https://storage.googleapis.com/upload/storage/v1" + bucket.path + "/o"
32963312
)
3313+
3314+
qs_params = [("uploadType", "resumable")]
3315+
if if_generation_match is not None:
3316+
qs_params.append(("ifGenerationMatch", if_generation_match))
3317+
3318+
if if_generation_not_match is not None:
3319+
qs_params.append(("ifGenerationNotMatch", if_generation_not_match))
3320+
3321+
if if_metageneration_match is not None:
3322+
qs_params.append(("ifMetagenerationMatch", if_metageneration_match))
3323+
3324+
if if_metageneration_not_match is not None:
3325+
qs_params.append(("ifMetaGenerationNotMatch", if_metageneration_not_match))
3326+
3327+
upload_url += "?" + urlencode(qs_params)
32973328
payload = b'{"name": "blob-name"}'
32983329
expected_headers = {
32993330
"content-type": "application/json; charset=UTF-8",
@@ -3319,6 +3350,16 @@ def test_create_resumable_upload_session_with_custom_timeout(self):
33193350
def test_create_resumable_upload_session_with_origin(self):
33203351
self._create_resumable_upload_session_helper(origin="http://google.com")
33213352

3353+
def test_create_resumable_upload_session_with_conditional_retry_success(self):
3354+
self._create_resumable_upload_session_helper(
3355+
retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, if_generation_match=1
3356+
)
3357+
3358+
def test_create_resumable_upload_session_with_conditional_retry_failure(self):
3359+
self._create_resumable_upload_session_helper(
3360+
retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED
3361+
)
3362+
33223363
def test_create_resumable_upload_session_with_failure(self):
33233364
from google.resumable_media import InvalidResponse
33243365
from google.cloud import exceptions

0 commit comments

Comments
 (0)