Skip to content

Commit cf20966

Browse files
andrewmathew1Andrew Mathew
and
Andrew Mathew
authored
Added samples for timeout and retries (#39867)
* added samples for timeout and retries * added changes for separate throttle and connection retry kwargs * added test_retry_policy_async and fixed pylint issue for cosmos clients * marked test_retry_policy_async as cosmosQuery * added in documentation for new throttle and config retries setup * fixed async tests to use with client instead of asyncSetUp * changing annotation to cosmosEmulator to cosmosQuery for testing * made mockExecute functions async, removed part of test that was never supposed to be hit * edited timeout and retries config * changed marking to cosmosEmulator to see if it fails ci tests * changed mf.counter to 1 to match sync test * reset connectionRetryConfiguration in service_retry_policies_async * created separate connection policy for async test * edited test_default_retry_policy_for_create_async to use create_container_if_not_exists * edited spacing --------- Co-authored-by: Andrew Mathew <[email protected]>
1 parent b39715e commit cf20966

File tree

7 files changed

+807
-18
lines changed

7 files changed

+807
-18
lines changed

sdk/cosmos/azure-cosmos/azure/cosmos/aio/_cosmos_client.py

+8-4
Original file line numberDiff line numberDiff line change
@@ -98,11 +98,15 @@ def _build_connection_policy(kwargs: Dict[str, Any]) -> ConnectionPolicy:
9898
# Retry config
9999
retry_options = policy.RetryOptions
100100
total_retries = kwargs.pop('retry_total', None)
101-
retry_options._max_retry_attempt_count = total_retries or retry_options._max_retry_attempt_count
101+
total_throttle_retries = kwargs.pop('retry_throttle_total', None)
102+
retry_options._max_retry_attempt_count = \
103+
total_throttle_retries or total_retries or retry_options._max_retry_attempt_count
102104
retry_options._fixed_retry_interval_in_milliseconds = \
103105
kwargs.pop('retry_fixed_interval', retry_options._fixed_retry_interval_in_milliseconds)
104-
max_backoff = kwargs.pop('retry_backoff_max', policy.MaxBackoff)
105-
retry_options._max_wait_time_in_seconds = max_backoff or retry_options._max_wait_time_in_seconds
106+
max_backoff = kwargs.pop('retry_backoff_max', None)
107+
max_throttle_backoff = kwargs.pop('retry_throttle_backoff_max', None)
108+
retry_options._max_wait_time_in_seconds = \
109+
max_throttle_backoff or max_backoff or retry_options._max_wait_time_in_seconds
106110
policy.RetryOptions = retry_options
107111
connection_retry = policy.ConnectionRetryConfiguration
108112
if not connection_retry:
@@ -111,7 +115,7 @@ def _build_connection_policy(kwargs: Dict[str, Any]) -> ConnectionPolicy:
111115
retry_connect=kwargs.pop('retry_connect', None),
112116
retry_read=kwargs.pop('retry_read', None),
113117
retry_status=kwargs.pop('retry_status', None),
114-
retry_backoff_max=max_backoff,
118+
retry_backoff_max=max_backoff or retry_options._max_wait_time_in_seconds,
115119
retry_mode=kwargs.pop('retry_mode', RetryMode.Fixed),
116120
retry_on_status_codes=kwargs.pop('retry_on_status_codes', []),
117121
retry_backoff_factor=kwargs.pop('retry_backoff_factor', 1),

sdk/cosmos/azure-cosmos/azure/cosmos/cosmos_client.py

+8-4
Original file line numberDiff line numberDiff line change
@@ -114,11 +114,15 @@ def _build_connection_policy(kwargs: Dict[str, Any]) -> ConnectionPolicy:
114114
)
115115
retry_options = policy.RetryOptions
116116
total_retries = kwargs.pop('retry_total', None)
117-
retry_options._max_retry_attempt_count = total_retries or retry_options._max_retry_attempt_count
117+
total_throttle_retries = kwargs.pop('retry_throttle_total', None)
118+
retry_options._max_retry_attempt_count = \
119+
total_throttle_retries or total_retries or retry_options._max_retry_attempt_count
118120
retry_options._fixed_retry_interval_in_milliseconds = kwargs.pop('retry_fixed_interval', None) or \
119121
retry_options._fixed_retry_interval_in_milliseconds
120-
max_backoff = kwargs.pop('retry_backoff_max', policy.MaxBackoff)
121-
retry_options._max_wait_time_in_seconds = max_backoff or retry_options._max_wait_time_in_seconds
122+
max_backoff = kwargs.pop('retry_backoff_max', None)
123+
max_throttle_backoff = kwargs.pop('retry_throttle_backoff_max', None)
124+
retry_options._max_wait_time_in_seconds = \
125+
max_throttle_backoff or max_backoff or retry_options._max_wait_time_in_seconds
122126
policy.RetryOptions = retry_options
123127
connection_retry = kwargs.pop('connection_retry_policy', None)
124128
if connection_retry is not None:
@@ -132,7 +136,7 @@ def _build_connection_policy(kwargs: Dict[str, Any]) -> ConnectionPolicy:
132136
retry_connect=kwargs.pop('retry_connect', None),
133137
retry_read=kwargs.pop('retry_read', None),
134138
retry_status=kwargs.pop('retry_status', None),
135-
retry_backoff_max=max_backoff,
139+
retry_backoff_max=max_backoff or retry_options._max_wait_time_in_seconds,
136140
retry_mode=kwargs.pop('retry_mode', RetryMode.Fixed),
137141
retry_on_status_codes=kwargs.pop('retry_on_status_codes', []),
138142
retry_backoff_factor=kwargs.pop('retry_backoff_factor', 1),

sdk/cosmos/azure-cosmos/docs/TimeoutAndRetriesConfig.md

+65-4
Original file line numberDiff line numberDiff line change
@@ -11,23 +11,84 @@ connecting to the socket. Default value is 5s.
1111
- `Read Timeout`: can be changed through `read_timeout` option. Changes the value on the client's http transport timeout when
1212
reading the service buffer stream, or receiving responses from the server. Default value is 65s.
1313

14+
These options can also be combined, as seen in the example below.
15+
This will set the client timeout to 10 seconds, connection timeout to 3 seconds, and read timeout to 60 seconds.
16+
```python
17+
from azure.cosmos import CosmosClient
18+
19+
import os
20+
URL = os.environ['ACCOUNT_URI']
21+
KEY = os.environ['ACCOUNT_KEY']
22+
23+
client = CosmosClient(URL, credential=KEY, timeout=10, connection_timeout=3, read_timeout=60)
24+
```
25+
1426

1527
### Resource Throttle Retry Policy config
1628

1729
The options for the resource throttle retry policy used for 429 error codes can be changed from the default client configurations with the options below. These are:
18-
- `Retry Total`: Represents the total amount of retries. Can be changed by passing the `retry_total` option. Default value is 9.
30+
- `Retry Throttle Total`: Represents the total amount of retries. Can be changed by passing the `retry_throttle_total` option. Default value is 9.
1931
- `Retry Fixed Interval`: Represents a fixed wait period between retry attempts in seconds. Can be changed by passing the
2032
`retry_fixed_interval` option (`float`). The default behaviour is to use an exponential retry policy. Must be a value >= 1.
21-
- `Retry Backoff Max`: Represents the maximum retry wait time. Can be changed by passing the `retry_backoff_max` option. Default value is 30s.
33+
- `Retry Throttle Backoff Max`: Represents the maximum retry wait time. Can be changed by passing the `retry_throttle_backoff_max` option. Default value is 30s.
2234

35+
Previously, `retry_total` and `retry_backoff_max` were used to set both the throttle and connection retry policy configurations.
36+
This option is backwards compatible though, the option remains to set both throttle and connection retry configurations with the same `retry_total` and `retry_backoff_max` value.
37+
38+
In the example below the total number of throttle retries is 5, the fixed interval is set to 3 seconds, and the maximum throttle retry backoff wait time is 15 seconds.
39+
This leaves the connection retry policies set to their default values.
40+
41+
```python
42+
from azure.cosmos import CosmosClient
43+
44+
import os
45+
URL = os.environ['ACCOUNT_URI']
46+
KEY = os.environ['ACCOUNT_KEY']
47+
48+
client = CosmosClient(URL, credential=KEY, retry_throttle_total=5, retry_fixed_interval=3.0, retry_throttle_backoff_max=15)
49+
```
2350

2451
### Connection Retry config
2552

2653
The retry options below can be changed from the default client configurations. These are:
54+
- `Retry Total`: Represents the total amount of retries. Can be changed by passing the `retry_total` option. Default value is 9.
2755
- `Retry Connect`: Maximum number of connection error retry attempts. Can be changed by passing the `retry_connect` option. Default value is 3.
2856
- `Retry Read`: Maximum number of socket read retry attempts. Can be changed by passing the `retry_read` option. Default value is 3.
29-
- `Retry Status`: Maximum number of retry attempts on error status codes. Can be changed by passing the `retry_status` option. Default value is 3.
30-
- `Retry On Status Codes`: A list of specific status codes to retry on. Can be changed by passing the `retry_on_status_codes` option. Default value is an empty list.
3157
- `Retry Backoff Factor`: Factor to calculate wait time between retry attempts. Can be changed by passing the `retry_backoff_factor` option. Default value is 1.
58+
- `Retry Backoff Max`: Represents the maximum retry wait time. Can be changed by passing the `retry_backoff_max` option. Default value is 30s.
59+
60+
61+
In the example below, the maximum total retries is 7 attempts, the connection error retry is 5 attempts, and the socket read retry is 4 attempts.
62+
The retry backoff factor is set to 2. When the retry backoff factor is 2, it means the wait time for retries will increase exponentially (1s, 2s, 4s, 8s, 16s...).
63+
Lastly, the retry backoff max is 20, meaning that the maximum wait time between retries will be 20 seconds.
64+
```python
65+
from azure.cosmos import CosmosClient
66+
67+
import os
68+
URL = os.environ['ACCOUNT_URI']
69+
KEY = os.environ['ACCOUNT_KEY']
70+
71+
client = CosmosClient(URL, credential=KEY, retry_total = 7, retry_connect=5, retry_read=4, retry_backoff_factor=2, retry_backoff_max=20)
72+
```
73+
Additionally, in the above example the throttle retry total will be 7, and throttle max backoff will be 20 seconds.
74+
75+
76+
To configure throttle and connection retries separately, both sets of parameters would need to be passed in as shown in the example below.
77+
In this example, the connection retry backoff max time is 35 seconds, while setting the throttle backoff max to 25 seconds.
78+
Additionally, the total throttle retry attempts is 5, while the total connection retry attempts is 10.
79+
80+
```python
81+
from azure.cosmos import CosmosClient
82+
83+
import os
84+
URL = os.environ['ACCOUNT_URI']
85+
KEY = os.environ['ACCOUNT_KEY']
86+
87+
client = CosmosClient(URL, credential=KEY,
88+
retry_backoff_max=35,
89+
retry_throttle_backoff_max=25,
90+
retry_throttle_total=5,
91+
retry_total=10)
92+
```
3293

3394
More information on the SDK's default retry behaviors can be found in our error codes and retries [document](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/cosmos/azure-cosmos/docs/ErrorCodesAndRetries.md).

sdk/cosmos/azure-cosmos/tests/test_crud.py

-1
Original file line numberDiff line numberDiff line change
@@ -1848,7 +1848,6 @@ def test_client_request_timeout_when_connection_retry_configuration_specified(se
18481848
databaseForTest = client.get_database_client(self.configs.TEST_DATABASE_ID)
18491849
container = databaseForTest.get_container_client(self.configs.TEST_SINGLE_PARTITION_CONTAINER_ID)
18501850
container.create_item(body={'id': str(uuid.uuid4()), 'name': 'sample'})
1851-
print('Async initialization')
18521851

18531852
# TODO: Skipping this test to debug later
18541853
@unittest.skip

0 commit comments

Comments
 (0)