Skip to content

Commit 5cab892

Browse files
authored
[Streaming] retry on requests errors (#6963)
* [Streaming] retry on requests errors * lucain's comment
1 parent 83d2860 commit 5cab892

File tree

1 file changed

+8
-3
lines changed

1 file changed

+8
-3
lines changed

src/datasets/utils/file_utils.py

+8-3
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
Copyright by the AllenNLP authors.
55
"""
66

7+
import asyncio
78
import copy
89
import glob
910
import io
@@ -20,7 +21,6 @@
2021
import warnings
2122
import xml.dom.minidom
2223
import zipfile
23-
from asyncio import TimeoutError
2424
from contextlib import closing, contextmanager
2525
from functools import partial
2626
from io import BytesIO
@@ -31,10 +31,10 @@
3131
from urllib.parse import urljoin, urlparse
3232
from xml.etree import ElementTree as ET
3333

34+
import aiohttp.client_exceptions
3435
import fsspec
3536
import huggingface_hub
3637
import requests
37-
from aiohttp.client_exceptions import ClientError
3838
from fsspec.core import strip_protocol, url_to_fs
3939
from fsspec.utils import can_be_local
4040
from huggingface_hub.utils import EntryNotFoundError, insecure_hashlib
@@ -1093,7 +1093,12 @@ def read_with_retries(*args, **kwargs):
10931093
try:
10941094
out = read(*args, **kwargs)
10951095
break
1096-
except (ClientError, TimeoutError) as err:
1096+
except (
1097+
aiohttp.client_exceptions.ClientError,
1098+
asyncio.TimeoutError,
1099+
requests.exceptions.ConnectTimeout,
1100+
requests.exceptions.ConnectionError,
1101+
) as err:
10971102
disconnect_err = err
10981103
logger.warning(
10991104
f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_RETRY_INTERVAL}sec [{retry}/{max_retries}]"

0 commit comments

Comments
 (0)