Merge pull request #781 from OMEGARAZER/requests

Author: Serene-Arc

bdfr/__main__.py

@@ -77,12 +77,16 @@ def wrap(func):
     return wrap
 
 
-def _check_version(context, param, value):
+def _check_version(context, _param, value):
     if not value or context.resilient_parsing:
         return
     current = __version__
-    latest = requests.get("https://pypi.org/pypi/bdfr/json", timeout=10).json()["info"]["version"]
-    print(f"You are currently using v{current} the latest is v{latest}")
+    try:
+        latest = requests.get("https://pypi.org/pypi/bdfr/json", timeout=10).json()["info"]["version"]
+        print(f"You are currently using v{current} the latest is v{latest}")
+    except TimeoutError:
+        logger.exception(f"Timeout reached fetching current version from Pypi - BDFR v{current}")
+        raise
     context.exit()
 
 

bdfr/oauth2.py

@@ -24,11 +24,14 @@ def __init__(self, wanted_scopes: set[str], client_id: str, client_secret: str):
 
     @staticmethod
     def _check_scopes(wanted_scopes: set[str]):
-        response = requests.get(
-            "https://www.reddit.com/api/v1/scopes.json",
-            headers={"User-Agent": "fetch-scopes test"},
-            timeout=10,
-        )
+        try:
+            response = requests.get(
+                "https://www.reddit.com/api/v1/scopes.json",
+                headers={"User-Agent": "fetch-scopes test"},
+                timeout=10,
+            )
+        except TimeoutError:
+            raise BulkDownloaderException("Reached timeout fetching scopes")
         known_scopes = [scope for scope, data in response.json().items()]
         known_scopes.append("*")
         for scope in wanted_scopes:

bdfr/site_downloaders/base_downloader.py

@@ -31,8 +31,11 @@ def retrieve_url(url: str, cookies: dict = None, headers: dict = None) -> reques
         except requests.exceptions.RequestException as e:
             logger.exception(e)
             raise SiteDownloaderError(f"Failed to get page {url}")
+        except TimeoutError as e:
+            logger.exception(e)
+            raise SiteDownloaderError(f"Timeout reached attempting to get page {url}")
         if res.status_code != 200:
-            raise ResourceNotFound(f"Server responded with {res.status_code} to {url}")
+            raise ResourceNotFound(f"Server responded with {res.status_code} at {url}")
         return res
 
     @staticmethod
@@ -42,6 +45,21 @@ def post_url(url: str, cookies: dict = None, headers: dict = None, payload: dict
         except requests.exceptions.RequestException as e:
             logger.exception(e)
             raise SiteDownloaderError(f"Failed to post to {url}")
+        except TimeoutError as e:
+            logger.exception(e)
+            raise SiteDownloaderError(f"Timeout reached attempting to post to page {url}")
         if res.status_code != 200:
             raise ResourceNotFound(f"Server responded with {res.status_code} to {url}")
         return res
+
+    @staticmethod
+    def head_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response:
+        try:
+            res = requests.head(url, cookies=cookies, headers=headers, timeout=10)
+        except requests.exceptions.RequestException as e:
+            logger.exception(e)
+            raise SiteDownloaderError(f"Failed to check head at {url}")
+        except TimeoutError as e:
+            logger.exception(e)
+            raise SiteDownloaderError(f"Timeout reached attempting to check head at {url}")
+        return res

bdfr/site_downloaders/gallery.py

@@ -3,7 +3,6 @@
 import logging
 from typing import Optional
 
-import requests
 from praw.models import Submission
 
 from bdfr.exceptions import SiteDownloaderError
@@ -41,8 +40,7 @@ def _get_links(id_dict: list[dict]) -> list[str]:
             possible_extensions = (".jpg", ".png", ".gif", ".gifv", ".jpeg")
             for extension in possible_extensions:
                 test_url = f"https://i.redd.it/{image_id}{extension}"
-                response = requests.head(test_url, timeout=10)
-                if response.status_code == 200:
+                if Gallery.head_url(test_url).status_code == 200:
                     out.append(test_url)
                     break
         return out

bdfr/site_downloaders/redgifs.py

@@ -4,7 +4,6 @@
 import re
 from typing import Optional
 
-import requests
 from cachetools import TTLCache, cached
 from praw.models import Submission
 
@@ -67,7 +66,7 @@ def _get_link(url: str) -> set[str]:
         out = set()
         try:
             if response_json["gif"]["type"] == 1:  # type 1 is a video
-                if requests.head(response_json["gif"]["urls"]["hd"], headers=headers, timeout=10).ok:
+                if Redgifs.head_url(response_json["gif"]["urls"]["hd"], headers=headers).status_code == 200:
                     out.add(response_json["gif"]["urls"]["hd"])
                 else:
                     out.add(response_json["gif"]["urls"]["sd"])

bdfr/site_downloaders/vidble.py

@@ -6,7 +6,6 @@
 from typing import Optional
 
 import bs4
-import requests
 from praw.models import Submission
 
 from bdfr.exceptions import SiteDownloaderError
@@ -36,7 +35,7 @@ def get_links(url: str) -> set[str]:
         if not re.search(r"vidble.com/(show/|album/|watch\?v)", url):
             url = re.sub(r"/(\w*?)$", r"/show/\1", url)
 
-        page = requests.get(url, timeout=10)
+        page = Vidble.retrieve_url(url)
         soup = bs4.BeautifulSoup(page.text, "html.parser")
         content_div = soup.find("div", attrs={"id": "ContentPlaceHolder1_divContent"})
         images = content_div.find_all("img")