Move retry to parameter, fix wait time flag

Author: Erik Johnson

bdfr/__main__.py

@@ -51,6 +51,7 @@
 _downloader_options = [
     click.option("--make-hard-links", is_flag=True, default=None),
     click.option("--max-wait-time", type=int, default=None),
+    click.option("--max-wait-retry", type=int, default=None),
     click.option("--no-dupes", is_flag=True, default=None),
     click.option("--search-existing", is_flag=True, default=None),
     click.option("--skip", default=None, multiple=True),

bdfr/archiver.py

@@ -31,7 +31,7 @@ def __init__(self, args: Configuration, logging_handlers: Iterable[logging.Handl
 
     def download(self):
         for generator in self.reddit_lists:
-            for retry in range(5):
+            for retry in range(self.args.max_wait_retry or 1):
                 try:
                     for submission in generator:
                         try:
@@ -53,8 +53,8 @@ def download(self):
                     break
                 except prawcore.PrawcoreException as e:
                     logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
-                    logger.debug("Waiting 60 seconds to continue")
-                    sleep(60)
+                    logger.debug(f"Waiting {self.args.max_wait_time} seconds to continue")
+                    sleep(self.args.max_wait_time)
 
     def get_submissions_from_link(self) -> list[list[praw.models.Submission]]:
         supplied_submissions = []

bdfr/cloner.py

@@ -20,7 +20,7 @@ def __init__(self, args: Configuration, logging_handlers: Iterable[logging.Handl
 
     def download(self):
         for generator in self.reddit_lists:
-            for retry in range(5):
+            for retry in range(self.args.max_wait_retry or 1):
                 try:
                     for submission in generator:
                         try:
@@ -31,5 +31,5 @@ def download(self):
                     break
                 except prawcore.PrawcoreException as e:
                     logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
-                    logger.debug("Waiting 60 seconds to continue")
-                    sleep(60)
+                    logger.debug(f"Waiting {self.args.max_wait_time} seconds to continue")
+                    sleep(self.args.max_wait_time)

bdfr/configuration.py

@@ -31,7 +31,8 @@ def __init__(self):
         self.link: list[str] = []
         self.log: Optional[str] = None
         self.make_hard_links = False
-        self.max_wait_time = None
+        self.max_wait_time = 60
+        self.max_wait_retry = 5
         self.multireddit: list[str] = []
         self.no_dupes: bool = False
         self.saved: bool = False

bdfr/connector.py

@@ -362,7 +362,7 @@ def get_user_data(self) -> list[Iterator]:
                 return []
             generators = []
             for user in self.args.user:
-                for retry in range(5):
+                for retry in range(self.args.max_wait_retry or 1):
                     try:
                         try:
                             self.check_user_existence(user)
@@ -388,8 +388,8 @@ def get_user_data(self) -> list[Iterator]:
                         break
                     except prawcore.PrawcoreException as e:
                         logger.error(f"User {user} failed to be retrieved due to a PRAW exception: {e}")
-                        logger.debug("Waiting 60 seconds to continue")
-                        sleep(60)
+                        logger.debug(f"Waiting {self.args.max_wait_time} seconds to continue")
+                        sleep(self.args.max_wait_time)
             return generators
         else:
             return []

bdfr/downloader.py

@@ -44,7 +44,7 @@ def __init__(self, args: Configuration, logging_handlers: Iterable[logging.Handl
 
     def download(self):
         for generator in self.reddit_lists:
-            for retry in range(5):
+            for retry in range(self.args.max_wait_retry or 1):
                 try:
                     for submission in generator:
                         try:
@@ -54,8 +54,8 @@ def download(self):
                     break
                 except prawcore.PrawcoreException as e:
                     logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
-                    logger.debug("Waiting 60 seconds to continue")
-                    sleep(60)
+                    logger.debug(f"Waiting {self.args.max_wait_time} seconds to continue")
+                    sleep(self.args.max_wait_time)
 
     def _download_submission(self, submission: praw.models.Submission):
         if submission.id in self.excluded_submission_ids: