Add limiter for image downloads

comictagger · Dec 25, 2023 · f19977c · f19977c
1 parent b625275
commit f19977c
Showing 1 changed file with 13 additions and 8 deletions.
diff --git a/gcd_talker/gcd.py b/gcd_talker/gcd.py
@@ -35,6 +35,7 @@
 from comictalker.comiccacher import Issue as CCIssue
 from comictalker.comiccacher import Series as CCSeries
 from comictalker.comictalker import ComicTalker, TalkerDataError, TalkerNetworkError
+from pyrate_limiter import Limiter, RequestRate
 from typing_extensions import TypedDict
 
 logger = logging.getLogger(f"comictalker.{__name__}")
@@ -82,6 +83,9 @@ class GCDIssue(TypedDict, total=False):
     covers_downloaded: bool
 
 
+limiter = Limiter(RequestRate(10, 10))
+
+
 class GCDCredit(TypedDict):
     name: str
     gcd_role: str
@@ -557,14 +561,15 @@ def _find_issue_images(self, issue_id: int) -> tuple[str, list[str]]:
         cover = ""
         variants = []
 
-        try:
-            covers_html = requests.get(f"{self.website}/issue/{issue_id}/cover/4").text
-        except requests.exceptions.Timeout:
-            logger.debug(f"Connection to {self.website} timed out.")
-            raise TalkerNetworkError(self.website, 4)
-        except requests.exceptions.RequestException as e:
-            logger.debug(f"Request exception: {e}")
-            raise TalkerNetworkError(self.website, 0, str(e)) from e
+        with limiter.ratelimit("default", delay=True):
+            try:
+                covers_html = requests.get(f"{self.website}/issue/{issue_id}/cover/4").text
+            except requests.exceptions.Timeout:
+                logger.debug(f"Connection to {self.website} timed out.")
+                raise TalkerNetworkError(self.website, 4)
+            except requests.exceptions.RequestException as e:
+                logger.debug(f"Request exception: {e}")
+                raise TalkerNetworkError(self.website, 0, str(e)) from e
 
         covers_page = BeautifulSoup(covers_html, "html.parser")