From f19977cfba93888a7ccddbb741f6a096be24df9b Mon Sep 17 00:00:00 2001 From: Mizaki Date: Mon, 25 Dec 2023 22:43:21 +0000 Subject: [PATCH] Add limiter for image downloads --- gcd_talker/gcd.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/gcd_talker/gcd.py b/gcd_talker/gcd.py index ed51c24..31c4c5a 100644 --- a/gcd_talker/gcd.py +++ b/gcd_talker/gcd.py @@ -35,6 +35,7 @@ from comictalker.comiccacher import Issue as CCIssue from comictalker.comiccacher import Series as CCSeries from comictalker.comictalker import ComicTalker, TalkerDataError, TalkerNetworkError +from pyrate_limiter import Limiter, RequestRate from typing_extensions import TypedDict logger = logging.getLogger(f"comictalker.{__name__}") @@ -82,6 +83,9 @@ class GCDIssue(TypedDict, total=False): covers_downloaded: bool +limiter = Limiter(RequestRate(10, 10)) + + class GCDCredit(TypedDict): name: str gcd_role: str @@ -557,14 +561,15 @@ def _find_issue_images(self, issue_id: int) -> tuple[str, list[str]]: cover = "" variants = [] - try: - covers_html = requests.get(f"{self.website}/issue/{issue_id}/cover/4").text - except requests.exceptions.Timeout: - logger.debug(f"Connection to {self.website} timed out.") - raise TalkerNetworkError(self.website, 4) - except requests.exceptions.RequestException as e: - logger.debug(f"Request exception: {e}") - raise TalkerNetworkError(self.website, 0, str(e)) from e + with limiter.ratelimit("default", delay=True): + try: + covers_html = requests.get(f"{self.website}/issue/{issue_id}/cover/4").text + except requests.exceptions.Timeout: + logger.debug(f"Connection to {self.website} timed out.") + raise TalkerNetworkError(self.website, 4) + except requests.exceptions.RequestException as e: + logger.debug(f"Request exception: {e}") + raise TalkerNetworkError(self.website, 0, str(e)) from e covers_page = BeautifulSoup(covers_html, "html.parser")