From d80a4cf5a3d88fe77149e0fda1dffe669c52dabd Mon Sep 17 00:00:00 2001 From: reloxx13 Date: Tue, 9 May 2023 08:03:38 +0200 Subject: [PATCH] add retrier for cloudflare protection "too many requests" while crawling images from sq --- QuestAssetGenerator.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/QuestAssetGenerator.py b/QuestAssetGenerator.py index 71782eb..5021490 100644 --- a/QuestAssetGenerator.py +++ b/QuestAssetGenerator.py @@ -552,7 +552,23 @@ def fetch_sidequest_images(sidequest_entry, idx, force_banner=False): print( f"{idx} => load image for {sidequest_entry['name']} | {sidequest_entry['packagename']} => {image_url}") - r = requests.get(image_url + "?size=705", allow_redirects=True, headers={'Accept': '*/*'}) + + + for i in range(5): + r = requests.get(image_url + "?size=705", allow_redirects=True, headers={'Accept': '*/*'}) + if r.status_code == 200: + print('fetch_sidequest_images => Request successful') + break + else: + print(f'ERROR fetch_sidequest_images => Request failed with status code {response.status_code}: {r.reason}. Retrying...') + # if r.status_code != 200: + # print("ERROR fetch_sidequest_images => Status code was not 200 ") + # print(r.status_code) + # print(r.headers) + # print(r.content) + # print(r.reason) + # print(r.text) + # r.close() # print(r.headers) # print(r.content) @@ -561,13 +577,6 @@ def fetch_sidequest_images(sidequest_entry, idx, force_banner=False): # r.close() # exit(1) - if r.status_code != 200: - print(r.headers) - print(r.content) - print(r.reason) - print(r.text) - r.close() - exit(1) content_type = r.headers['content-type'] # print(f"content_type => {content_type}")