From 0d8d24c45e57176320f06a79435fccc452c9fdf7 Mon Sep 17 00:00:00 2001 From: reloxx13 Date: Thu, 4 Feb 2021 00:18:38 +0100 Subject: [PATCH] weight sidequest categories, exclude app lab category, fix threadpool --- QuestAssetGenerator.py | 105 +++++++++++++++++++++++++++++++---------- 1 file changed, 80 insertions(+), 25 deletions(-) diff --git a/QuestAssetGenerator.py b/QuestAssetGenerator.py index 43921d3..b84b985 100644 --- a/QuestAssetGenerator.py +++ b/QuestAssetGenerator.py @@ -46,7 +46,7 @@ VRDB_CACHETIME = 24 # time in hours to refresh the cachefile SIDEQUEST_CACHETIME = 24 # time in hours to refresh the cachefile -IMGFETCHER_WORKERS = 5 +IMGFETCHER_WORKERS = 10 # Change the region # VRDB_URL = "https://vrdb.app/quest/index_us.json" @@ -58,6 +58,24 @@ # VRDB_URL = "https://vrdb.app/quest/index_gb.json" +CATEGORY_WEIGHTS = { + "Shooter": 9000, + "FPS": 5000, + "Music": 8000, + "Fitness": 7000, + "Combat": 6000, + "Horror": 5750, + "Escape": 5500, + "Puzzle": 5350, + "Adventure": 3000, + "Multiplayer": 100, + "Early Access": 95, + "Streaming": 80, + "All Games & Apps": 0, + } + + + def main(): parser = argparse.ArgumentParser(description='Quest Asset Generator - Genrefied') parser.add_argument('-a', '--access-token', help='GitHub access token') @@ -194,8 +212,11 @@ def get_sidequet_categories(): categories = demjson.decode(js_cats) + if len(categories) > 0: - with concurrent.futures.ThreadPoolExecutor(max_workers=IMGFETCHER_WORKERS) as executor: + + processes = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: for idx, category in enumerate(categories): # print(f"{idx} => {category}") print(f"Doing {idx} => {category['name']}...") @@ -203,12 +224,30 @@ def get_sidequet_categories(): # if not "tag" in category: # continue - res = [executor.submit(download_sidequest_assets, category, idx)] + processes.append(executor.submit(get_sidequest_caegory_Data, category, idx)) + # print(res) # concurrent.futures.wait(res) # APPNAMES_SIDEQUEST_DATA = APPNAMES_SIDEQUEST_DATA + appnames_sidequest_data_sub - concurrent.futures.wait(res, timeout=None, return_when=concurrent.futures.ALL_COMPLETED) + concurrent.futures.wait(processes, timeout=None, return_when=concurrent.futures.ALL_COMPLETED) + + for _ in concurrent.futures.as_completed(processes): + result = _.result() + # print('Result: ', result) + categories[result["idx"]]["count"] = result["count"] + weight = CATEGORY_WEIGHTS.get(categories[result["idx"]]["name"], result["count"]) + categories[result["idx"]]["weight"] = weight + categories[result["idx"]]["data"] = result["data"] + + + # Order categories by weight + categories = sorted(categories, key=lambda k: k['weight'], reverse=True) + # print(categories) + # exit(0) + + for idx, category in enumerate(categories): + download_sidequest_assets(category,idx) # print(APPNAMES_SIDEQUEST_DATA) @@ -216,7 +255,7 @@ def get_sidequet_categories(): # cache loaded app data print(f"write {APPNAMES_SIDEQUEST}") with open(get_cache_file(APPNAMES_SIDEQUEST), 'w', encoding='utf8') as outfile: - json.dump(APPNAMES_SIDEQUEST_DATA, outfile, indent=4, ensure_ascii=False) + json.dump(APPNAMES_SIDEQUEST_DATA, outfile, indent=4, ensure_ascii=False) # , sort_keys=True # Extract quest icons print(f"zip images {ICONPACK_SIDEQUEST}") @@ -228,10 +267,13 @@ def get_sidequet_categories(): -def download_sidequest_assets(category,cidx): - print("=== START download_sidequest_assets ===") - +def get_sidequest_caegory_Data(category,cidx): + print("=== START get_sidequest_caegory_Data ===") + result = { + "idx": cidx, + "count": 0, + } # curl "https://api.sidequestvr.com/search-apps # ?search=&page=0&order=rating&direction=desc&app_categories_id=1&tag=null&users_id=null&limit=51&device_filter=all&license_filter=all" # -H "Accept: application/json" -H "Accept-Language: de,en;q=0.7,en-US;q=0.3" @@ -299,18 +341,31 @@ def download_sidequest_assets(category,cidx): else: results = False - print(f"total results => {len(sidequest_data['data'])}") + print(f"Tag {tag} ({category['name']}) | TOTAL RESULTS => {len(sidequest_data['data'])}") + + result["count"] = len(sidequest_data['data']) + result["data"] = sidequest_data["data"] # print(sidequest_data) + print("=== END get_sidequest_caegory_Data ===") + + return result + +def download_sidequest_assets(category,cidx): + print("=== START download_sidequest_assets ===") + + + - if len(sidequest_data["data"]) > 0: + if category["count"] > 0: + processes = []; with concurrent.futures.ThreadPoolExecutor(max_workers=IMGFETCHER_WORKERS) as executor: - for idx, sidequest_entry in enumerate(sidequest_data["data"]): + for idx, sidequest_entry in enumerate(category["data"]): # print(f"{idx} => {sidequest_entry}") - print(f"Doing {cidx},{idx} => {category['name']} | {sidequest_entry['name']} | {sidequest_entry['packagename']}...") + print(f"Doing {cidx} | {idx} => {category['name']} | {sidequest_entry['name']} | {sidequest_entry['packagename']}...") catname = "SideQuest" - if category['name'] not in {"All Games & Apps", "Staff Picks"}: + if category['name'] not in {"All Games & Apps", "Staff Picks", "App Lab"}: catname = category['name'] if sidequest_entry["packagename"] in APPNAMES_SIDEQUEST_DATA: @@ -331,9 +386,9 @@ def download_sidequest_assets(category,cidx): } - res = [executor.submit(fetch_sidequest_images, sidequest_entry, idx)] + processes.append(executor.submit(fetch_sidequest_images, sidequest_entry, idx)) - concurrent.futures.wait(res, timeout=None, return_when=concurrent.futures.ALL_COMPLETED) + concurrent.futures.wait(processes, timeout=None, return_when=concurrent.futures.ALL_COMPLETED) # return APPNAMES_SIDEQUEST_DATA @@ -349,7 +404,7 @@ def fetch_sidequest_images(sidequest_entry, idx, force_banner=False): if img_file_age_in_hours is not False: - print(f"Cached img file {file_path} is {img_file_age_in_hours} h old") + # print(f"Cached img file {file_path} is {img_file_age_in_hours} h old") if img_file_age_in_hours <= SIDEQUEST_CACHETIME: return True @@ -363,7 +418,7 @@ def fetch_sidequest_images(sidequest_entry, idx, force_banner=False): # change_aspect_ratio(override_path,file_path_new) else: - image_url = sidequest_entry["image_url"]+"?size=705" + image_url = sidequest_entry["image_url"] if not image_url or image_url == "" or force_banner: print(f"{idx} => No image, use banner") image_url = sidequest_entry["app_banner"] @@ -373,7 +428,7 @@ def fetch_sidequest_images(sidequest_entry, idx, force_banner=False): else: print(f"{idx} => load image for {sidequest_entry['name']} | {sidequest_entry['packagename']} => {image_url}") - r = requests.get(image_url, allow_redirects=True, headers={'Accept': '*/*'}) + r = requests.get(image_url+"?size=705", allow_redirects=True, headers={'Accept': '*/*'}) # print(r.headers) # print(r.content) @@ -391,9 +446,9 @@ def fetch_sidequest_images(sidequest_entry, idx, force_banner=False): exit(1) content_type = r.headers['content-type'] - print(f"content_type => {content_type}") + # print(f"content_type => {content_type}") extension = mimetypes.guess_extension(content_type) - print(f"extension => {extension}") + # print(f"extension => {extension}") if not extension: print(f"{idx} => EXTENSION ERROR: {extension}") @@ -402,14 +457,14 @@ def fetch_sidequest_images(sidequest_entry, idx, force_banner=False): file_path = os.path.join(get_cache_file(SIDEQUEST_DIR), slugify(sidequest_entry["packagename"])) + extension - print("write") + # print("write") open(file_path, 'wb').write(r.content) - print("written") + # print("written") r.close() if extension != ".jpg": - print(f"{idx} => Convert {extension} to JPG => {slugify(sidequest_entry['packagename']) + extension}") + # print(f"{idx} => Convert {extension} to JPG => {slugify(sidequest_entry['packagename']) + extension}") try: file_path_new = os.path.join(get_cache_file(SIDEQUEST_DIR), slugify(sidequest_entry["packagename"])) + ".jpg" @@ -442,7 +497,7 @@ def optimize_image(file_path, file_path_new=""): if not file_path_new or file_path_new == "": file_path_new = file_path - print(f"Optimize: {file_path}") + # print(f"Optimize: {file_path}") try: # optimize images img = Image.open(file_path) @@ -618,7 +673,7 @@ def get_file_age_in_hours(file_path): if os.path.isfile(file_path): if os.path.getsize(file_path) != 0: app_file_age_in_hours = round(((int(time.time() - os.path.getmtime(file_path))) / 60 / 60), 2) - print(f"Cached file {file_path} is {app_file_age_in_hours}h old") + # print(f"Cached file {file_path} is {app_file_age_in_hours}h old") return app_file_age_in_hours def populate_genre():