Skip to content

Commit

Permalink
weight sidequest categories, exclude app lab category,
Browse files Browse the repository at this point in the history 
fix threadpool
  • Loading branch information
@reloxx13
reloxx13 committed Feb 3, 2021
1 parent f3f4ac5 commit 0d8d24c
Showing 1 changed file with 80 additions and 25 deletions.
105 changes: 80 additions & 25 deletions QuestAssetGenerator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
VRDB_CACHETIME = 24 # time in hours to refresh the cachefile
SIDEQUEST_CACHETIME = 24 # time in hours to refresh the cachefile

IMGFETCHER_WORKERS = 5
IMGFETCHER_WORKERS = 10

# Change the region
# VRDB_URL = "https://vrdb.app/quest/index_us.json"
Expand All @@ -58,6 +58,24 @@
# VRDB_URL = "https://vrdb.app/quest/index_gb.json"


CATEGORY_WEIGHTS = {
"Shooter": 9000,
"FPS": 5000,
"Music": 8000,
"Fitness": 7000,
"Combat": 6000,
"Horror": 5750,
"Escape": 5500,
"Puzzle": 5350,
"Adventure": 3000,
"Multiplayer": 100,
"Early Access": 95,
"Streaming": 80,
"All Games & Apps": 0,
}



def main():
parser = argparse.ArgumentParser(description='Quest Asset Generator - Genrefied')
parser.add_argument('-a', '--access-token', help='GitHub access token')
Expand Down Expand Up @@ -194,29 +212,50 @@ def get_sidequet_categories():
categories = demjson.decode(js_cats)



if len(categories) > 0:
with concurrent.futures.ThreadPoolExecutor(max_workers=IMGFETCHER_WORKERS) as executor:

processes = []
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
for idx, category in enumerate(categories):
# print(f"{idx} => {category}")
print(f"Doing {idx} => {category['name']}...")

# if not "tag" in category:
# continue

res = [executor.submit(download_sidequest_assets, category, idx)]
processes.append(executor.submit(get_sidequest_caegory_Data, category, idx))

# print(res)
# concurrent.futures.wait(res)
# APPNAMES_SIDEQUEST_DATA = APPNAMES_SIDEQUEST_DATA + appnames_sidequest_data_sub

concurrent.futures.wait(res, timeout=None, return_when=concurrent.futures.ALL_COMPLETED)
concurrent.futures.wait(processes, timeout=None, return_when=concurrent.futures.ALL_COMPLETED)

for _ in concurrent.futures.as_completed(processes):
result = _.result()
# print('Result: ', result)
categories[result["idx"]]["count"] = result["count"]
weight = CATEGORY_WEIGHTS.get(categories[result["idx"]]["name"], result["count"])
categories[result["idx"]]["weight"] = weight
categories[result["idx"]]["data"] = result["data"]


# Order categories by weight
categories = sorted(categories, key=lambda k: k['weight'], reverse=True)
# print(categories)
# exit(0)

for idx, category in enumerate(categories):
download_sidequest_assets(category,idx)


# print(APPNAMES_SIDEQUEST_DATA)

# cache loaded app data
print(f"write {APPNAMES_SIDEQUEST}")
with open(get_cache_file(APPNAMES_SIDEQUEST), 'w', encoding='utf8') as outfile:
json.dump(APPNAMES_SIDEQUEST_DATA, outfile, indent=4, ensure_ascii=False)
json.dump(APPNAMES_SIDEQUEST_DATA, outfile, indent=4, ensure_ascii=False) # , sort_keys=True

# Extract quest icons
print(f"zip images {ICONPACK_SIDEQUEST}")
Expand All @@ -228,10 +267,13 @@ def get_sidequet_categories():



def download_sidequest_assets(category,cidx):
print("=== START download_sidequest_assets ===")

def get_sidequest_caegory_Data(category,cidx):
print("=== START get_sidequest_caegory_Data ===")

result = {
"idx": cidx,
"count": 0,
}
# curl "https://api.sidequestvr.com/search-apps
# ?search=&page=0&order=rating&direction=desc&app_categories_id=1&tag=null&users_id=null&limit=51&device_filter=all&license_filter=all"
# -H "Accept: application/json" -H "Accept-Language: de,en;q=0.7,en-US;q=0.3"
Expand Down Expand Up @@ -299,18 +341,31 @@ def download_sidequest_assets(category,cidx):
else:
results = False

print(f"total results => {len(sidequest_data['data'])}")
print(f"Tag {tag} ({category['name']}) | TOTAL RESULTS => {len(sidequest_data['data'])}")

result["count"] = len(sidequest_data['data'])
result["data"] = sidequest_data["data"]
# print(sidequest_data)

print("=== END get_sidequest_caegory_Data ===")

return result

def download_sidequest_assets(category,cidx):
print("=== START download_sidequest_assets ===")




if len(sidequest_data["data"]) > 0:
if category["count"] > 0:
processes = [];
with concurrent.futures.ThreadPoolExecutor(max_workers=IMGFETCHER_WORKERS) as executor:
for idx, sidequest_entry in enumerate(sidequest_data["data"]):
for idx, sidequest_entry in enumerate(category["data"]):
# print(f"{idx} => {sidequest_entry}")
print(f"Doing {cidx},{idx} => {category['name']} | {sidequest_entry['name']} | {sidequest_entry['packagename']}...")
print(f"Doing {cidx} | {idx} => {category['name']} | {sidequest_entry['name']} | {sidequest_entry['packagename']}...")

catname = "SideQuest"
if category['name'] not in {"All Games & Apps", "Staff Picks"}:
if category['name'] not in {"All Games & Apps", "Staff Picks", "App Lab"}:
catname = category['name']

if sidequest_entry["packagename"] in APPNAMES_SIDEQUEST_DATA:
Expand All @@ -331,9 +386,9 @@ def download_sidequest_assets(category,cidx):
}


res = [executor.submit(fetch_sidequest_images, sidequest_entry, idx)]
processes.append(executor.submit(fetch_sidequest_images, sidequest_entry, idx))

concurrent.futures.wait(res, timeout=None, return_when=concurrent.futures.ALL_COMPLETED)
concurrent.futures.wait(processes, timeout=None, return_when=concurrent.futures.ALL_COMPLETED)

# return APPNAMES_SIDEQUEST_DATA

Expand All @@ -349,7 +404,7 @@ def fetch_sidequest_images(sidequest_entry, idx, force_banner=False):


if img_file_age_in_hours is not False:
print(f"Cached img file {file_path} is {img_file_age_in_hours} h old")
# print(f"Cached img file {file_path} is {img_file_age_in_hours} h old")
if img_file_age_in_hours <= SIDEQUEST_CACHETIME:
return True

Expand All @@ -363,7 +418,7 @@ def fetch_sidequest_images(sidequest_entry, idx, force_banner=False):
# change_aspect_ratio(override_path,file_path_new)

else:
image_url = sidequest_entry["image_url"]+"?size=705"
image_url = sidequest_entry["image_url"]
if not image_url or image_url == "" or force_banner:
print(f"{idx} => No image, use banner")
image_url = sidequest_entry["app_banner"]
Expand All @@ -373,7 +428,7 @@ def fetch_sidequest_images(sidequest_entry, idx, force_banner=False):
else:
print(f"{idx} => load image for {sidequest_entry['name']} | {sidequest_entry['packagename']} => {image_url}")

r = requests.get(image_url, allow_redirects=True, headers={'Accept': '*/*'})
r = requests.get(image_url+"?size=705", allow_redirects=True, headers={'Accept': '*/*'})

# print(r.headers)
# print(r.content)
Expand All @@ -391,9 +446,9 @@ def fetch_sidequest_images(sidequest_entry, idx, force_banner=False):
exit(1)

content_type = r.headers['content-type']
print(f"content_type => {content_type}")
# print(f"content_type => {content_type}")
extension = mimetypes.guess_extension(content_type)
print(f"extension => {extension}")
# print(f"extension => {extension}")

if not extension:
print(f"{idx} => EXTENSION ERROR: {extension}")
Expand All @@ -402,14 +457,14 @@ def fetch_sidequest_images(sidequest_entry, idx, force_banner=False):

file_path = os.path.join(get_cache_file(SIDEQUEST_DIR), slugify(sidequest_entry["packagename"])) + extension

print("write")
# print("write")
open(file_path, 'wb').write(r.content)
print("written")
# print("written")

r.close()

if extension != ".jpg":
print(f"{idx} => Convert {extension} to JPG => {slugify(sidequest_entry['packagename']) + extension}")
# print(f"{idx} => Convert {extension} to JPG => {slugify(sidequest_entry['packagename']) + extension}")
try:
file_path_new = os.path.join(get_cache_file(SIDEQUEST_DIR),
slugify(sidequest_entry["packagename"])) + ".jpg"
Expand Down Expand Up @@ -442,7 +497,7 @@ def optimize_image(file_path, file_path_new=""):
if not file_path_new or file_path_new == "":
file_path_new = file_path

print(f"Optimize: {file_path}")
# print(f"Optimize: {file_path}")
try:
# optimize images
img = Image.open(file_path)
Expand Down Expand Up @@ -618,7 +673,7 @@ def get_file_age_in_hours(file_path):
if os.path.isfile(file_path):
if os.path.getsize(file_path) != 0:
app_file_age_in_hours = round(((int(time.time() - os.path.getmtime(file_path))) / 60 / 60), 2)
print(f"Cached file {file_path} is {app_file_age_in_hours}h old")
# print(f"Cached file {file_path} is {app_file_age_in_hours}h old")
return app_file_age_in_hours

def populate_genre():
Expand Down

0 comments on commit 0d8d24c

Please sign in to comment.