diff --git a/scraper/api.py b/scraper/api.py index 30d38f8..6f950bb 100644 --- a/scraper/api.py +++ b/scraper/api.py @@ -1,4 +1,6 @@ +import functools import json +import time from typing import List, Optional import requests @@ -12,6 +14,32 @@ config = init_config() base_url = config.app.api_base_url +# decorator to retry operations +def retry(delay=1, times=5): + def outer_wrapper(function): + @functools.wraps(function) + def inner_wrapper(*args, **kwargs): + final_excep = None + for counter in range(times): + if counter > 0: + time.sleep(delay) + final_excep = None + try: + value = function(*args, **kwargs) + return value + except Exception as e: + final_excep = e + logger.warning( + f"Error during function call, count={counter}: error:{e}" + ) + + if final_excep is not None: + raise final_excep + + return inner_wrapper + + return outer_wrapper + def get_latest_posts(count: int) -> List[AnalogDisplayPost]: @@ -69,6 +97,7 @@ def upload_to_analogdb(post: AnalogPost, username: str, password: str): ) +@retry(delay=1, times=5) def patch_to_analogdb(patch: PatchPost, id: int, username: str, password: str): dict_patch = patch_to_json(patch) json_patch = json.dumps(dict_patch) @@ -176,21 +205,6 @@ def json_to_post(data: dict) -> AnalogDisplayPost: raw_url=images[3]["url"], raw_width=images[3]["width"], raw_height=images[3]["height"], - c1_hex=colors[0]["hex"], - c1_css=colors[0]["css"], - c1_percent=colors[0]["percent"], - c2_hex=colors[0]["hex"], - c2_css=colors[0]["css"], - c2_percent=colors[0]["percent"], - c3_hex=colors[0]["hex"], - c3_css=colors[0]["css"], - c3_percent=colors[0]["percent"], - c4_hex=colors[0]["hex"], - c4_css=colors[0]["css"], - c4_percent=colors[0]["percent"], - c5_hex=colors[0]["hex"], - c5_css=colors[0]["css"], - c5_percent=colors[0]["percent"], ) except Exception as e: raise Exception(f"Error unmarshalling json posts from analogdb: {e}") diff --git a/scraper/batch.py b/scraper/batch.py index 479efb5..ea56da0 100644 --- a/scraper/batch.py +++ b/scraper/batch.py @@ -101,9 +101,14 @@ def _update_post_colors( # extract primary colors try: image = request_image(url=url) + except Exception as e: + logger.error(f"Error fetching image with url: {url}, with error: {e}") + return + + try: colors = extract_colors(image) except Exception as e: - logger.error(f"Error fetching iamge with url: {url}, with error: {e}") + logger.error(f"Error extracting colors from image: {url}, with error: {e}") return # update post in analogdb @@ -115,7 +120,7 @@ def _update_post_colors( def update_posts_colors(deps: Dependencies, count: int): - posts = unlimited_posts(count=count) + posts = reversed(unlimited_posts(count=count)) for post in posts: _update_post_colors( reddit=deps.reddit_client, diff --git a/scraper/image_process.py b/scraper/image_process.py index 1845101..61c40c7 100644 --- a/scraper/image_process.py +++ b/scraper/image_process.py @@ -1,19 +1,95 @@ from io import BytesIO -from typing import List, Optional, Tuple +from typing import Dict, List, Optional, Tuple import extcolors import requests -from loguru import logger from PIL import ImageChops from PIL.Image import ANTIALIAS, Image, new, open from scipy.spatial import KDTree from webcolors import (CSS3_HEX_TO_NAMES, HTML4_HEX_TO_NAMES, hex_to_rgb, rgb_to_hex) +from api import retry from constants import COLOR_LIMIT, COLOR_TOLERANCE, LOW_RES from models import Color - +htmlOverrides: Dict[str, str] = { + "silver": "gray", + "fuschia": "purple", + "blue": "teal", + "aqua": "teal", +} + +cssOverrides: Dict[str, str] = { + "maroon": "red", + "firebrick": "red", + "salmon": "red", + "darkred": "red", + "lightsalmon": "orange", + "orange": "orange", + "darkorange": "orange", + "orangered": "orange", + "coral": "orange", + "mediumseagreen": "green", + "seagreen": "green", + "yellowgreen": "green", + "greenyellow": "green", + "steelblue": "teal", + "lightsteelblue": "teal", + "mediumaquamarine": "teal", + "darkcyan": "teal", + "darkseagreen": "teal", + "paleturquoise": "teal", + "cadetblue": "teal", + "cornflowerblue": "teal", + "lightblue": "teal", + "skyblue": "teal", + "lightskyblue": "teal", + "wienna": "brown", + "chocolate": "brown", + "rosybrown": "brown", + "saddlebrown": "brown", + "darkkhaki": "brown", + "darksalmon": "brown", + "brown": "brown", + "burlywood": "tan", + "bisque": "tan", + "antiquewhite ": "tan", + "blanchedalmond": "tan", + "peru": "tan", + "sandybrown": "tan", + "papayawhip ": "tan", + "tan": "tan", + "navajowhite ": "tan", + "moccasin ": "tan", + "peachpuff": "tan", + "wheat": "tan", + "khaki": "tan", + "darkgray": "gray", + "dimgray": "gray", + "thistle": "gray", + "silver": "gray", + "lightslategray": "gray", + "darkslategray": "gray", + "gainsboro": "gray", + "lightyellow": "yellow", + "lightgoldenrodyellow": "yellow", + "lemonchiffon": "yellow", + "goldenrod": "yellow", + "darkolivegreen": "olive", + "olivedrab": "olive", + "darkslateblue": "navy", + "midnightblue": "navy", + "violet": "purple", + "lightcoral": "purple", + "lightpink": "purple", + "royalblue": "purple", + "seashell": "white", + "snow": "white", +} + + +@retry(delay=1, times=5) def request_image(url: str) -> Image: pic = requests.get(url, stream=True) image = open(pic.raw) @@ -93,6 +169,27 @@ def rgb_to_html(rgb: Tuple[int, int, int]) -> str: return match +def override_color_names(color: Color) -> Color: + + # don't override these colors + if color.html in {"navy", "purple"}: + return color + + css = color.css + if css in cssOverrides.keys(): + new = cssOverrides.get(css) + if new is not None: + color.html = new + + html = color.html + if html in htmlOverrides.keys(): + new = htmlOverrides.get(html) + if new is not None: + color.html = new + + return color + + def extract_colors(image: Image, count: int = COLOR_LIMIT) -> List[Color]: """ @@ -124,25 +221,22 @@ def extract_colors(image: Image, count: int = COLOR_LIMIT) -> List[Color]: # get percent of image with this color percent = round(pixels / total_pixels, 8) - # append it - extracted.append(Color(hex=hex, css=css, html=html, percent=percent)) + # create color + color = Color(hex=hex, css=css, html=html, percent=percent) - # we need to send 5 colors to analogdb - # if we dont have 5 colors, append fillers - num_filler = COLOR_LIMIT - len(extracted) - if num_filler > 0: - filler = Color(hex="null", css="null", html="null", percent=0.0) - for _ in range(num_filler): - extracted.append(filler) + # override color names + color = override_color_names(color) + + # append it + extracted.append(color) return extracted def test_extract_colors(): - import PIL - url = "https://d3i73ktnzbi69i.cloudfront.net/98fe51da-4b04-47db-b529-ce94f2c31219.jpeg" - im = PIL.Image.open(requests.get(url, stream=True).raw) + url = "https://d3i73ktnzbi69i.cloudfront.net/9c995e5b-9307-4f51-a58b-170e41e5fef3.jpeg" + im = request_image(url) extracted = extract_colors(im) for e in extracted: diff --git a/scraper/models.py b/scraper/models.py index 6b97000..8c9c5f4 100644 --- a/scraper/models.py +++ b/scraper/models.py @@ -119,22 +119,6 @@ class AnalogDisplayPost: raw_width: int raw_height: int - c1_hex: str - c1_css: str - c1_percent: float - c2_hex: str - c2_css: str - c2_percent: float - c3_hex: str - c3_css: str - c3_percent: float - c4_hex: str - c4_css: str - c4_percent: float - c5_hex: str - c5_css: str - c5_percent: float - @dataclass class CloudfrontImage: diff --git a/web/components/gallery.js b/web/components/gallery.js index d0d1766..bc6601d 100644 --- a/web/components/gallery.js +++ b/web/components/gallery.js @@ -21,6 +21,7 @@ import { Menu, Radio, Checkbox, + Tooltip, } from "@mantine/core"; import { baseURL } from "../constants.js"; @@ -71,16 +72,21 @@ function filterQueryParams(sort, nsfw, bw, sprocket, text, color) { if (color !== "") { queryParams = queryParams.concat("&color=" + color); - if (color === "black" || color === "gray") { + if (color === "gray") { queryParams = queryParams.concat("&min_color=" + "0.8"); - } - if (color === "white") { - queryParams = queryParams.concat("&min_color=" + "0.6"); - } - if (color === "teal") { + } else if (color === "black") { + queryParams = queryParams.concat("&min_color=" + "0.7"); + } else if (color === "white") { + queryParams = queryParams.concat("&min_color=" + "0.50"); + } else if (color === "teal") { + queryParams = queryParams.concat("&min_color=" + "0.35"); + } else if (color === "olive" || color === "brown") { + queryParams = queryParams.concat("&min_color=" + "0.35"); + } else if (color === "tan") { + queryParams = queryParams.concat("&min_color=" + "0.30"); + } else if (color === "navy" || color === "green") { queryParams = queryParams.concat("&min_color=" + "0.25"); - } - if (color === "navy" || color === "green") { + } else { queryParams = queryParams.concat("&min_color=" + "0.15"); } } @@ -163,10 +169,6 @@ export default function Gallery(props) { onlyIcon = true; } - // const textPlaceholder = () => { - // onlyIcon ? "films, cameras..." : "films, cameras, places..."; - // }; - const textPlaceholder = () => { const placeholder = onlyIcon ? "films, cameras..." @@ -214,114 +216,266 @@ export default function Gallery(props) {