Merge pull request #306 from openzim/upgrade_deps

Upgrade dependencies, including zimscraperlib 4

CHANGELOG

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fix main playlist selection to respect the order of provided playlist IDs (#286)
 - Update `zimui` title dynamically with the selected playlist/video title (#298)
 - Fix `PLAYLISTS` tab not being highlighted when the page is reloaded (#299)
+- Update dependencies, including zimscraperlib 4 (#306)
 
 ## [3.0.1] - 2024-08-13
 

scraper/pyproject.toml

@@ -12,11 +12,11 @@ dependencies = [
   "yt-dlp",                      # youtube-dl should be updated as frequently as possible
   "python-dateutil==2.9.0.post0",
   "jinja2==3.1.4",
-  "zimscraperlib==3.4.0",
+  "zimscraperlib==4.0.0",
   "requests==2.32.3",
   "kiwixstorage==0.9.0",
   "pif==0.8.2",
-  "pydantic==2.7.2",
+  "pydantic==2.8.2",
   "pyhumps==3.8.0",
   "schedule==1.2.2",
 ]
@@ -31,12 +31,12 @@ additional-keywords = ["youtube"]
 
 [project.optional-dependencies]
 scripts = ["invoke==2.2.0"]
-lint = ["black==24.4.2", "ruff==0.4.10"]
-check = ["pyright==1.1.371"]
-test = ["pytest==8.2.2", "coverage==7.5.3"]
+lint = ["black==24.8.0", "ruff==0.6.3"]
+check = ["pyright==1.1.378"]
+test = ["pytest==8.3.2", "coverage==7.6.1"]
 dev = [
-  "pre-commit==3.7.1",
-  "debugpy==1.8.1",
+  "pre-commit==3.8.0",
+  "debugpy==1.8.5",
   "youtube2zim[scripts]",
   "youtube2zim[lint]",
   "youtube2zim[test]",

scraper/src/youtube2zim/scraper.py

@@ -22,19 +22,19 @@
 
 import yt_dlp
 from kiwixstorage import KiwixStorage
-from libzim.writer import IndexData  # type: ignore
 from pif import get_public_ip
 from schedule import every, run_pending
 from zimscraperlib.download import stream_file
-from zimscraperlib.i18n import NotFound, get_language_details
-from zimscraperlib.image.convertion import convert_image
+from zimscraperlib.i18n import NotFoundError, get_language_details
+from zimscraperlib.image.conversion import convert_image
 from zimscraperlib.image.presets import WebpHigh
 from zimscraperlib.image.probing import get_colors, is_hex_color
 from zimscraperlib.image.transformation import resize_image
 from zimscraperlib.inputs import compute_descriptions
 from zimscraperlib.video.presets import VideoMp4Low, VideoWebmLow
-from zimscraperlib.zim import Creator, StaticItem
+from zimscraperlib.zim import Creator
 from zimscraperlib.zim.filesystem import validate_zimfile_creatable
+from zimscraperlib.zim.indexing import IndexData
 from zimscraperlib.zim.metadata import (
     validate_description,
     validate_longdescription,
@@ -87,29 +87,6 @@
 MAXIMUM_YOUTUBEID_LENGTH = 24
 
 
-class CustomIndexData(IndexData):
-    """Custom IndexData class to allow for custom title and content"""
-
-    def __init__(self, title: str, content: str):
-        self.title = title
-        self.content = content
-
-    def has_indexdata(self):
-        return True
-
-    def get_title(self):
-        return self.title
-
-    def get_content(self):
-        return self.content
-
-    def get_keywords(self):
-        return ""
-
-    def get_wordcount(self):
-        return len(self.content.split()) if self.content else 0
-
-
 class Youtube2Zim:
     def __init__(
         self,
@@ -390,8 +367,8 @@ def run(self):
                 disable_metadata_checks=self.disable_metadata_checks,
             )
             self.zim_file.config_metadata(
-                Name=self.name,  # pyright: ignore[reportArgumentType]
-                Language=self.language,  # pyright: ignore[reportArgumentType]
+                Name=self.name,
+                Language=self.language,
                 Title=self.title,
                 Description=self.description,
                 LongDescription=self.long_description,
@@ -872,14 +849,17 @@ def to_subtitle_object(lang) -> Subtitle:
             try:
                 try:
                     subtitle = get_language_details(YOUTUBE_LANG_MAP.get(lang, lang))
-                except NotFound:
+                except NotFoundError:
                     lang_simpl = re.sub(r"^([a-z]{2})-.+$", r"\1", lang)
                     subtitle = get_language_details(
                         YOUTUBE_LANG_MAP.get(lang_simpl, lang_simpl)
                     )
             except Exception:
                 logger.error(f"Failed to get language details for {lang}")
                 raise
+            if not subtitle:
+                logger.error(f"Empty language details retrieved for {lang}")
+                raise Exception("Empty language details")
             return Subtitle(
                 code=lang,
                 name=f"{subtitle['english'].title()} - {subtitle['query']}",
@@ -1294,16 +1274,14 @@ def add_custom_item_to_zim_index(
             f"</head><body></body></html>"
         )
 
-        item = StaticItem(
+        logger.debug(f"Adding {fname} to ZIM index")
+        self.zim_file.add_item_for(
             title=title,
             path="index/" + fname,
             content=bytes(html_content, "utf-8"),
             mimetype="text/html",
+            index_data=IndexData(title=title, content=content),
         )
-        item.get_indexdata = lambda: CustomIndexData(title, content)
-
-        logger.debug(f"Adding {fname} to ZIM index")
-        self.zim_file.add_item(item)
 
     def report_progress(self):
         """report progress to stats file"""

zimui/package.json

@@ -15,43 +15,43 @@
     "format": "prettier --write src/"
   },
   "dependencies": {
-    "@fontsource/roboto": "^5.0.13",
-    "axios": "^1.7.4",
-    "dayjs": "^1.11.11",
-    "pinia": "^2.1.7",
+    "@fontsource/roboto": "^5.0.14",
+    "axios": "^1.7.7",
+    "dayjs": "^1.11.13",
+    "pinia": "^2.2.2",
     "resize-observer-polyfill": "^1.5.1",
     "video.js": "^8.17.3",
-    "vite-plugin-vuetify": "^2.0.3",
-    "vue": "^3.4.21",
-    "vue-router": "^4.3.0",
-    "vuetify": "^3.6.7",
+    "vite-plugin-vuetify": "^2.0.4",
+    "vue": "^3.4.38",
+    "vue-router": "^4.4.3",
+    "vuetify": "^3.7.1",
     "webp-hero": "^0.0.2"
   },
   "devDependencies": {
     "@mdi/font": "^7.4.47",
-    "@rushstack/eslint-patch": "^1.8.0",
+    "@rushstack/eslint-patch": "^1.10.4",
     "@tsconfig/node20": "^20.1.4",
     "@types/jsdom": "^21.1.6",
-    "@types/node": "^20.12.5",
-    "@vitejs/plugin-legacy": "^5.4.1",
-    "@vitejs/plugin-vue": "^5.0.4",
+    "@types/node": "^22.5.2",
+    "@vitejs/plugin-legacy": "^5.4.2",
+    "@vitejs/plugin-vue": "^5.1.3",
     "@vue/eslint-config-prettier": "^9.0.0",
     "@vue/eslint-config-typescript": "^13.0.0",
     "@vue/test-utils": "^2.4.5",
     "@vue/tsconfig": "^0.5.1",
-    "cypress": "^13.11.0",
+    "cypress": "^13.14.1",
     "eslint": "^8.57.0",
-    "eslint-plugin-vue": "^9.23.0",
-    "jsdom": "^24.0.0",
-    "npm-run-all2": "^6.1.2",
-    "ogv": "1.8.9",
-    "prettier": "^3.2.5",
-    "terser": "^5.31.3",
-    "typescript": "~5.4.0",
-    "vite": "^5.2.8",
-    "vite-plugin-static-copy": "^1.0.5",
-    "vitest": "^1.4.0",
-    "vue-tsc": "^2.0.11"
+    "eslint-plugin-vue": "^9.28.0",
+    "jsdom": "^25.0.0",
+    "npm-run-all2": "^6.2.2",
+    "ogv": "1.9.0",
+    "prettier": "^3.3.3",
+    "terser": "^5.31.6",
+    "typescript": "~5.5.4",
+    "vite": "^5.4.2",
+    "vite-plugin-static-copy": "^1.0.6",
+    "vitest": "^2.0.5",
+    "vue-tsc": "^2.1.4"
   },
   "packageManager": "[email protected]"
 }