Skip to content

Commit

Permalink
Merge branch 'main' into fix-build-errors
Browse files Browse the repository at this point in the history
  • Loading branch information
ivanleomk authored Oct 5, 2024
2 parents c2d1c8d + 5e6ba05 commit 1e77e87
Showing 1 changed file with 9 additions and 8 deletions.
17 changes: 9 additions & 8 deletions make_sitemap.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import os
import asyncio
import yaml
from typing import Generator, Tuple, Dict, Optional, List
from typing import Optional
from collections.abc import Generator
from openai import AsyncOpenAI
import typer
from rich.console import Console
Expand All @@ -15,7 +16,7 @@

def traverse_docs(
root_dir: str = "docs",
) -> Generator[Tuple[str, str, str], None, None]:
) -> Generator[tuple[str, str, str], None, None]:
"""
Recursively traverse the docs folder and yield the path, content, and content hash of each file.
Expand All @@ -31,7 +32,7 @@ def traverse_docs(
file_path = os.path.join(root, file)
relative_path = os.path.relpath(file_path, root_dir)

with open(file_path, "r", encoding="utf-8") as f:
with open(file_path, encoding="utf-8") as f:
content = f.read()

content_hash = hashlib.md5(content.encode()).hexdigest()
Expand Down Expand Up @@ -101,16 +102,16 @@ async def generate_sitemap(
client = AsyncOpenAI(api_key=api_key)

# Load existing sitemap if it exists
existing_sitemap: Dict[str, Dict[str, str]] = {}
existing_sitemap: dict[str, dict[str, str]] = {}
if os.path.exists(output_file):
with open(output_file, "r", encoding="utf-8") as sitemap_file:
with open(output_file, encoding="utf-8") as sitemap_file:
existing_sitemap = yaml.safe_load(sitemap_file) or {}

sitemap_data: Dict[str, Dict[str, str]] = {}
sitemap_data: dict[str, dict[str, str]] = {}

async def process_file(
path: str, content: str, content_hash: str
) -> Tuple[str, Dict[str, str]]:
) -> tuple[str, dict[str, str]]:
if (
path in existing_sitemap
and existing_sitemap[path].get("hash") == content_hash
Expand All @@ -125,7 +126,7 @@ async def process_file(
)
return path, {"summary": "Failed to generate summary", "hash": content_hash}

files_to_process: List[Tuple[str, str, str]] = list(traverse_docs(root_dir))
files_to_process: list[tuple[str, str, str]] = list(traverse_docs(root_dir))
total_files = len(files_to_process)

with Progress() as progress:
Expand Down

0 comments on commit 1e77e87

Please sign in to comment.